Skip to content

C#: Improvements to buildless extraction #3136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hvitved merged 5 commits into from
Apr 7, 2020
Merged

C#: Improvements to buildless extraction #3136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8797033
C#: Improvements to buildless extraction, particularly for .NET Core.
calumgrant Jan 7, 2020
71e0dc0
C#: General code tidy.
calumgrant Mar 26, 2020
b94b4b7
C#: Fix tests
calumgrant Mar 26, 2020
9481fad
C#: Address review comments.
calumgrant Apr 2, 2020
adde52d
C#: Add missing files
calumgrant Apr 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,6 @@

# It's useful (though not required) to be able to unpack codeql in the ql checkout itself
/codeql/
.vscode/settings.json

csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
.vscode
14 changes: 13 additions & 1 deletion csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,19 @@ public AssemblyInfo ResolveReference(string id)
/// </summary>
/// <param name="filepath">The filename to query.</param>
/// <returns>The assembly info.</returns>
public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath];
public AssemblyInfo GetAssemblyInfo(string filepath)
{
if(assemblyInfo.TryGetValue(filepath, out var info))
{
return info;
}
else
{
info = AssemblyInfo.ReadFromFile(filepath);
assemblyInfo.Add(filepath, info);
return info;
}
}

// List of pending DLLs to index.
readonly List<string> dlls = new List<string>();
Expand Down
241 changes: 147 additions & 94 deletions csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
using System;
using Semmle.Util;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using Semmle.Util;
using Semmle.Extraction.CSharp.Standalone;
using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Text;
using System.Security.Cryptography;

namespace Semmle.BuildAnalyser
{
Expand Down Expand Up @@ -43,19 +46,18 @@ interface IBuildAnalysis
/// <summary>
/// Main implementation of the build analysis.
/// </summary>
class BuildAnalysis : IBuildAnalysis
class BuildAnalysis : IBuildAnalysis, IDisposable
{
readonly AssemblyCache assemblyCache;
readonly NugetPackages nuget;
readonly IProgressMonitor progressMonitor;
HashSet<string> usedReferences = new HashSet<string>();
readonly HashSet<string> usedSources = new HashSet<string>();
readonly HashSet<string> missingSources = new HashSet<string>();
readonly Dictionary<string, string> unresolvedReferences = new Dictionary<string, string>();
readonly DirectoryInfo sourceDir;
int failedProjects, succeededProjects;
readonly string[] allSources;
int conflictedReferences = 0;
private readonly AssemblyCache assemblyCache;
private readonly NugetPackages nuget;
private readonly IProgressMonitor progressMonitor;
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
Copy link
Copy Markdown
Contributor

@hvitved hvitved Apr 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps we should add this to Semmle.Util and use instead:

using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;

namespace Semmle.Util
{
    public class ConcurrentHashSet<T> : IEnumerable<T>
    {
        private readonly ConcurrentDictionary<T, object> dict;
        public ConcurrentHashSet()
        {
            dict = new ConcurrentDictionary<T, object>();
        }

        public void Add(T t) => dict[t] = null;

        public bool Contains(T t) => dict.ContainsKey(t);

        public IEnumerator<T> GetEnumerator() => dict.Keys.GetEnumerator();

        IEnumerator IEnumerable.GetEnumerator() => dict.Keys.GetEnumerator();
    }
}

Copy link
Copy Markdown
Contributor Author

@calumgrant calumgrant Apr 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My reservation with this is that this would be a half-baked implementation, that should also implement

[System.Serializable]
public class ConcurrentHashSet<T> : System.Collections.Generic.ICollection<T>, System.Collections.Generic.IEnumerable<T>, System.Collections.Generic.IReadOnlyCollection<T>, System.Collections.Generic.ISet<T>, System.Runtime.Serialization.IDeserializationCallback, System.Runtime.Serialization.ISerializable

So we should really just reference the nuget package ConcurrentHashSet which I presume is totally correct and bug-free.....

Then it means updating resources/lib which is a bit of a pain.

Copy link
Copy Markdown
Contributor

@hvitved hvitved Apr 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think a half-baked implementation suffices, as we only need to add the things that we actually need. If we ever need, say, to use it as an ICollection<T>, then we can implement it then.

Copy link
Copy Markdown
Contributor Author

@calumgrant calumgrant Apr 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's literally one place where we use ConcurrentDictionary instead of ConcurrentHashSet so I'm really not convinced it's worth the effort. Particularly because we'd need a unit test suite for the new ConcurrentHashSet class.

Copy link
Copy Markdown
Contributor

@hvitved hvitved Apr 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK 👍

private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
private readonly DirectoryInfo sourceDir;
private int failedProjects, succeededProjects;
private readonly string[] allSources;
private int conflictedReferences = 0;

/// <summary>
/// Performs a C# build analysis.
Expand All @@ -64,6 +66,8 @@ class BuildAnalysis : IBuildAnalysis
/// <param name="progress">Display of analysis progress.</param>
public BuildAnalysis(Options options, IProgressMonitor progress)
{
var startTime = DateTime.Now;

progressMonitor = progress;
sourceDir = new DirectoryInfo(options.SrcDir);

Expand All @@ -74,36 +78,43 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
Where(d => !options.ExcludesFile(d)).
ToArray();

var dllDirNames = options.DllDirs.Select(Path.GetFullPath);
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
PackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));

if (options.UseNuGet)
{
nuget = new NugetPackages(sourceDir.FullName);
ReadNugetFiles();
dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory));
try
{
nuget = new NugetPackages(sourceDir.FullName, PackageDirectory);
ReadNugetFiles();
}
catch(FileNotFoundException)
{
progressMonitor.MissingNuGet();
}
}

// Find DLLs in the .Net Framework
if (options.ScanNetFrameworkDlls)
{
dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1));
dllDirNames.Add(Runtime.Runtimes.First());
}

assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);

// Analyse all .csproj files in the source tree.
if (options.SolutionFile != null)
{
AnalyseSolution(options.SolutionFile);
}
else if (options.AnalyseCsProjFiles)
// These files can sometimes prevent `dotnet restore` from working correctly.
using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
using (new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)))
{
AnalyseProjectFiles();
}
var solutions = options.SolutionFile != null ?
new[] { options.SolutionFile } :
sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);

if (!options.AnalyseCsProjFiles)
{
usedReferences = new HashSet<string>(assemblyCache.AllAssemblies.Select(a => a.Filename));
RestoreSolutions(solutions);
dllDirNames.Add(PackageDirectory.DirInfo.FullName);
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
AnalyseSolutions(solutions);

foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
UseReference(filename);
}

ResolveConflicts();
Expand All @@ -114,7 +125,7 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
}

// Output the findings
foreach (var r in usedReferences)
foreach (var r in usedReferences.Keys)
{
progressMonitor.ResolvedReference(r);
}
Expand All @@ -132,7 +143,27 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
UnresolvedReferences.Count(),
conflictedReferences,
succeededProjects + failedProjects,
failedProjects);
failedProjects,
DateTime.Now - startTime);
}

/// <summary>
/// Computes a unique temp directory for the packages associated
/// with this source tree. Use a SHA1 of the directory name.
/// </summary>
/// <param name="srcDir"></param>
/// <returns>The full path of the temp directory.</returns>
private static string ComputeTempDirectory(string srcDir)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);

using var sha1 = new SHA1CryptoServiceProvider();
var sha = sha1.ComputeHash(bytes);
var sb = new StringBuilder();
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);

return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
}

/// <summary>
Expand All @@ -143,7 +174,7 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
void ResolveConflicts()
{
var sortedReferences = usedReferences.
Select(r => assemblyCache.GetAssemblyInfo(r)).
Select(r => assemblyCache.GetAssemblyInfo(r.Key)).
OrderBy(r => r.Version).
ToArray();

Expand All @@ -154,7 +185,9 @@ void ResolveConflicts()
finalAssemblyList[r.Name] = r;

// Update the used references list
usedReferences = new HashSet<string>(finalAssemblyList.Select(r => r.Value.Filename));
usedReferences.Clear();
foreach (var r in finalAssemblyList.Select(r => r.Value.Filename))
UseReference(r);

// Report the results
foreach (var r in sortedReferences)
Expand Down Expand Up @@ -183,7 +216,7 @@ void ReadNugetFiles()
/// <param name="reference">The filename of the reference.</param>
void UseReference(string reference)
{
usedReferences.Add(reference);
usedReferences[reference] = true;
}

/// <summary>
Expand All @@ -192,25 +225,18 @@ void UseReference(string reference)
/// <param name="sourceFile">The source file.</param>
void UseSource(FileInfo sourceFile)
{
if (sourceFile.Exists)
{
usedSources.Add(sourceFile.FullName);
}
else
{
missingSources.Add(sourceFile.FullName);
}
sources[sourceFile.FullName] = sourceFile.Exists;
Copy link
Copy Markdown
Contributor

@hvitved hvitved Apr 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice rewrite.

}

/// <summary>
/// The list of resolved reference files.
/// </summary>
public IEnumerable<string> ReferenceFiles => this.usedReferences;
public IEnumerable<string> ReferenceFiles => this.usedReferences.Keys;

/// <summary>
/// The list of source files used in projects.
/// </summary>
public IEnumerable<string> ProjectSourceFiles => usedSources;
public IEnumerable<string> ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);

/// <summary>
/// All of the source files in the source directory.
Expand All @@ -226,7 +252,7 @@ void UseSource(FileInfo sourceFile)
/// List of source files which were mentioned in project files but
/// do not exist on the file system.
/// </summary>
public IEnumerable<string> MissingSourceFiles => missingSources;
public IEnumerable<string> MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key);

/// <summary>
/// Record that a particular reference couldn't be resolved.
Expand All @@ -239,74 +265,101 @@ void UnresolvedReference(string id, string projectFile)
unresolvedReferences[id] = projectFile;
}

/// <summary>
/// Performs an analysis of all .csproj files.
/// </summary>
void AnalyseProjectFiles()
{
AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories));
}
readonly TemporaryDirectory PackageDirectory;

/// <summary>
/// Reads all the source files and references from the given list of projects.
/// </summary>
/// <param name="projectFiles">The list of projects to analyse.</param>
void AnalyseProjectFiles(FileInfo[] projectFiles)
void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
{
progressMonitor.AnalysingProjectFiles(projectFiles.Count());

foreach (var proj in projectFiles)
AnalyseProject(proj);
}

void AnalyseProject(FileInfo project)
{
if(!project.Exists)
{
try
{
var csProj = new CsProjFile(proj);
progressMonitor.MissingProject(project.FullName);
return;
}

try
{
var csProj = new CsProjFile(project);

foreach (var @ref in csProj.References)
foreach (var @ref in csProj.References)
{
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
if (!resolved.Valid)
{
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
if (!resolved.Valid)
{
UnresolvedReference(@ref, proj.FullName);
}
else
{
UseReference(resolved.Filename);
}
UnresolvedReference(@ref, project.FullName);
}

foreach (var src in csProj.Sources)
else
{
// Make a note of which source files the projects use.
// This information doesn't affect the build but is dumped
// as diagnostic output.
UseSource(new FileInfo(src));
UseReference(resolved.Filename);
}
++succeededProjects;
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]

foreach (var src in csProj.Sources)
{
++failedProjects;
progressMonitor.FailedProjectFile(proj.FullName, ex.Message);
// Make a note of which source files the projects use.
// This information doesn't affect the build but is dumped
// as diagnostic output.
UseSource(new FileInfo(src));
}

++succeededProjects;
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
++failedProjects;
progressMonitor.FailedProjectFile(project.FullName, ex.Message);
}

}

/// <summary>
/// Delete packages directory.
/// </summary>
public void Cleanup()
void Restore(string projectOrSolution)
{
int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName);
switch(exit)
{
case 0:
case 1:
// No errors
Comment thread
hvitved marked this conversation as resolved.
break;
default:
progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
break;
}
}

public void RestoreSolutions(IEnumerable<string> solutions)
{
if (nuget != null) nuget.Cleanup(progressMonitor);
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
}

/// <summary>
/// Analyse all project files in a given solution only.
/// </summary>
/// <param name="solutionFile">The filename of the solution.</param>
public void AnalyseSolution(string solutionFile)
public void AnalyseSolutions(IEnumerable<string> solutions)
{
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 } , solutionFile =>
{
try
{
var sln = new SolutionFile(solutionFile);
progressMonitor.AnalysingSolution(solutionFile);
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists));
}
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
{
progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
}
});
}

public void Dispose()
{
var sln = new SolutionFile(solutionFile);
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray());
PackageDirectory?.Dispose();
}
}
}
Loading