Skip to content

Commit 8797033

Browse files
calumgrantcalumgrant
committed
C#: Improvements to buildless extraction, particularly for .NET Core.
1 parent 3767794 commit 8797033

File tree

20 files changed

+1131
-142
lines changed

20 files changed

+1131
-142
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,6 @@
1616

1717
# It's useful (though not required) to be able to unpack codeql in the ql checkout itself
1818
/codeql/
19-
.vscode/settings.json
19+
2020
csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
21+
.vscode

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,19 @@ public AssemblyInfo ResolveReference(string id)
163163
/// </summary>
164164
/// <param name="filepath">The filename to query.</param>
165165
/// <returns>The assembly info.</returns>
166-
public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath];
166+
public AssemblyInfo GetAssemblyInfo(string filepath)
167+
{
168+
if(assemblyInfo.TryGetValue(filepath, out var info))
169+
{
170+
return info;
171+
}
172+
else
173+
{
174+
info = AssemblyInfo.ReadFromFile(filepath);
175+
assemblyInfo.Add(filepath, info);
176+
return info;
177+
}
178+
}
167179

168180
// List of pending DLLs to index.
169181
readonly List<string> dlls = new List<string>();

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs‎

Lines changed: 105 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
using System.Runtime.InteropServices;
66
using Semmle.Util;
77
using Semmle.Extraction.CSharp.Standalone;
8+
using System.Threading.Tasks;
9+
using System.Collections.Concurrent;
810

911
namespace Semmle.BuildAnalyser
1012
{
@@ -56,6 +58,7 @@ class BuildAnalysis : IBuildAnalysis
5658
int failedProjects, succeededProjects;
5759
readonly string[] allSources;
5860
int conflictedReferences = 0;
61+
object mutex = new object();
5962

6063
/// <summary>
6164
/// Performs a C# build analysis.
@@ -64,6 +67,8 @@ class BuildAnalysis : IBuildAnalysis
6467
/// <param name="progress">Display of analysis progress.</param>
6568
public BuildAnalysis(Options options, IProgressMonitor progress)
6669
{
70+
var startTime = DateTime.Now;
71+
6772
progressMonitor = progress;
6873
sourceDir = new DirectoryInfo(options.SrcDir);
6974

@@ -74,38 +79,51 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
7479
Where(d => !options.ExcludesFile(d)).
7580
ToArray();
7681

77-
var dllDirNames = options.DllDirs.Select(Path.GetFullPath);
82+
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
83+
PackageDirectory = TemporaryDirectory.CreateTempDirectory(sourceDir.FullName, progressMonitor);
7884

7985
if (options.UseNuGet)
8086
{
81-
nuget = new NugetPackages(sourceDir.FullName);
82-
ReadNugetFiles();
83-
dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory));
87+
try
88+
{
89+
nuget = new NugetPackages(sourceDir.FullName, PackageDirectory);
90+
ReadNugetFiles();
91+
}
92+
catch(FileNotFoundException)
93+
{
94+
progressMonitor.MissingNuGet();
95+
}
8496
}
8597

8698
// Find DLLs in the .Net Framework
8799
if (options.ScanNetFrameworkDlls)
88100
{
89-
dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1));
101+
dllDirNames.Add(Runtime.Runtimes.First());
90102
}
103+
104+
{
105+
using var renamer1 = new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories));
106+
using var renamer2 = new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories));
91107

92-
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
108+
var solutions = options.SolutionFile != null ?
109+
new[] { options.SolutionFile } :
110+
sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);
93111

94-
// Analyse all .csproj files in the source tree.
95-
if (options.SolutionFile != null)
96-
{
97-
AnalyseSolution(options.SolutionFile);
98-
}
99-
else if (options.AnalyseCsProjFiles)
100-
{
101-
AnalyseProjectFiles();
112+
113+
RestoreSolutions(solutions);
114+
dllDirNames.Add(PackageDirectory.DirInfo.FullName);
115+
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
116+
AnalyseSolutions(solutions);
117+
118+
usedReferences = new HashSet<string>(assemblyCache.AllAssemblies.Select(a => a.Filename));
102119
}
103120

104121
if (!options.AnalyseCsProjFiles)
105122
{
106123
usedReferences = new HashSet<string>(assemblyCache.AllAssemblies.Select(a => a.Filename));
107124
}
108125

126+
109127
ResolveConflicts();
110128

111129
if (options.UseMscorlib)
@@ -133,6 +151,8 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
133151
conflictedReferences,
134152
succeededProjects + failedProjects,
135153
failedProjects);
154+
155+
Console.WriteLine($"Build analysis completed in {DateTime.Now - startTime}");
136156
}
137157

138158
/// <summary>
@@ -183,7 +203,8 @@ void ReadNugetFiles()
183203
/// <param name="reference">The filename of the reference.</param>
184204
void UseReference(string reference)
185205
{
186-
usedReferences.Add(reference);
206+
lock (mutex)
207+
usedReferences.Add(reference);
187208
}
188209

189210
/// <summary>
@@ -194,11 +215,13 @@ void UseSource(FileInfo sourceFile)
194215
{
195216
if (sourceFile.Exists)
196217
{
197-
usedSources.Add(sourceFile.FullName);
218+
lock(mutex)
219+
usedSources.Add(sourceFile.FullName);
198220
}
199221
else
200222
{
201-
missingSources.Add(sourceFile.FullName);
223+
lock(mutex)
224+
missingSources.Add(sourceFile.FullName);
202225
}
203226
}
204227

@@ -236,77 +259,100 @@ void UseSource(FileInfo sourceFile)
236259
/// <param name="projectFile">The project file making the reference.</param>
237260
void UnresolvedReference(string id, string projectFile)
238261
{
239-
unresolvedReferences[id] = projectFile;
262+
lock(mutex)
263+
unresolvedReferences[id] = projectFile;
240264
}
241265

242-
/// <summary>
243-
/// Performs an analysis of all .csproj files.
244-
/// </summary>
245-
void AnalyseProjectFiles()
246-
{
247-
AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories));
248-
}
266+
TemporaryDirectory PackageDirectory;
249267

250268
/// <summary>
251269
/// Reads all the source files and references from the given list of projects.
252270
/// </summary>
253271
/// <param name="projectFiles">The list of projects to analyse.</param>
254-
void AnalyseProjectFiles(FileInfo[] projectFiles)
272+
void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
255273
{
256-
progressMonitor.AnalysingProjectFiles(projectFiles.Count());
257-
258274
foreach (var proj in projectFiles)
275+
AnalyseProject(proj);
276+
}
277+
278+
void AnalyseProject(FileInfo project)
279+
{
280+
if(!project.Exists)
259281
{
260-
try
261-
{
262-
var csProj = new CsProjFile(proj);
282+
progressMonitor.MissingProject(project.FullName);
283+
return;
284+
}
285+
286+
try
287+
{
288+
var csProj = new CsProjFile(project);
263289

264-
foreach (var @ref in csProj.References)
290+
foreach (var @ref in csProj.References)
291+
{
292+
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
293+
if (!resolved.Valid)
265294
{
266-
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
267-
if (!resolved.Valid)
268-
{
269-
UnresolvedReference(@ref, proj.FullName);
270-
}
271-
else
272-
{
273-
UseReference(resolved.Filename);
274-
}
295+
UnresolvedReference(@ref, project.FullName);
275296
}
276-
277-
foreach (var src in csProj.Sources)
297+
else
278298
{
279-
// Make a note of which source files the projects use.
280-
// This information doesn't affect the build but is dumped
281-
// as diagnostic output.
282-
UseSource(new FileInfo(src));
299+
UseReference(resolved.Filename);
283300
}
284-
++succeededProjects;
285301
}
286-
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
302+
303+
foreach (var src in csProj.Sources)
287304
{
288-
++failedProjects;
289-
progressMonitor.FailedProjectFile(proj.FullName, ex.Message);
305+
// Make a note of which source files the projects use.
306+
// This information doesn't affect the build but is dumped
307+
// as diagnostic output.
308+
UseSource(new FileInfo(src));
290309
}
310+
311+
++succeededProjects;
312+
}
313+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
314+
{
315+
++failedProjects;
316+
progressMonitor.FailedProjectFile(project.FullName, ex.Message);
291317
}
318+
292319
}
293320

294321
/// <summary>
295322
/// Delete packages directory.
296323
/// </summary>
297324
public void Cleanup()
298325
{
299-
if (nuget != null) nuget.Cleanup(progressMonitor);
326+
PackageDirectory?.Cleanup();
300327
}
301328

302-
/// <summary>
303-
/// Analyse all project files in a given solution only.
304-
/// </summary>
305-
/// <param name="solutionFile">The filename of the solution.</param>
306-
public void AnalyseSolution(string solutionFile)
329+
void Restore(string projectOrSolution)
307330
{
308-
var sln = new SolutionFile(solutionFile);
309-
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray());
331+
int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName);
332+
if (exit != 0)
333+
progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
334+
}
335+
336+
public void RestoreSolutions(IEnumerable<string> solutions)
337+
{
338+
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
339+
}
340+
341+
public void AnalyseSolutions(IEnumerable<string> solutions)
342+
{
343+
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 } , solutionFile =>
344+
{
345+
try
346+
{
347+
var sln = new SolutionFile(solutionFile);
348+
progressMonitor.AnalysingSolution(solutionFile);
349+
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists).ToArray());
350+
}
351+
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
352+
{
353+
progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
354+
}
355+
});
310356
}
311357
}
312358
}

0 commit comments

Comments
 (0)