Skip to content

Commit 99ffda7

Browse files
brettfobrettfo
committed
add MSBuild binary log (.binlog) component detector
1 parent 8360853 commit 99ffda7

File tree

7 files changed

+699
-5
lines changed

7 files changed

+699
-5
lines changed

‎Directory.Packages.props

+3
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,12 @@
2121
<PackageVersion Include="MinVer" Version="5.0.0" />
2222
<PackageVersion Include="Moq" Version="4.18.4" />
2323
<PackageVersion Include="morelinq" Version="4.2.0" />
24+
<PackageVersion Include="MSBuild.StructuredLogger" Version="2.2.317" />
2425
<PackageVersion Include="MSTest.TestFramework" Version="3.5.1" />
2526
<PackageVersion Include="MSTest.Analyzers" Version="3.5.1" />
2627
<PackageVersion Include="MSTest.TestAdapter" Version="3.5.1" />
28+
<PackageVersion Include="Microsoft.Build.Framework" Version="17.5.0" />
29+
<PackageVersion Include="Microsoft.Build.Locator" Version="1.6.1" />
2730
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
2831
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
2932
<PackageVersion Include="Newtonsoft.Json.Schema" Version="3.0.16" />

‎src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj

+3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@
99
<PackageReference Include="Polly" />
1010
<PackageReference Include="SemanticVersioning" />
1111
<PackageReference Include="yamldotnet" />
12+
<PackageReference Include="Microsoft.Build.Framework" ExcludeAssets="Runtime" PrivateAssets="All" />
13+
<PackageReference Include="Microsoft.Build.Locator" />
1214
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
15+
<PackageReference Include="MSBuild.StructuredLogger" />
1316
<PackageReference Include="Newtonsoft.Json" />
1417
<PackageReference Include="System.Reactive" />
1518
<PackageReference Include="System.Threading.Tasks.Dataflow" />

‎src/Microsoft.ComponentDetection.Detectors/nuget/NuGetMSBuildBinaryLogComponentDetector.cs

+212
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
namespace Microsoft.ComponentDetection.Detectors.NuGet;
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
using System.Threading;
7+
using Microsoft.Build.Locator;
8+
using Microsoft.Build.Logging.StructuredLogger;
9+
using Microsoft.ComponentDetection.Contracts;
10+
using Microsoft.ComponentDetection.Contracts.Internal;
11+
using Microsoft.ComponentDetection.Contracts.TypedComponent;
12+
using Microsoft.Extensions.Logging;
13+
14+
using Task = System.Threading.Tasks.Task;
15+
16+
public class NuGetMSBuildBinaryLogComponentDetector : FileComponentDetector
17+
{
18+
private static readonly HashSet<string> TopLevelPackageItemNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
19+
{
20+
"PackageReference",
21+
};
22+
23+
// the items listed below represent collection names that NuGet will resolve a package into, along with the metadata value names to get the package name and version
24+
private static readonly Dictionary<string, (string NameMetadata, string VersionMetadata)> ResolvedPackageItemNames = new Dictionary<string, (string, string)>(StringComparer.OrdinalIgnoreCase)
25+
{
26+
["NativeCopyLocalItems"] = ("NuGetPackageId", "NuGetPackageVersion"),
27+
["ResourceCopyLocalItems"] = ("NuGetPackageId", "NuGetPackageVersion"),
28+
["RuntimeCopyLocalItems"] = ("NuGetPackageId", "NuGetPackageVersion"),
29+
["ResolvedAnalyzers"] = ("NuGetPackageId", "NuGetPackageVersion"),
30+
["_PackageDependenciesDesignTime"] = ("Name", "Version"),
31+
};
32+
33+
private static bool isMSBuildRegistered;
34+
35+
public NuGetMSBuildBinaryLogComponentDetector(
36+
IObservableDirectoryWalkerFactory walkerFactory,
37+
ILogger<NuGetMSBuildBinaryLogComponentDetector> logger)
38+
{
39+
this.Scanner = walkerFactory;
40+
this.Logger = logger;
41+
}
42+
43+
public override string Id { get; } = "NuGetMSBuildBinaryLog";
44+
45+
public override IEnumerable<string> Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.NuGet) };
46+
47+
public override IList<string> SearchPatterns { get; } = new List<string> { "*.binlog" };
48+
49+
public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = new[] { ComponentType.NuGet };
50+
51+
public override int Version { get; } = 1;
52+
53+
private static void ProcessResolvedPackageReference(Dictionary<string, HashSet<string>> topLevelDependencies, Dictionary<string, Dictionary<string, string>> projectResolvedDependencies, NamedNode node)
54+
{
55+
var doRemoveOperation = node is RemoveItem;
56+
var doAddOperation = node is AddItem;
57+
if (TopLevelPackageItemNames.Contains(node.Name))
58+
{
59+
var projectEvaluation = node.GetNearestParent<ProjectEvaluation>();
60+
if (projectEvaluation is not null)
61+
{
62+
foreach (var child in node.Children.OfType<Item>())
63+
{
64+
var packageName = child.Name;
65+
if (!topLevelDependencies.TryGetValue(projectEvaluation.ProjectFile, out var topLevel))
66+
{
67+
topLevel = new(StringComparer.OrdinalIgnoreCase);
68+
topLevelDependencies[projectEvaluation.ProjectFile] = topLevel;
69+
}
70+
71+
if (doRemoveOperation)
72+
{
73+
topLevel.Remove(packageName);
74+
}
75+
76+
if (doAddOperation)
77+
{
78+
topLevel.Add(packageName);
79+
}
80+
}
81+
}
82+
}
83+
else if (ResolvedPackageItemNames.TryGetValue(node.Name, out var metadataNames))
84+
{
85+
var nameMetadata = metadataNames.NameMetadata;
86+
var versionMetadata = metadataNames.VersionMetadata;
87+
var originalProject = node.GetNearestParent<Project>();
88+
if (originalProject is not null)
89+
{
90+
foreach (var child in node.Children.OfType<Item>())
91+
{
92+
var packageName = GetChildMetadataValue(child, nameMetadata);
93+
var packageVersion = GetChildMetadataValue(child, versionMetadata);
94+
if (packageName is not null && packageVersion is not null)
95+
{
96+
var project = originalProject;
97+
while (project is not null)
98+
{
99+
if (!projectResolvedDependencies.TryGetValue(project.ProjectFile, out var projectDependencies))
100+
{
101+
projectDependencies = new(StringComparer.OrdinalIgnoreCase);
102+
projectResolvedDependencies[project.ProjectFile] = projectDependencies;
103+
}
104+
105+
if (doRemoveOperation)
106+
{
107+
projectDependencies.Remove(packageName);
108+
}
109+
110+
if (doAddOperation)
111+
{
112+
projectDependencies[packageName] = packageVersion;
113+
}
114+
115+
project = project.GetNearestParent<Project>();
116+
}
117+
}
118+
}
119+
}
120+
}
121+
}
122+
123+
private static string GetChildMetadataValue(TreeNode node, string metadataItemName)
124+
{
125+
var metadata = node.Children.OfType<Metadata>();
126+
var metadataValue = metadata.FirstOrDefault(m => m.Name.Equals(metadataItemName, StringComparison.OrdinalIgnoreCase))?.Value;
127+
return metadataValue;
128+
}
129+
130+
protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
131+
{
132+
try
133+
{
134+
if (!isMSBuildRegistered)
135+
{
136+
// this must happen once per process, and never again
137+
var defaultInstance = MSBuildLocator.QueryVisualStudioInstances().First();
138+
MSBuildLocator.RegisterInstance(defaultInstance);
139+
isMSBuildRegistered = true;
140+
}
141+
142+
var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(processRequest.ComponentStream.Location);
143+
var buildRoot = BinaryLog.ReadBuild(processRequest.ComponentStream.Stream);
144+
this.RecordLockfileVersion(buildRoot.FileFormatVersion);
145+
this.ProcessBinLog(buildRoot, singleFileComponentRecorder);
146+
}
147+
catch (Exception e)
148+
{
149+
// If something went wrong, just ignore the package
150+
this.Logger.LogError(e, "Failed to process MSBuild binary log {BinLogFile}", processRequest.ComponentStream.Location);
151+
}
152+
153+
return Task.CompletedTask;
154+
}
155+
156+
protected override Task OnDetectionFinishedAsync()
157+
{
158+
return Task.CompletedTask;
159+
}
160+
161+
private void ProcessBinLog(Build buildRoot, ISingleFileComponentRecorder componentRecorder)
162+
{
163+
// maps a project path to a set of resolved dependencies
164+
var projectTopLevelDependencies = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
165+
var projectResolvedDependencies = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
166+
buildRoot.VisitAllChildren<BaseNode>(node =>
167+
{
168+
switch (node)
169+
{
170+
case NamedNode namedNode when namedNode is AddItem or RemoveItem:
171+
ProcessResolvedPackageReference(projectTopLevelDependencies, projectResolvedDependencies, namedNode);
172+
break;
173+
default:
174+
break;
175+
}
176+
});
177+
178+
// dependencies were resolved per project, we need to re-arrange them to be per package/version
179+
var projectsPerPackage = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
180+
foreach (var projectPath in projectResolvedDependencies.Keys)
181+
{
182+
var projectDependencies = projectResolvedDependencies[projectPath];
183+
foreach (var (packageName, packageVersion) in projectDependencies)
184+
{
185+
var key = $"{packageName}/{packageVersion}";
186+
if (!projectsPerPackage.TryGetValue(key, out var projectPaths))
187+
{
188+
projectPaths = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
189+
projectsPerPackage[key] = projectPaths;
190+
}
191+
192+
projectPaths.Add(projectPath);
193+
}
194+
}
195+
196+
// report it all
197+
foreach (var (packageNameAndVersion, projectPaths) in projectsPerPackage)
198+
{
199+
var parts = packageNameAndVersion.Split('/', 2);
200+
var packageName = parts[0];
201+
var packageVersion = parts[1];
202+
var component = new NuGetComponent(packageName, packageVersion);
203+
var libraryComponent = new DetectedComponent(component);
204+
foreach (var projectPath in projectPaths)
205+
{
206+
libraryComponent.FilePaths.Add(projectPath);
207+
}
208+
209+
componentRecorder.RegisterUsage(libraryComponent);
210+
}
211+
}
212+
}

‎src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs

+1
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s
105105
services.AddSingleton<IComponentDetector, NuGetComponentDetector>();
106106
services.AddSingleton<IComponentDetector, NuGetPackagesConfigDetector>();
107107
services.AddSingleton<IComponentDetector, NuGetProjectModelProjectCentricComponentDetector>();
108+
services.AddSingleton<IComponentDetector, NuGetMSBuildBinaryLogComponentDetector>();
108109

109110
// PIP
110111
services.AddSingleton<IPyPiClient, PyPiClient>();

0 commit comments

Comments
 (0)