Add additional removal indicator (#1245)

* add additional removal indicator

* remove param

* pr feedback

Author: pauld-msft

src/Microsoft.ComponentDetection.Common/FileUtilityService.cs

@@ -2,6 +2,7 @@ namespace Microsoft.ComponentDetection.Common;
 
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
 
@@ -39,14 +40,14 @@ public Stream MakeFileStream(string fileName)
     }
 
     /// <inheritdoc />
-    public (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, string removalIndicator)
+    public (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, params string[] removalIndicators)
     {
         // Read all lines from the file and filter out the lines that start with the removal indicator.
         var removedAnyLines = false;
         var linesToKeep = new List<string>();
         foreach (var line in File.ReadLines(fileName))
         {
-            if (line == null || line.Trim().StartsWith(removalIndicator, System.StringComparison.OrdinalIgnoreCase))
+            if (string.IsNullOrEmpty(line) || removalIndicators.Any(removalIndicator => line.Trim().StartsWith(removalIndicator, System.StringComparison.OrdinalIgnoreCase)))
             {
                 removedAnyLines = true;
             }

src/Microsoft.ComponentDetection.Contracts/IFileUtilityService.cs

@@ -47,7 +47,7 @@ public interface IFileUtilityService
     /// Duplicates a file, removing any lines that starts with the given string.
     /// </summary>
     /// <param name="fileName">Path to the file.</param>
-    /// <param name="removalIndicator">The string that indicates a line should be removed.</param>
+    /// <param name="removalIndicators">The strings that indicates a line should be removed.</param>
     /// <returns>Returns the path of the new file, and whether or not one was created.</returns>
-    (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, string removalIndicator);
+    (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, params string[] removalIndicators);
 }

src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj

@@ -1,5 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
+  
     <ItemGroup>
         <PackageReference Include="DotNet.Glob" />
         <PackageReference Include="Microsoft.Extensions.Logging" />
@@ -14,7 +14,7 @@
         <PackageReference Include="System.Reactive" />
         <PackageReference Include="System.Threading.Tasks.Dataflow" />
         <PackageReference Include="Tomlyn.Signed" />
-        <PackageReference Include="Valleysoft.DockerfileModel " />
+        <PackageReference Include="Valleysoft.DockerfileModel" />
 
         <!-- Fix alerts -->
         <PackageReference Include="System.Formats.Asn1" />

src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs

@@ -149,7 +149,7 @@ private async Task<CommandLineExecutionResult> ExecuteCommandAsync(
                 if (this.environmentService.IsEnvironmentVariableValueTrue(PipReportIgnoreFileLevelIndexUrlEnvVar))
                 {
                     // check for --index-url in requirements.txt and remove it from the file, since we want to use PIP_INDEX_URL from the environment.
-                    var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(formattedPath, "--index-url");
+                    var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(formattedPath, "--index-url", "-i");
                     if (createdDuplicate)
                     {
                         var duplicateFileName = Path.GetFileName(duplicateFilePath);

test/Microsoft.ComponentDetection.Common.Tests/FileUtilityServiceTests.cs

@@ -23,16 +23,21 @@ public void DuplicateFileWithoutLines_WithLinesToRemove_ShouldCreateDuplicateFil
 
         // Arrange
         var fileName = $"{directory}/Resources/test-file-duplicate.txt";
-        var removalIndicator = "//REMOVE";
         var expectedDuplicateFilePath = Path.Combine(directory, "Resources", "temp.test-file-duplicate.txt");
 
         // Act
-        var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(fileName, removalIndicator);
+        var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(fileName, "//REMOVE", "//ME");
 
         // Assert
         createdDuplicate.Should().BeTrue();
         duplicateFilePath.Should().Be(expectedDuplicateFilePath);
         File.Exists(expectedDuplicateFilePath).Should().BeTrue();
+
+        var contents = File.ReadAllText(expectedDuplicateFilePath);
+        contents.Should().NotContain("//REMOVE");
+        contents.Should().NotContain("//ME");
+        contents.Should().Contain("hello");
+        contents.Should().Contain("world");
     }
 
     [TestMethod]

test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
     <ItemGroup Label="Project References">
         <ProjectReference Include="..\Microsoft.ComponentDetection.TestsUtilities\Microsoft.ComponentDetection.TestsUtilities.csproj" />
@@ -9,8 +9,8 @@
       <PackageReference Include="FluentAssertions.Analyzers" PrivateAssets="all" />
       <PackageReference Include="Microsoft.Extensions.Logging" />
       <PackageReference Include="System.Threading.Tasks.Dataflow" />
-      <PackageReference Include="MSTest.TestAdapter "/>
-      <PackageReference Include="Microsoft.NET.Test.Sdk "/>
+      <PackageReference Include="MSTest.TestAdapter" />
+      <PackageReference Include="Microsoft.NET.Test.Sdk" />
     </ItemGroup>
 
     <ItemGroup>

test/Microsoft.ComponentDetection.Common.Tests/Resources/test-file-duplicate.txt

@@ -1,3 +1,4 @@
 hello
 //REMOVE
 world
+//ME

test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs

@@ -500,7 +500,7 @@ public async Task PipCommandService_GeneratesReport_WithDuplicate_Async()
         this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("PipReportIgnoreFileLevelIndexUrl")).Returns(true);
         this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("pip", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
 
-        this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny<string>(), "--index-url"))
+        this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny<string>(), "--index-url", "-i"))
             .Returns(("C:/asdf/temp.requirements.txt", true))
             .Verifiable();
         this.fileUtilityService.Setup(x => x.ReadAllTextAsync(It.IsAny<FileInfo>()))
@@ -535,7 +535,7 @@ public async Task PipCommandService_GeneratesReport_WithoutDuplicate_Async()
         this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("PipReportIgnoreFileLevelIndexUrl")).Returns(true);
         this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("pip", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
 
-        this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny<string>(), "--index-url"))
+        this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny<string>(), "--index-url", "-i"))
             .Returns((null, false))
             .Verifiable();
         this.fileUtilityService.Setup(x => x.ReadAllTextAsync(It.IsAny<FileInfo>()))
@@ -590,7 +590,7 @@ public async Task PipCommandService_GeneratesReport_UseFileIndex_Async()
             this.logger.Object);
 
         var (report, reportFile) = await service.GenerateInstallationReportAsync(testPath);
-        this.fileUtilityService.Verify(x => x.DuplicateFileWithoutLines(It.IsAny<string>(), "--index-url"), Times.Never);
+        this.fileUtilityService.Verify(x => x.DuplicateFileWithoutLines(It.IsAny<string>(), "--index-url", "-i"), Times.Never);
         this.commandLineInvokationService.Verify();
     }