Merge pull request #20832 from michaelnebel/csharp/dependencycaching

C#:  Add extractor option for the dependency directory in BMN.

Author: michaelnebel

csharp/codeql-extractor.yml

@@ -74,3 +74,8 @@ options:
       [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
       This option only works when `--build-mode none` is also specified.
     type: array
+  buildless_dependency_dir:
+    title: The path where buildless (standalone) extraction should keep dependencies.
+    description: >
+      If set, the buildless (standalone) extractor will store dependencies in this directory.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs

@@ -0,0 +1,62 @@
+using System;
+using System.IO;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// A directory used for storing fetched dependencies.
+    /// When a specific directory is set via the dependency directory extractor option,
+    /// we store dependencies in that directory for caching purposes.
+    /// Otherwise, we create a temporary directory that is deleted upon disposal.
+    /// </summary>
+    public sealed class DependencyDirectory : IDisposable
+    {
+        private readonly string userReportedDirectoryPurpose;
+        private readonly ILogger logger;
+        private readonly bool attemptCleanup;
+
+        public DirectoryInfo DirInfo { get; }
+
+        public DependencyDirectory(string subfolderName, string userReportedDirectoryPurpose, ILogger logger)
+        {
+            this.logger = logger;
+            this.userReportedDirectoryPurpose = userReportedDirectoryPurpose;
+
+            string path;
+            if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir)
+            {
+                path = dir;
+                attemptCleanup = false;
+            }
+            else
+            {
+                path = FileUtils.GetTemporaryWorkingDirectory(out _);
+                attemptCleanup = true;
+            }
+            DirInfo = new DirectoryInfo(Path.Join(path, subfolderName));
+            DirInfo.Create();
+        }
+
+        public void Dispose()
+        {
+            if (!attemptCleanup)
+            {
+                logger.LogInfo($"Keeping {userReportedDirectoryPurpose} directory {DirInfo.FullName} for possible caching purposes.");
+                return;
+            }
+
+            try
+            {
+                DirInfo.Delete(true);
+            }
+            catch (Exception exc)
+            {
+                logger.LogInfo($"Couldn't delete {userReportedDirectoryPurpose} directory {exc.Message}");
+            }
+        }
+
+        public override string ToString() => DirInfo.FullName;
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -24,16 +24,16 @@ internal class NugetExeWrapper : IDisposable
         private readonly FileProvider fileProvider;
 
         /// <summary>
-        /// The computed packages directory.
-        /// This will be in the Temp location
+        /// The packages directory.
+        /// This will be in the user-specified or computed Temp location
         /// so as to not trample the source tree.
         /// </summary>
-        private readonly TemporaryDirectory packageDirectory;
+        private readonly DependencyDirectory packageDirectory;
 
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
+        public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
         {
             this.fileProvider = fileProvider;
             this.packageDirectory = packageDirectory;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -24,12 +24,12 @@ internal sealed partial class NugetPackageRestorer : IDisposable
         private readonly IDotNet dotnet;
         private readonly DependabotProxy? dependabotProxy;
         private readonly IDiagnosticsWriter diagnosticsWriter;
-        private readonly TemporaryDirectory legacyPackageDirectory;
-        private readonly TemporaryDirectory missingPackageDirectory;
+        private readonly DependencyDirectory legacyPackageDirectory;
+        private readonly DependencyDirectory missingPackageDirectory;
         private readonly ILogger logger;
         private readonly ICompilationInfoContainer compilationInfoContainer;
 
-        public TemporaryDirectory PackageDirectory { get; }
+        public DependencyDirectory PackageDirectory { get; }
 
         public NugetPackageRestorer(
             FileProvider fileProvider,
@@ -48,9 +48,9 @@ public NugetPackageRestorer(
             this.logger = logger;
             this.compilationInfoContainer = compilationInfoContainer;
 
-            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger);
-            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger);
-            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger);
+            PackageDirectory = new DependencyDirectory("packages", "package", logger);
+            legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger);
+            missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger);
         }
 
         public string? TryRestore(string package)

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -76,5 +76,14 @@ public static IEnumerable<string> GetURLs(string name)
         {
             return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_BASE_METADATA_OUT");
         }
+
+        /// <summary>
+        /// If set, returns the directory where buildless dependencies should be stored.
+        /// This can be used for caching dependencies.
+        /// </summary>
+        public static string? GetBuildlessDependencyDir()
+        {
+            return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR");
+        }
     }
 }

csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.expected

@@ -0,0 +1 @@
+| dependencies/packages/newtonsoft.json/13.0.1/lib/netstandard2.0/Newtonsoft.Json.dll:0:0:0:0 | Newtonsoft.Json, Version=13.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed |

csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.ql

@@ -0,0 +1,7 @@
+import csharp
+
+from Assembly a
+where
+  not a.getCompilation().getOutputAssembly() = a and
+  a.getName().matches("%Newtonsoft%")
+select a

csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/Program.cs

@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}

csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/global.json

@@ -0,0 +1,5 @@
+{
+  "sdk": {
+    "version": "9.0.304"
+  }
+}

csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/standalone.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net9.0</TargetFrameworks>
+  </PropertyGroup>
+
+  <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+    <RemoveDir Directories=".\bin" />
+    <RemoveDir Directories=".\obj" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+  </ItemGroup>
+</Project>