feat(tarball): exclude files and glob patterns from NpmTarball.scanFiles (#646)

Author: fraxken

.changeset/cyan-hats-obey.md

@@ -0,0 +1,5 @@
+---
+"@nodesecure/tarball": minor
+---
+
+Allow to exclude files and patterns in NpmTarball scanFiles function

workspaces/tarball/docs/NpmTarball.md

@@ -27,14 +27,26 @@ Create a new NpmTarball instance.
 > [!CAUTION]
 > ManifestManager instance must have a location defined
 
-### scanFiles(): Promise< ScannedFilesResult >
+### scanFiles(astAnalyserOptions?: AstAnalyserOptions, options?: NpmTarballScanFilesOptions): Promise< ScannedFilesResult >
 
 Scan all the files contained in the tarball and obtain a complete report, including detection of JavaScript threats.
 
+```ts
+interface NpmTarballScanFilesOptions {
+  /**
+   * List of files and directories to exclude from the scan.
+   * Support glob patterns (e.g., "node_modules/**", "dist/**")
+   */
+  exclude?: string[];
+}
+```
+
+The function return the following object as response:
+
 ```ts
 interface ScannedFilesResult {
   composition: TarballComposition;
-  conformance: SpdxExtractedResult;
+  conformance: conformance.SpdxExtractedResult;
   code: SourceCodeReport;
 }
 ```

workspaces/tarball/src/class/NpmTarball.class.ts

@@ -11,6 +11,7 @@ import {
   AstAnalyser,
   DefaultCollectableSet,
   warnings,
+  TsSourceParser,
   type AstAnalyserOptions
 } from "@nodesecure/js-x-ray";
 
@@ -31,6 +32,14 @@ export interface ScannedFilesResult {
   code: SourceCodeReport;
 }
 
+export interface NpmTarballScanFilesOptions {
+  /**
+   * List of files and directories to exclude from the scan.
+   * Support glob patterns (e.g., "node_modules/**", "dist/**")
+   */
+  exclude?: string[];
+}
+
 export type NpmTarballOptions = {
   resolver?: Resolver;
 };
@@ -58,8 +67,11 @@ export class NpmTarball {
   }
 
   async scanFiles(
-    astAnalyserOptions?: AstAnalyserOptions
+    astAnalyserOptions?: AstAnalyserOptions,
+    options: NpmTarballScanFilesOptions = {}
   ): Promise<ScannedFilesResult> {
+    const { exclude = [] } = options;
+
     const location = this.manifest.location;
     const [
       composition,
@@ -74,17 +86,21 @@ export class NpmTarball {
       code = new SourceCodeReport();
     }
     else {
-      const options = this.#optionsWithHostnameSet(astAnalyserOptions ?? {});
+      const options = this.#optionsWithHostnameSet(
+        astAnalyserOptions ?? {}
+      );
 
-      const hostNameSet = options?.collectables?.find((collectable) => collectable.type === "hostname")!;
+      const hostNameSet = options?.collectables?.find(
+        (collectable) => collectable.type === "hostname"
+      )!;
 
       const astAnalyser = new AstAnalyser(options);
 
       code = await new SourceCodeScanner(this.manifest, { astAnalyser }).iterate({
         manifest: [...this.manifest.getEntryFiles()]
-          .flatMap(filterJavaScriptFiles()),
+          .flatMap(filterJavaScriptFiles(exclude)),
         javascript: composition.files
-          .flatMap(filterJavaScriptFiles())
+          .flatMap(filterJavaScriptFiles(exclude))
       });
 
       if (hostNameSet instanceof DefaultCollectableSet) {
@@ -117,19 +133,46 @@ export class NpmTarball {
     };
   }
 
-  #optionsWithHostnameSet(options: AstAnalyserOptions): AstAnalyserOptions {
-    const hasHostnameSet = options?.collectables?.some((collectable) => collectable.type === "hostname");
+  #optionsWithHostnameSet(
+    options: AstAnalyserOptions
+  ): AstAnalyserOptions {
+    const hasHostnameSet = options?.collectables?.some(
+      (collectable) => collectable.type === "hostname"
+    );
     if (hasHostnameSet) {
       return options;
     }
 
-    return { ...options, collectables: [...options.collectables ?? [], new DefaultCollectableSet("hostname")] };
+    return {
+      ...options,
+      collectables: [
+        ...options.collectables ?? [],
+        new DefaultCollectableSet("hostname")
+      ]
+    };
   }
 }
 
-function filterJavaScriptFiles() {
+function filterJavaScriptFiles(
+  exclude: string[] = []
+) {
   return (file: string) => {
-    if (NpmTarball.JS_EXTENSIONS.has(path.extname(file))) {
+    // Exclude .d.ts files
+    if (file.includes("d.ts")) {
+      return [];
+    }
+
+    // Exclude files matching any glob pattern
+    if (exclude.some((pattern) => path.matchesGlob(file, pattern))) {
+      return [];
+    }
+
+    const fileExt = path.extname(file);
+
+    if (NpmTarball.JS_EXTENSIONS.has(fileExt)) {
+      return file;
+    }
+    if (TsSourceParser.FileExtensions.has(fileExt)) {
       return file;
     }
 

workspaces/tarball/test/NpmTarball.spec.ts

@@ -23,6 +23,25 @@ type Metadata = {
 };
 
 describe("NpmTarball", () => {
+  test("it should exclude files matching glob patterns via the exclude option", async() => {
+    const fixturePath = path.join(__dirname, "fixtures", "exclude-test");
+    const mama = await ManifestManager.fromPackageJSON(
+      path.join(fixturePath, "package.json")
+    );
+    const npmTarball = new NpmTarball(mama);
+
+    const exclude: string[] = [
+      "ignoreme.js",
+      "subdir/**"
+    ];
+    const result = await npmTarball.scanFiles(
+      undefined,
+      { exclude }
+    );
+
+    assert.ok(result.code.warnings.length === 0);
+  });
+
   test("it should have a shady-link warning when a hostname resolve a private ip address with collectables", async() => {
     const mama = await ManifestManager.fromPackageJSON(path.join(kFixturePath, "shady-link", "package.json"));
     const npmTarball = new NpmTarball(mama);

workspaces/tarball/test/fixtures/exclude-test/ignoreme.js

@@ -0,0 +1,2 @@
+console.log('ignoreme.js should be excluded');
+const privateIp = "https://10.0.0.1.sslip.io/path";

workspaces/tarball/test/fixtures/exclude-test/index.js

@@ -0,0 +1 @@
+console.log('index.js included');

workspaces/tarball/test/fixtures/exclude-test/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "exclude-test",
+  "version": "1.0.0"
+}

workspaces/tarball/test/fixtures/exclude-test/subdir/shouldBeIgnored.js

@@ -0,0 +1,3 @@
+console.log('shouldBeIgnored.js should be excluded');
+const privateIp = "https://10.0.0.1.sslip.io/path";
+

workspaces/tarball/test/fixtures/exclude-test/subdir/shouldBeIgnoredBis.js

@@ -0,0 +1,2 @@
+console.log('shouldBeIncluded.js should be included');
+const privateIp = "https://10.0.0.1.sslip.io/path";