feat(js-x-ray)! dependencies is now considered as collectables and is not in the report anymore (#532)

Author: clemgbld

.changeset/short-jobs-visit.md

@@ -0,0 +1,5 @@
+---
+"@nodesecure/js-x-ray": major
+---
+
+feat(js-x-ray)! dependencies is now considered as collectables and is not in the report anymore

workspaces/js-x-ray/docs/AstAnalyser.md

@@ -55,11 +55,11 @@ class AstAnalyser {
   ) => Report;
   analyseFile(
     pathToFile: string | URL,
-    options?: RuntimeFileOptions
+    options?: RuntimeOptions
   ): Promise<ReportOnFile>;
   analyseFileSync(
     pathToFile: string | URL,
-    options?: RuntimeFileOptions
+    options?: RuntimeOptions
   ): ReportOnFile;
   prepareSource(source: string, options?: PrepareSourceOptions): string
 }
@@ -83,12 +83,12 @@ interface RuntimeOptions {
   isMinified?: boolean;
   initialize?: (sourceFile: SourceFile) => void;
   finalize?: (sourceFile: SourceFile) => void;
+  packageName?: string;
 }
 
 type SourceFlags = "fetch" | "oneline-require" | "is-minified";
 
 interface Report {
-  dependencies: Map<string, Dependency>;
   warnings: Warning[];
   flags: Set<SourceFlags>;
   idsLengthAvg: number;
@@ -99,7 +99,6 @@ type ReportOnFile = {
   ok: true,
   warnings: Warning[];
   flags: Set<SourceFlags>;
-  dependencies: Map<string, Dependency>;
 } | {
   ok: false,
   warnings: Warning[];
@@ -208,7 +207,6 @@ Result:
   idsLengthAvg: 0,
   stringScore: 0,
   warnings: [ { kind: 'unsafe-danger', location: [Array], source: 'JS-X-Ray' } ],
-  dependencies: Map(0) {},
   flags: Set(0) {},
   isOneLineRequire: false
 }

workspaces/js-x-ray/docs/CollectableSet.md

@@ -1,9 +1,10 @@
 # CollectableSet
 
-CollectableSet is a specialized data structure for collecting and aggregating infrastructure-related data points (e.g., URLs, hostnames, IPs) during JavaScript AST analysis. It groups locations by value and file, with optional metadata support. Post-analysis, the collected data can be exploited externally (e.g., for network monitoring, security audits, or infrastructure mapping) to derive insights beyond JS-X-Ray's built-in warnings.
+CollectableSet is a specialized data structure for collecting and aggregating infrastructure-related data points (e.g., URLs, hostnames, IPs, dependencies) during JavaScript AST analysis. It groups locations by value and file, with optional metadata support. Post-analysis, the collected data can be exploited externally (e.g., for network monitoring, security audits, or infrastructure mapping) to derive insights beyond JS-X-Ray's built-in warnings.
 
 - **type**: Type
 - **add(value, infos)**: Adds an entry to the set. Groups by value and file.
+- **values()**: Returns an iterable of all values of the CollectableSet.
 
 CollectableSet is only an interface but Js-X-Ray provides a default implementation named DefaultCollectableSet.
 The default implementation has an additional method to read what it has collected.
@@ -35,7 +36,7 @@ for (const { value, locations } of hostnameSet) {
 ## API
 
 ```ts
-export type Type = "url" | "hostname" | "ip" | "email" | (string & {});
+export type Type = "url" | "hostname" | "ip" | "email" | "dependency" | (string & {});
 
 export type Location<T = Record<string, unknown>> = {
   file: string | null;
@@ -52,6 +53,7 @@ export type CollectableInfos<T = Record<string, unknown>> = {
 export interface CollectableSet<T = Record<string, unknown>> {
   add(value: string, infos: CollectableInfos<T>): void;
   type: Type;
+  values(): Iterable<string>;
 }
 
 export class DefaultCollectableSet<T = Record<string, unknown>> implements CollectableSet<T> {

workspaces/js-x-ray/docs/EntryFilesAnalyser.md

@@ -43,7 +43,7 @@ declare class EntryFilesAnalyser {
    */
   analyse(
     entryFiles: Iterable<string | URL>,
-    options?: RuntimeFileOptions
+    options?: RuntimeOptions
   ): AsyncGenerator<ReportOnFile & { file: string }>;
 }
 ```

workspaces/js-x-ray/src/AstAnalyser.ts

@@ -29,14 +29,13 @@ import {
   type OptionalWarningName,
   type Warning
 } from "./warnings.ts";
-import type { CollectableSet } from "./CollectableSet.ts";
+import type { CollectableSet, Type } from "./CollectableSet.ts";
 import { CollectableSetRegistry } from "./CollectableSetRegistry.ts";
 
-export interface Dependency {
+export type Dependency = {
   unsafe: boolean;
   inTry: boolean;
-  location?: null | ESTree.SourceLocation;
-}
+};
 
 export interface RuntimeOptions {
   /**
@@ -58,14 +57,10 @@ export interface RuntimeOptions {
    */
   customParser?: SourceParser;
   metadata?: Record<string, unknown>;
-}
-
-export interface RuntimeFileOptions extends Omit<RuntimeOptions, "isMinified"> {
   packageName?: string;
 }
 
 export interface Report {
-  dependencies: Map<string, Dependency>;
   warnings: Warning[];
   flags: Set<SourceFlags>;
   idsLengthAvg: number;
@@ -75,7 +70,6 @@ export interface Report {
 export type ReportOnFile = {
   ok: true;
   warnings: Warning[];
-  dependencies: Map<string, Dependency>;
   flags: Set<SourceFlags>;
 } | {
   ok: false;
@@ -126,6 +120,7 @@ export class AstAnalyser {
   probes: Probe[];
   #collectables: CollectableSet[];
   #sensitivity: Sensitivity;
+  #collectableSetRegistry: CollectableSetRegistry;
 
   constructor(options: AstAnalyserOptions = {}) {
     const {
@@ -171,6 +166,7 @@ export class AstAnalyser {
     options: RuntimeOptions = {}
   ): Report {
     const {
+      packageName,
       location,
       isMinified = false,
       removeHTMLComments = false,
@@ -186,15 +182,22 @@ export class AstAnalyser {
       void 0
     );
 
-    const source = new SourceFile(location, metadata);
+    const source = new SourceFile(location, {
+      metadata,
+      collectables: this.#collectables,
+      packageName
+    });
+
+    this.#collectableSetRegistry = source.collectablesSetRegistry;
+
     source.sensitivity = this.#sensitivity;
     if (trojan.verify(str)) {
       source.warnings.push(
         generateWarning("obfuscated-code", { value: "trojan-source" })
       );
     }
 
-    const probeRunner = new ProbeRunner(source, new CollectableSetRegistry(this.#collectables), this.probes);
+    const probeRunner = new ProbeRunner(source, this.probes);
     if (initialize) {
       if (typeof initialize !== "function") {
         throw new TypeError("options.initialize must be a function");
@@ -221,7 +224,6 @@ export class AstAnalyser {
 
     return {
       ...source.getResult(isMinified),
-      dependencies: source.dependencies,
       flags: source.flags
     };
   }
@@ -250,11 +252,11 @@ export class AstAnalyser {
 
   async analyseFile(
     pathToFile: string | URL,
-    options: RuntimeFileOptions = {}
+    options: RuntimeOptions = {}
   ): Promise<ReportOnFile> {
     try {
       const {
-        packageName = null,
+        packageName,
         removeHTMLComments = false,
         initialize,
         finalize,
@@ -273,21 +275,17 @@ export class AstAnalyser {
         initialize,
         finalize,
         customParser,
-        metadata
+        metadata,
+        packageName
       });
 
-      if (packageName !== null) {
-        data.dependencies.delete(packageName);
-      }
-
       // Add is-minified flag if the file is minified and not a one-line require
       if (!data.flags.has("oneline-require") && isMin) {
         data.flags.add("is-minified");
       }
 
       return {
         ok: true,
-        dependencies: data.dependencies,
         warnings: data.warnings,
         flags: data.flags
       };
@@ -306,11 +304,11 @@ export class AstAnalyser {
 
   analyseFileSync(
     pathToFile: string | URL,
-    options: RuntimeFileOptions = {}
+    options: RuntimeOptions = {}
   ): ReportOnFile {
     try {
       const {
-        packageName = null,
+        packageName,
         removeHTMLComments = false,
         initialize,
         finalize,
@@ -329,21 +327,17 @@ export class AstAnalyser {
         initialize,
         finalize,
         customParser,
-        metadata
+        metadata,
+        packageName
       });
 
-      if (packageName !== null) {
-        data.dependencies.delete(packageName);
-      }
-
       // Add is-minified flag if the file is minified and not a one-line require
       if (!data.flags.has("oneline-require") && isMin) {
         data.flags.add("is-minified");
       }
 
       return {
         ok: true,
-        dependencies: data.dependencies,
         warnings: data.warnings,
         flags: data.flags
       };
@@ -384,4 +378,8 @@ export class AstAnalyser {
   #removeHTMLComment(str: string): string {
     return str.replaceAll(/<!--[\s\S]*?(?:-->)/g, "");
   }
+
+  getCollectableSet(type: Type) {
+    return this.#collectableSetRegistry.get(type);
+  }
 }

workspaces/js-x-ray/src/CollectableSet.ts

@@ -1,7 +1,7 @@
 // Import Internal Dependencies
 import { type SourceArrayLocation } from "./utils/toArrayLocation.ts";
 
-export type Type = "url" | "hostname" | "ip" | "email" | (string & {});
+export type Type = "url" | "hostname" | "ip" | "email" | "dependency" | (string & {});
 
 export type Location<T = Record<string, unknown>> = {
   file: string | null;
@@ -18,6 +18,7 @@ export type CollectableInfos<T = Record<string, unknown>> = {
 export interface CollectableSet<T = Record<string, unknown>> {
   add(value: string, infos: CollectableInfos<T>): void;
   type: Type;
+  values(): Iterable<string>;
 }
 
 export class DefaultCollectableSet<T = Record<string, unknown>> implements CollectableSet<T> {
@@ -45,6 +46,10 @@ export class DefaultCollectableSet<T = Record<string, unknown>> implements Colle
     files?.set(file, [{ location, metadata }]);
   }
 
+  values(): Iterable<string> {
+    return this.#entries.keys();
+  }
+
   * [Symbol.iterator]() {
     for (const [value, files] of this.#entries) {
       const locations: Location<T>[] = [];

workspaces/js-x-ray/src/CollectableSetRegistry.ts

@@ -23,4 +23,8 @@ export class CollectableSetRegistry {
   has(type: Type): boolean {
     return this.#collectableSets.has(type);
   }
+
+  get(type: Type): CollectableSet | undefined {
+    return this.#collectableSets.get(type);
+  }
 }

workspaces/js-x-ray/src/EntryFilesAnalyser.ts

@@ -14,7 +14,7 @@ import {
 import {
   AstAnalyser,
   type ReportOnFile,
-  type RuntimeFileOptions
+  type RuntimeOptions
 } from "./AstAnalyser.ts";
 import { TsSourceParser } from "./parsers/TsSourceParser.ts";
 import {
@@ -71,7 +71,7 @@ export class EntryFilesAnalyser {
 
   async* analyse(
     entryFiles: Iterable<string | URL>,
-    options: RuntimeFileOptions = {}
+    options: RuntimeOptions = {}
   ): AsyncGenerator<ReportOnFile & { file: string; }> {
     this.dependencies = new DiGraph();
 
@@ -132,7 +132,7 @@ export class EntryFilesAnalyser {
   async* #analyseFile(
     file: string,
     relativeFile: string,
-    options: RuntimeFileOptions
+    options: RuntimeOptions
   ) {
     this.dependencies.addVertex({
       id: relativeFile,
@@ -149,11 +149,13 @@ export class EntryFilesAnalyser {
     );
     yield { file: relativeFile, ...report };
 
-    if (!report.ok || typeof report.dependencies === "undefined") {
+    const dependencySet = this.astAnalyzer.getCollectableSet("dependency");
+
+    if (!report.ok || typeof dependencySet === "undefined") {
       return;
     }
 
-    for (const [name] of report.dependencies) {
+    for (const name of dependencySet.values()) {
       const depFile = await this.#getInternalDepPath(
         path.join(path.dirname(file), name)
       );

workspaces/js-x-ray/src/ProbeRunner.ts

@@ -29,7 +29,6 @@ import isPrototypePollution from "./probes/isPrototypePollution.ts";
 import type { TracedIdentifierReport } from "./VariableTracer.ts";
 import type { SourceFile } from "./SourceFile.ts";
 import type { OptionalWarningName } from "./warnings.ts";
-import type { CollectableSetRegistry } from "./CollectableSetRegistry.ts";
 import {
   getCallExpressionIdentifier
 } from "./estree/index.ts";
@@ -47,7 +46,6 @@ export type NamedMainHandlers<T extends ProbeContextDef = ProbeContextDef> = {
 
 export type ProbeContext<T extends ProbeContextDef = ProbeContextDef> = {
   sourceFile: SourceFile;
-  collectableSetRegistry: CollectableSetRegistry;
   context?: T;
   setEntryPoint: (handlerName: string) => void;
 };
@@ -75,7 +73,6 @@ export interface Probe<T extends ProbeContextDef = ProbeContextDef> {
 export class ProbeRunner {
   probes: Probe[];
   sourceFile: SourceFile;
-  #collectableSetRegistry: CollectableSetRegistry;
   #selectedEntryPoints: Map<Probe, string> = new Map();
 
   static Signals = Object.freeze({
@@ -116,11 +113,9 @@ export class ProbeRunner {
 
   constructor(
     sourceFile: SourceFile,
-    collectableSetRegistry: CollectableSetRegistry,
     probes: Probe[] = ProbeRunner.Defaults
   ) {
     this.sourceFile = sourceFile;
-    this.#collectableSetRegistry = collectableSetRegistry;
 
     for (const probe of probes) {
       assert(
@@ -173,7 +168,6 @@ export class ProbeRunner {
 
     return {
       sourceFile: this.sourceFile,
-      collectableSetRegistry: this.#collectableSetRegistry,
       context: probe.context,
       setEntryPoint
     };