feat: highlight infrastructure components

.changeset/four-icons-enjoy.md

@@ -0,0 +1,8 @@
+---
+"@nodesecure/tree-walker": minor
+"@nodesecure/scanner": minor
+"@nodesecure/tarball": minor
+"@nodesecure/rc": minor
+---
+
+feat: highlight infrastructure components

README.md

@@ -23,6 +23,7 @@ Scanner builds on [JS-X-Ray](https://github.com/NodeSecure/js-x-ray) (SAST) and
   - Typosquatting of popular package names
   - Install scripts (e.g. `install`, `preinstall`, `postinstall`, `preuninstall`, `postuninstall`)
 - Highlights packages by name, version(s), or maintainer
+- Highlights infrastructure components such as ip, hostname, email, url
 - Supports NPM and Yarn lockfiles
 
 ## Getting Started
@@ -134,7 +135,9 @@ interface Options {
   };
 
   highlight?: {
-    contacts: Contact[];
+    contacts?: Contact[];
+    packages?: HighlightPackages;
+    identifiers?: string[];
   };
 
   /**

workspaces/rc/package.json

@@ -45,7 +45,7 @@
     "ajv": "6.12.6"
   },
   "dependencies": {
-    "@nodesecure/js-x-ray": "11.3.0",
+    "@nodesecure/js-x-ray": "11.5.0",
     "@nodesecure/npm-types": "^1.2.0",
     "@nodesecure/vulnera": "^2.0.1",
     "@openally/config": "^1.0.1",

workspaces/scanner/package.json

@@ -68,7 +68,7 @@
     "@nodesecure/contact": "^3.0.0",
     "@nodesecure/flags": "^3.0.3",
     "@nodesecure/i18n": "^4.0.2",
-    "@nodesecure/js-x-ray": "11.3.0",
+    "@nodesecure/js-x-ray": "11.5.0",
     "@nodesecure/mama": "^2.1.1",
     "@nodesecure/npm-registry-sdk": "^4.4.0",
     "@nodesecure/npm-types": "^1.3.0",

workspaces/scanner/src/depWalker.ts

@@ -11,6 +11,7 @@ import {
   scanDirOrArchive,
   type PacoteProvider
 } from "@nodesecure/tarball";
+import { CollectableSet } from "@nodesecure/js-x-ray";
 import * as Vulnera from "@nodesecure/vulnera";
 import { npm } from "@nodesecure/tree-walker";
 import { parseAuthor } from "@nodesecure/utils";
@@ -81,6 +82,8 @@ const kDefaultDependencyMetadata: Dependency["metadata"] = {
 
 const kRootDependencyId = 0;
 
+const kCollectableTypes = ["url", "hostname", "ip", "email"];
+
 const { version: packageVersion } = JSON.parse(
   readFileSync(
     new URL(path.join("..", "package.json"), import.meta.url),
@@ -100,6 +103,10 @@ type InitialPayload =
     rootDependency: Payload["rootDependency"];
   };
 
+type Metadata = {
+  spec?: string;
+};
+
 export async function depWalker(
   manifest: PackageJSON | WorkspacesPackageJSON | ManifestVersion,
   options: WalkerOptions,
@@ -118,6 +125,8 @@ export async function depWalker(
 
   const statsCollector = new StatsCollector();
 
+  const collectables = kCollectableTypes.map((type) => new CollectableSet<Metadata>(type));
+
   const pacoteProvider: PacoteProvider = {
     async extract(spec, dest, opts): Promise<void> {
       await statsCollector.track(
@@ -148,6 +157,7 @@ export async function depWalker(
 
   const dependencies: Map<string, Dependency> = new Map();
   const highlightedPackages: Set<string> = new Set();
+  const identifiersToHighlight = new Set<string>(options.highlight?.identifiers ?? []);
   const npmTreeWalker = new npm.TreeWalker({
     registry,
     providers: {
@@ -277,7 +287,8 @@ export async function depWalker(
         isRootNode: scanRootNode && name === manifest.name,
         registry,
         statsCollector,
-        pacoteProvider
+        pacoteProvider,
+        collectables
       };
       operationsQueue.push(
         scanDirOrArchiveEx(name, version, locker, tempDir, logger, scanDirOptions)
@@ -381,7 +392,8 @@ export async function depWalker(
     payload.warnings = globalWarnings.concat(dependencyConfusionWarnings as GlobalWarning[]).concat(warnings);
     payload.highlighted = {
       contacts: illuminated,
-      packages: [...highlightedPackages]
+      packages: [...highlightedPackages],
+      identifiers: extractHighlightedIdentifiers(collectables, identifiersToHighlight)
     };
     payload.dependencies = Object.fromEntries(dependencies);
     payload.metadata = statsCollector.getStats();
@@ -393,6 +405,25 @@ export async function depWalker(
   }
 }
 
+function extractHighlightedIdentifiers(collectables: CollectableSet<Metadata>[], identifiersToHighlight: Set<string>) {
+  if (identifiersToHighlight.size === 0) {
+    return [];
+  }
+
+  return collectables.flatMap((collectableSet) => Array.from(collectableSet)
+    .flatMap(({ value, locations }) => (identifiersToHighlight.has(value) ?
+      locations.map(({ file, metadata, location }) => {
+        return {
+          value,
+          spec: metadata?.spec,
+          location: {
+            file,
+            lines: location
+          }
+        };
+      }) : [])));
+}
+
 // eslint-disable-next-line max-params
 async function scanDirOrArchiveEx(
   name: string,
@@ -407,6 +438,7 @@ async function scanDirOrArchiveEx(
     ref: any;
     statsCollector: StatsCollector;
     pacoteProvider?: PacoteProvider;
+    collectables: CollectableSet<Metadata>[];
   }
 ) {
   using _ = await locker.acquire();
@@ -420,7 +452,8 @@ async function scanDirOrArchiveEx(
       isRootNode,
       ref,
       statsCollector,
-      pacoteProvider
+      pacoteProvider,
+      collectables
     } = options;
 
     const mama = await (isRootNode ?
@@ -434,7 +467,8 @@ async function scanDirOrArchiveEx(
 
     await statsCollector.track(`tarball.scanDirOrArchive ${spec}`, () => scanDirOrArchive(mama, ref, {
       astAnalyserOptions: {
-        optionalWarnings: typeof location !== "undefined"
+        optionalWarnings: typeof location !== "undefined",
+        collectables
       }
     }));
   }

workspaces/scanner/src/types.ts

@@ -218,6 +218,15 @@ export type Stats = {
   apiCalls: ApiStats[];
 };
 
+export type Identifier = {
+  value: string;
+  spec?: string;
+  location: {
+    file: string | null;
+    lines: [[number, number], [number, number]][];
+  };
+};
+
 export interface Payload {
   /** Payload unique id */
   id: string;
@@ -233,6 +242,7 @@ export interface Payload {
   highlighted: {
     contacts: IlluminatedContact[];
     packages: string[];
+    identifiers: Identifier[];
   };
   /** All the dependencies of the package (flattened) */
   dependencies: Dependencies;
@@ -284,6 +294,7 @@ export interface Options {
   highlight?: {
     contacts?: Contact[];
     packages?: HighlightPackages;
+    identifiers?: string[];
   };
 
   /**

workspaces/scanner/test/depWalker.spec.ts

@@ -16,7 +16,8 @@ import {
   from,
   workingDir,
   type Payload,
-  type DependencyVersion
+  type DependencyVersion,
+  type Identifier
 } from "../src/index.ts";
 
 // CONSTANTS
@@ -171,7 +172,7 @@ test("execute depWalker on pkg.gitdeps", async(test) => {
   assert.strictEqual(typeof metadata.startedAt, "number");
   assert.strictEqual(typeof metadata.executionTime, "number");
   assert.strictEqual(Array.isArray(metadata.apiCalls), true);
-  assert.strictEqual(metadata.apiCallsCount, 37);
+  assert.strictEqual(metadata.apiCallsCount, 50);
 });
 
 test("execute depWalker on typo-squatting (with location)", async(test) => {
@@ -347,7 +348,13 @@ test("highlight contacts from a remote package", async() => {
 
 describe("scanner.cwd()", () => {
   test("should parse author, homepage and links for a local package who doesn't exist on the remote registry", async() => {
-    const result = await workingDir(path.join(kFixturePath, "non-npm-package"));
+    const file = path.join(kFixturePath, "non-npm-package");
+    const result = await workingDir(file, {
+      highlight: {
+        identifiers: ["foobar@gmail.com", "https://foobar.com/something", "foobar.com", "127.0.0.1"]
+      },
+      scanRootNode: true
+    });
 
     const dep = result.dependencies["non-npm-package"];
     const v1 = dep.versions["1.0.0"];
@@ -370,6 +377,44 @@ describe("scanner.cwd()", () => {
     });
     assert.strictEqual(dep.metadata.homepage, "https://nodesecure.com");
     assert.strictEqual(typeof result.rootDependency.integrity, "string");
+    const spec = "non-npm-package@1.0.0";
+    assert.partialDeepStrictEqual(sortIdentifiers(result.highlighted.identifiers), sortIdentifiers([
+      {
+        value: "foobar@gmail.com",
+        spec,
+        location: {
+          file
+        }
+      },
+      {
+        value: "foobar@gmail.com",
+        spec,
+        location: {
+          file: path.join(file, "email")
+        }
+      },
+      {
+        value: "https://foobar.com/something",
+        spec,
+        location: {
+          file
+        }
+      },
+      {
+        value: "foobar.com",
+        spec,
+        location: {
+          file
+        }
+      },
+      {
+        value: "127.0.0.1",
+        spec,
+        location: {
+          file
+        }
+      }
+    ]));
   });
 
   test("should parse local manifest author field without throwing when attempting to highlight contacts", async() => {
@@ -397,6 +442,16 @@ describe("scanner.cwd()", () => {
   });
 });
 
+type PartialIdentifer = Omit<Identifier, "location"> & { location: { file: string | null; }; };
+
+function sortIdentifiers(identifiers: PartialIdentifer[]) {
+  return identifiers.slice().sort((a, b) => uniqueIdenfier(a).localeCompare(uniqueIdenfier(b)));
+}
+
+function uniqueIdenfier(identifer: PartialIdentifer) {
+  return `${identifer.value} ${identifer.location.file}`;
+}
+
 function errorLogger() {
   const errors: ({ error: string; phase: string | undefined; })[] = [];
 

workspaces/scanner/test/fixtures/depWalker/non-npm-package/email/email.js

@@ -0,0 +1 @@
+export const email = "foobar@gmail.com";

workspaces/scanner/test/fixtures/depWalker/non-npm-package/index.js

@@ -0,0 +1,13 @@
+import { email } from "./email/email.js";
+
+export const email1 = "foobar@gmail.com";
+
+export const email2 = "john.doe@gmail.com";
+
+export const email3 = email;
+
+export const url = "https://foobar.com/something";
+
+export const hostname = "foobar.com";
+
+export const ip = "127.0.0.1";

workspaces/tarball/package.json

@@ -47,7 +47,7 @@
   "dependencies": {
     "@nodesecure/conformance": "^1.2.1",
     "@nodesecure/fs-walk": "^2.0.0",
-    "@nodesecure/js-x-ray": "11.3.0",
+    "@nodesecure/js-x-ray": "11.5.0",
     "@nodesecure/mama": "^2.1.1",
     "@nodesecure/npm-types": "^1.2.0",
     "@nodesecure/utils": "^2.3.0",

workspaces/tarball/src/class/SourceCodeScanner.class.ts

@@ -182,7 +182,11 @@ export class SourceCodeScanner<
       (filePath) => path.join(location, filePath)
     );
 
-    for await (const fileReport of efa.analyse(absoluteEntryFiles)) {
+    for await (const fileReport of efa.analyse(absoluteEntryFiles, {
+      metadata: {
+        spec: this.manifest.spec
+      }
+    })) {
       report.push(fileReport);
     }
 
@@ -210,7 +214,10 @@ export class SourceCodeScanner<
         const fileReport = await this.#astAnalyser.analyseFile(
           filePath,
           {
-            packageName
+            packageName,
+            metadata: {
+              spec: this.manifest.spec
+            }
           }
         );
 

workspaces/tarball/test/NpmTarball.spec.ts

@@ -18,6 +18,10 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const kFixturePath = path.join(__dirname, "fixtures", "npmTarball");
 const kShadyLinkPath = path.join(kFixturePath, "shady-link");
 
+type Metadata = {
+  spec?: string;
+};
+
 describe("NpmTarball", () => {
   test("it should have a shady-link warning when a hostname resolve a private ip address with collectables", async() => {
     const mama = await ManifestManager.fromPackageJSON(path.join(kFixturePath, "shady-link", "package.json"));
@@ -128,8 +132,25 @@ describe("NpmTarball", () => {
       }].sort(compareWarning)
     );
   });
+
+  test("it should add the spec to collectables", async() => {
+    const mama = await ManifestManager.fromPackageJSON(path.join(kFixturePath, "shady-link", "package.json"));
+    const npmTarball = new NpmTarball(mama);
+    const hostnameSet = new CollectableSet<Metadata>("hostname");
+
+    await npmTarball.scanFiles({
+      collectables: [hostnameSet]
+    });
+
+    assert.deepEqual(extractSpecs(hostnameSet), Array(5).fill("shady-link@0.1.0"));
+  });
 });
 
+function extractSpecs(collectableSet: CollectableSet<Metadata>) {
+  return Array.from(collectableSet)
+    .flatMap(({ locations }) => locations.flatMap(({ metadata }) => metadata?.spec ?? []));
+}
+
 function compareWarning(a: Warning, b: Warning): number {
   const fileComparison = a.file?.localeCompare(b.file ?? "");