feat(scanner): implement Payload extractors

workspaces/scanner/package.json

@@ -48,6 +48,7 @@
   },
   "homepage": "https://github.com/NodeSecure/tree/master/workspaces/scanner#readme",
   "dependencies": {
+    "@fastify/deepmerge": "^2.0.1",
     "@nodesecure/conformance": "^1.0.0",
     "@nodesecure/contact": "^1.0.0",
     "@nodesecure/flags": "^2.4.0",
@@ -62,6 +63,7 @@
     "@nodesecure/vulnera": "^2.0.1",
     "@openally/mutex": "^1.0.0",
     "pacote": "^18.0.6",
-    "semver": "^7.5.4"
+    "semver": "^7.5.4",
+    "type-fest": "^4.30.2"
   }
 }

workspaces/scanner/src/extractors/index.ts

@@ -0,0 +1,20 @@
+// Import Internal Dependencies
+import {
+  Payload,
+  type ProbeExtractor,
+  type PackumentProbeExtractor,
+  type ManifestProbeExtractor
+} from "./payload.js";
+
+import * as Probes from "./probes/index.js";
+
+export const Extractors = {
+  Payload,
+  Probes
+} as const;
+
+export type {
+  ProbeExtractor,
+  PackumentProbeExtractor,
+  ManifestProbeExtractor
+};

workspaces/scanner/src/extractors/payload.ts

@@ -0,0 +1,92 @@
+// Import Third-party Dependencies
+import type { Simplify } from "type-fest";
+import deepmerge from "@fastify/deepmerge";
+
+// Import Internal Dependencies
+import * as Scanner from "../types.js";
+import { isNodesecurePayload } from "../utils/index.js";
+
+// CONSTANTS
+const kFastMerge = deepmerge({ all: true });
+
+type MergeDeep<T extends unknown[]> =
+    T extends [a: infer A, ...rest: infer R] ? A & MergeDeep<R> : {};
+
+type ExtractProbeResult<T extends ProbeExtractor<any>[]> = {
+  [K in keyof T]: T[K] extends ProbeExtractor<any> ? ReturnType<T[K]["done"]> : never;
+};
+
+export type ProbeExtractorLevel = "packument" | "manifest";
+export type ProbeExtractorManifestParent = {
+  name: string;
+  dependency: Scanner.Dependency;
+};
+
+export interface ProbeExtractor<Defs> {
+  level: ProbeExtractorLevel;
+  next(...args: any[]): void;
+  done(): Defs;
+}
+
+export interface PackumentProbeExtractor<Defs> extends ProbeExtractor<Defs> {
+  level: "packument";
+  next(name: string, dependency: Scanner.Dependency): void;
+}
+
+export interface ManifestProbeExtractor<Defs> extends ProbeExtractor<Defs> {
+  level: "manifest";
+  next(
+    spec: string,
+    dependencyVersion: Scanner.DependencyVersion,
+    parent: ProbeExtractorManifestParent
+  ): void;
+}
+
+export class Payload<T extends ProbeExtractor<any>[]> {
+  private dependencies: Scanner.Payload["dependencies"];
+  private probes: Record<ProbeExtractorLevel, T>;
+  private cachedResult: ExtractProbeResult<T>;
+
+  constructor(
+    data: Scanner.Payload | Scanner.Payload["dependencies"],
+    probes: [...T]
+  ) {
+    this.dependencies = isNodesecurePayload(data) ?
+      data.dependencies :
+      data;
+
+    this.probes = probes.reduce((data, probe) => {
+      data[probe.level].push(probe);
+
+      return data;
+    }, { packument: [] as unknown as T, manifest: [] as unknown as T });
+  }
+
+  extract() {
+    if (this.cachedResult) {
+      return this.cachedResult;
+    }
+
+    for (const [name, dependency] of Object.entries(this.dependencies)) {
+      this.probes.packument.forEach((probe) => probe.next(name, dependency));
+      if (this.probes.manifest.length > 0) {
+        for (const [spec, depVersion] of Object.entries(dependency.versions)) {
+          this.probes.manifest.forEach((probe) => probe.next(spec, depVersion, { name, dependency }));
+        }
+      }
+    }
+
+    this.cachedResult = [
+      ...this.probes.packument.map((probe) => probe.done()),
+      ...this.probes.manifest.map((probe) => probe.done())
+    ] as ExtractProbeResult<T>;
+
+    return this.cachedResult;
+  }
+
+  extractAndMerge() {
+    return kFastMerge(
+      ...this.extract()
+    ) as unknown as Simplify<MergeDeep<ExtractProbeResult<T>>>;
+  }
+}

workspaces/scanner/src/extractors/probes/SizeExtractor.class.ts

@@ -0,0 +1,65 @@
+// Import Internal Dependencies
+import type {
+  ManifestProbeExtractor,
+  ProbeExtractorManifestParent
+} from "../payload.js";
+import type { DependencyVersion } from "../../types.js";
+
+// Import Third-party Dependencies
+import { formatBytes } from "@nodesecure/utils";
+
+export type SizeExtractorResult = {
+  size: {
+    all: string;
+    internal: string;
+    external: string;
+  };
+};
+
+export interface SizeExtractorOptions {
+  organizationPrefix?: string;
+}
+
+export class SizeExtractor implements ManifestProbeExtractor<SizeExtractorResult> {
+  level = "manifest" as const;
+
+  #size = {
+    all: 0,
+    internal: 0,
+    external: 0
+  };
+  #organizationPrefix: string | null = null;
+
+  constructor(
+    options: SizeExtractorOptions = {}
+  ) {
+    const { organizationPrefix = null } = options;
+
+    this.#organizationPrefix = organizationPrefix;
+  }
+
+  next(
+    _: string,
+    version: DependencyVersion,
+    parent: ProbeExtractorManifestParent
+  ) {
+    const { size } = version;
+
+    const isExternal = this.#organizationPrefix === null ?
+      true :
+      !parent.name.startsWith(`${this.#organizationPrefix}/`);
+
+    this.#size.all += size;
+    this.#size[isExternal ? "external" : "internal"] += size;
+  }
+
+  done() {
+    return {
+      size: {
+        all: formatBytes(this.#size.all),
+        internal: formatBytes(this.#size.internal),
+        external: formatBytes(this.#size.external)
+      }
+    };
+  }
+}

workspaces/scanner/src/extractors/probes/index.ts

@@ -0,0 +1 @@
+export * from "./SizeExtractor.class.js";

workspaces/scanner/src/index.ts

@@ -25,6 +25,7 @@ const kDefaultCwdOptions = {
 };
 
 export * from "./types.js";
+export * from "./extractors/index.js";
 
 export async function cwd(
   location = process.cwd(),

workspaces/scanner/src/utils/index.ts

@@ -4,6 +4,7 @@ export * from "./addMissingVersionFlags.js";
 export * from "./getLinks.js";
 export * from "./urlToString.js";
 export * from "./getUsedDeps.js";
+export * from "./isNodesecurePayload.js";
 
 export const NPM_TOKEN = typeof process.env.NODE_SECURE_TOKEN === "string" ?
   { token: process.env.NODE_SECURE_TOKEN } :

workspaces/scanner/src/utils/isNodesecurePayload.ts

@@ -0,0 +1,8 @@
+// Import Internal Dependencies
+import type { Payload } from "../types.js";
+
+export function isNodesecurePayload(
+  data: Payload | Payload["dependencies"]
+): data is Payload {
+  return "dependencies" in data && "id" in data && "scannerVersion" in data;
+}

workspaces/scanner/test/extractors/payload.spec.ts

@@ -0,0 +1,45 @@
+// Require Node.js Dependencies
+import { describe, it } from "node:test";
+import assert from "node:assert";
+import path from "node:path";
+import fs from "node:fs";
+
+// Import Internal Dependencies
+import {
+  Extractors,
+  type Payload
+} from "../../src/index.js";
+
+// CONSTANTS
+const FIXTURE_PATH = path.join("fixtures", "extractors");
+
+// JSON PAYLOADS
+const expressNodesecurePayload = JSON.parse(fs.readFileSync(
+  new URL(path.join("..", FIXTURE_PATH, "express.json"), import.meta.url),
+  "utf8"
+)) as Payload;
+
+describe("Extractors.Payload", () => {
+  it("should extract Express.js dependencies size", () => {
+    const extractor = new Extractors.Payload(
+      expressNodesecurePayload,
+      [
+        new Extractors.Probes.SizeExtractor()
+      ]
+    );
+
+    const expectedSize = {
+      all: "2.09 MB",
+      internal: "0 B",
+      external: "2.09 MB"
+    };
+
+    const extractResult = extractor.extract();
+    assert.strictEqual(extractResult.length, 1);
+    assert.deepEqual(extractResult, [{ size: expectedSize }]);
+
+    const mergedResult = extractor.extractAndMerge();
+    assert.deepEqual(mergedResult, { size: expectedSize });
+    assert.deepEqual(mergedResult, extractResult[0]);
+  });
+});