Ruby reachables test - WIP (#1574)

* Ignore vendor/cache directories Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
CycloneDX · Jan 17, 2025 · 15c5e26 · 15c5e26
1 parent cd742fb
commit 15c5e26
Show file tree

Hide file tree

Showing 8 changed files with 131 additions and 42 deletions.
diff --git a/.github/workflows/java-reachables-test.yml b/.github/workflows/java-reachables-test.yml
@@ -70,48 +70,51 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
-      - name: npm install, build
+      - name: setup paths
         run: |
-          corepack enable
-          corepack pnpm install
+          corepack enable pnpm
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
           echo "${GITHUB_WORKSPACE}/node_modules/.bin" >> "$GITHUB_PATH"
           mkdir -p repotests
           mkdir -p rubyresults
+      - name: npm install, build
+        run: |
+          pnpm install --package-import-method copy
+          pnpm config set global-bin-dir "${RUNNER_TEMP}/bin"
+          pnpm install -g .
       - uses: actions/checkout@v4
         with:
           repository: 'campsite/campsite'
           path: 'repotests/campsite'
           ref: '10197238bbbefd9c9ac7c77467b647fd93993ba0'
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+        id: ruby34
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.3.4'
-          bundler-cache: true
-        id: ruby33
       - name: bundle install
         run: |
           cd repotests/campsite/api
           bundle install
+          rm -rf vendor/cache vendor/yarn
         env:
-          BUNDLE_PATH: vendor/bundle
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.4'
-          bundler-cache: true
-        id: ruby34
+          BUNDLE_PATH: ${{ runner.temp }}/vendor/bundle
       - name: generate sbom with reachables
         run: |
-          cd repotests/campsite/api
-          echo "PATH: ${PATH}"
-          rbastgen --help
           echo "ATOM_RUBY_HOME: ${ATOM_RUBY_HOME}"
-          echo "BUNDLE_PATH: ${BUNDLE_PATH}"
-          node $GITHUB_WORKSPACE/bin/cdxgen.js -p -t ruby --profile research -o bom.json . --lifecycle pre-build
+          echo "CDXGEN_GEM_HOME: ${CDXGEN_GEM_HOME}"
+          atom --help
+          rbastgen --help
+          node $GITHUB_WORKSPACE/bin/cdxgen.js -t ruby --profile research -o bom.json .
           ls -lh
           cp bom.json *.slices.json $GITHUB_WORKSPACE/rubyresults
         env:
           CDXGEN_DEBUG_MODE: debug
           ATOM_RUBY_HOME: ${{ steps.ruby34.outputs.ruby-prefix }}
-          CDXGEN_GEM_HOME: ${{ github.workspace }}/repotests/campsite/api/vendor/bundle
+          CDXGEN_GEM_HOME: ${{ runner.temp }}/vendor/bundle/ruby/3.3.0
+        working-directory: ./repotests/campsite/api
       - uses: actions/upload-artifact@v4
         with:
           name: rubyresults

diff --git a/lib/cli/index.js b/lib/cli/index.js
@@ -5160,33 +5160,48 @@ export function createPHPBom(path, options) {
  * @param {Object} options Parse options from the cli
  */
 export async function createRubyBom(path, options) {
+  const excludeList = (options.exclude || []).concat(["**/vendor/cache/**"]);
+  const gemLockExcludeList = (options.exclude || []).concat([
+    "**/vendor/bundle/ruby/**/Gemfile.lock",
+  ]);
+  if (!hasAnyProjectType(["oci"], options, false)) {
+    excludeList.push("**/vendor/bundle/**");
+    gemLockExcludeList.push("**/vendor/cache/**");
+  }
   const gemFiles = getAllFiles(
     path,
     `${options.multiProject ? "**/" : ""}Gemfile`,
-    options,
+    {
+      ...options,
+      exclude: excludeList,
+    },
   );
   let gemLockFiles = getAllFiles(
     path,
     `${options.multiProject ? "**/" : ""}Gemfile*.lock`,
     {
       ...options,
-      exclude: (options.exclude || []).concat([
-        "**/vendor/bundle/ruby/**/Gemfile.lock",
-      ]),
+      exclude: gemLockExcludeList,
     },
   );
   let gemspecFiles = getAllFiles(
     path,
     `${options.multiProject ? "**/" : ""}*.gemspec`,
-    options,
+    {
+      ...options,
+      exclude: excludeList,
+    },
   );
-  const gemHome = process.env.CDXGEN_GEM_HOME || process.env.GEM_HOME;
+  let gemHome = process.env.CDXGEN_GEM_HOME || process.env.GEM_HOME;
+  if (!gemHome && (process.env.BUNDLE_PATH || process.env.GEM_PATH)) {
+    gemHome = process.env.BUNDLE_PATH || process.env.GEM_PATH;
+  }
   let isGemHomeEmpty = true;
   // In deep mode, let's collect all gems that got installed in our custom GEM_HOME directory.
   // This would improve the accuracy of any security analysis downstream at cost of a slight increase in time.
   if (options.deep && process.env.CDXGEN_GEM_HOME) {
     const gemHomeSpecFiles = getAllFiles(
-      process.env.CDXGEN_GEM_HOME,
+      process.env.CDXGEN_GEM_HOME || process.env.BUNDLE_PATH,
       "**/specifications/**/*.gemspec",
       options,
     );
@@ -5223,9 +5238,7 @@ export async function createRubyBom(path, options) {
     `${options.multiProject ? "**/" : ""}Gemfile*.lock`,
     {
       ...options,
-      exclude: (options.exclude || []).concat([
-        "**/vendor/bundle/ruby/**/Gemfile.lock",
-      ]),
+      exclude: gemLockExcludeList,
     },
   );
   if (gemLockFiles.length) {

diff --git a/lib/evinser/evinser.js b/lib/evinser/evinser.js
@@ -1206,6 +1206,8 @@ export function extractEndpoints(language, code) {
       code = code.replaceAll("...", "");
       if (code.includes("namespace ")) {
         urlPrefix = code.split("namespace ").pop().split(" ")[0];
+      } else if (code.includes("collection do get ")) {
+        urlPrefix = code.split("collection do get ").pop().split(" ")[0];
       }
       for (const m of ["get", "post", "delete", "options", "put", "head"]) {
         if (code.includes(`${m} `)) {

diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js
@@ -6248,6 +6248,7 @@ export function collectGemModuleNames(
   if (!gemName || gemName.startsWith("/") || gemName === ".") {
     return [];
   }
+  gemName = gemName.replace(/["']/g, "");
   // Module names for some gems cannot be obtained with our one-liner
   // So we keep a hard-coded list of such problematic ones.
   if (RUBY_KNOWN_MODULES[gemName]) {
@@ -6289,6 +6290,7 @@ export function collectGemModuleNames(
     // Let's retry for simple mismatches
     if (gemName?.includes("-")) {
       return collectGemModuleNames(
+        rubyCommand,
         bundleCommand,
         gemHome,
         gemName.replaceAll("-", "/"),
@@ -6312,6 +6314,7 @@ export function collectGemModuleNames(
           console.log("Retrying", gemName, "with", altGemName);
         }
         return collectGemModuleNames(
+          rubyCommand,
           bundleCommand,
           gemHome,
           altGemName,
@@ -6320,18 +6323,27 @@ export function collectGemModuleNames(
       }
       if (DEBUG_MODE) {
         console.log(
-          `Is ${altGemName} an alternative gem name for the package ${gemName}? Please let us know.`,
+          `Is ${altGemName} an alternative gem name for '${gemName}' package? Please let us know if this is correct.`,
         );
       }
     }
     // Gem wasn't installed or the GEM_HOME was not set correctly.
-    if (result?.stderr?.includes("Bundler::GemNotFound")) {
+    if (
+      result?.stderr?.includes("Bundler::GemNotFound") ||
+      result?.stderr?.includes("(LoadError)")
+    ) {
       return [];
     }
-    if (DEBUG_MODE) {
+    if (
+      !result?.stderr?.includes("(NameError)") &&
+      !result?.stderr?.includes("(NoMethodError)") &&
+      !result?.stderr?.includes("(ArgumentError)") &&
+      DEBUG_MODE
+    ) {
       console.log(
         `Unable to collect the module names exported by the gem ${gemName}.`,
       );
+      console.log(result.stderr);
     }
     // Let's guess the module name based on common naming convention.
     return toGemModuleNames(gemName);
@@ -6374,23 +6386,41 @@ export async function parseGemspecData(gemspecData, gemspecFile) {
   }
   let versionHackMatch = false;
   gemspecData.split("\n").forEach((l) => {
+    versionHackMatch = false;
     l = l.replace("\r", "");
     l = l.replace(/\s+/g, " ").replaceAll("%q{", "").trim().replace(/}$/, "");
     if (l.startsWith("#")) {
       return;
     }
     for (const aprop of ["name", "version", "description", "homepage"]) {
       if (l.includes(`.${aprop} = `)) {
-        pkg[aprop] = l
+        let value = l
           .split(`.${aprop} = `)[1]
           .replace(".freeze", "")
           .replaceAll("''', ", "")
           .replace(/"/g, "");
+        if (["name", "version"].includes(aprop)) {
+          value = value.replace(/["']/g, "");
+        }
+        pkg[aprop] = value;
         return;
       }
     }
-    // Cleanup bad version strings
-    if (pkg?.version?.includes("::")) {
+    // Handle common problems
+    if (pkg.name === "name") {
+      console.log(
+        "Unable to identify the package name by parsing the file",
+        gemspecFile,
+      );
+      return;
+    }
+    if (
+      pkg?.version === "version" ||
+      pkg?.version?.includes("$") ||
+      pkg?.version?.includes("gem_version") ||
+      pkg?.version?.includes("File.") ||
+      pkg?.version?.includes("::")
+    ) {
       pkg.version = undefined;
       // Can we find the version from the directory name?
       if (gemspecFile) {
@@ -6400,6 +6430,12 @@ export async function parseGemspecData(gemspecData, gemspecFile) {
           versionHackMatch = true;
         }
       }
+      if (!versionHackMatch && !pkg.version) {
+        console.log(
+          "Unable to identify the package version by parsing the file",
+          gemspecFile,
+        );
+      }
     }
     for (const aprop of ["authors", "licenses"]) {
       if (l.includes(`.${aprop} = `)) {
@@ -6618,7 +6654,7 @@ export async function parseGemfileLockData(gemLockData, lockFile) {
     if (specsFound) {
       const tmpA = l.split(" (");
       const nameWithPrefix = tmpA[0];
-      const name = tmpA[0].trim();
+      const name = tmpA[0].replace(/["']/g, "").trim();
       const level = nameWithPrefix.replace(name, "").split("  ").length % 2;
       if (
         !name.length ||

diff --git a/lib/stages/pregen/pregen.js b/lib/stages/pregen/pregen.js
@@ -6,6 +6,7 @@ import process from "node:process";
 import {
   SDKMAN_JAVA_TOOL_ALIASES,
   bundleInstallWithDocker,
+  collectRubyInfo,
   getOrInstallNvmTool,
   installRubyBundler,
   installRubyVersion,
@@ -357,15 +358,27 @@ export function prepareRubyEnv(filePath, options) {
   const gemFiles = getAllFiles(
     filePath,
     `${options.multiProject ? "**/" : ""}Gemfile`,
-    options,
+    {
+      ...options,
+      exclude: (options.exclude || []).concat([
+        "**/vendor/cache/**",
+        "**/vendor/bundle/**",
+      ]),
+    },
   );
   if (!gemFiles.length) {
     return;
   }
   const gemLockFiles = getAllFiles(
     filePath,
     `${options.multiProject ? "**/" : ""}Gemfile*.lock`,
-    options,
+    {
+      ...options,
+      exclude: (options.exclude || []).concat([
+        "**/vendor/cache/**",
+        "**/vendor/bundle/**",
+      ]),
+    },
   );
   if (gemLockFiles.length && !options.deep) {
     return;
@@ -429,7 +442,21 @@ export function prepareRubyEnv(filePath, options) {
       }
     }
   }
-  if (!rbenvPresent) {
+  // Do we already have this version
+  const existingRuby = collectRubyInfo(filePath);
+  if (
+    rubyVersionNeeded &&
+    existingRuby?.version?.startsWith(`ruby ${rubyVersionNeeded} `)
+  ) {
+    if (DEBUG_MODE) {
+      console.log(`Required Ruby version ${rubyVersionNeeded} is present.`);
+    }
+    process.env.CDXGEN_RUBY_CMD = "ruby";
+    process.env.CDXGEN_GEM_CMD = "gem";
+    process.env.CDXGEN_BUNDLE_CMD = "bundle";
+    return;
+  }
+  if (rubyVersionNeeded && !rbenvPresent) {
     console.log(
       `This project requires Ruby ${rubyVersionNeeded}. cdxgen can automatically install the required version of Ruby with rbenv command.`,
     );
@@ -443,7 +470,8 @@ export function prepareRubyEnv(filePath, options) {
         );
       }
     }
-  } else if (rubyVersionNeeded) {
+  }
+  if (rubyVersionNeeded) {
     // Should we use docker
     if (isFeatureEnabled(options, "ruby-docker-install") || isWin) {
       for (const agemf of gemFiles) {
@@ -488,6 +516,7 @@ export function prepareRubyEnv(filePath, options) {
         }
         process.env.CDXGEN_RUBY_CMD = join(fullToolBinDir, "ruby");
         process.env.CDXGEN_GEM_CMD = join(fullToolBinDir, "gem");
+        process.env.CDXGEN_BUNDLE_CMD = join(fullToolBinDir, "bundle");
         bundleTool = join(fullToolBinDir, "bundle");
         process.env.CDXGEN_BUNDLE_CMD = bundleTool;
         if (!existsSync(bundleTool)) {
@@ -507,7 +536,13 @@ export function prepareRubyEnv(filePath, options) {
         const gemspecFiles = getAllFiles(
           cdxgenGemHome,
           "**/specifications/**/*.gemspec",
-          options,
+          {
+            ...options,
+            exclude: (options.exclude || []).concat([
+              "**/vendor/cache/**",
+              "**/vendor/bundle/**",
+            ]),
+          },
         );
         if (gemspecFiles.length > 3) {
           if (DEBUG_MODE) {

diff --git a/types/lib/cli/index.d.ts.map b/types/lib/cli/index.d.ts.map