From f047f6b19dd01bbb7ea2ca029b7373fa992a2eb0 Mon Sep 17 00:00:00 2001 From: Jacek Puchta Date: Wed, 25 Oct 2023 19:15:10 +0200 Subject: [PATCH] python services Signed-off-by: Prabhu Subramanian Update packages. Add sample test for python evinse Signed-off-by: Prabhu Subramanian --- .github/workflows/app-release.yml | 2 +- .github/workflows/dockertests.yml | 12 +- .github/workflows/nodejs.yml | 2 +- .github/workflows/python-atom-tests.yml | 2 +- .github/workflows/repotests.yml | 8 +- bin/cdxgen.js | 3 + data/frameworks-list.json | 35 +++-- data/pypi-pkg-aliases.json | 6 + evinser.js | 174 ++++++++++++++++++------ index.js | 37 +++-- package-lock.json | 12 +- package.json | 4 +- utils.js | 102 ++++++++++---- 13 files changed, 294 insertions(+), 105 deletions(-) diff --git a/.github/workflows/app-release.yml b/.github/workflows/app-release.yml index 8788b03d1c..4a378579e9 100644 --- a/.github/workflows/app-release.yml +++ b/.github/workflows/app-release.yml @@ -14,7 +14,7 @@ jobs: - name: Use Node.js uses: actions/setup-node@v3 with: - node-version: 20.5 + node-version: '21.x' - name: Install dependencies run: | sudo apt-get install -y python3.8 python3.8-dev python3-pip python3-testresources python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev diff --git a/.github/workflows/dockertests.yml b/.github/workflows/dockertests.yml index 5262f20823..8648c7367c 100644 --- a/.github/workflows/dockertests.yml +++ b/.github/workflows/dockertests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [18.x] + node-version: ['21.x'] java-version: ['19'] steps: - uses: actions/checkout@v4 @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up JDK uses: actions/setup-java@v3 with: @@ -68,7 +68,7 @@ jobs: strategy: matrix: - node-version: [18.x] + node-version: ['21.x'] java-version: ['19'] steps: - uses: actions/checkout@v4 @@ -79,7 +79,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up JDK uses: actions/setup-java@v3 with: @@ -108,7 +108,7 @@ jobs: strategy: matrix: - node-version: [18.x] + node-version: ['21.x'] java-version: ['19'] steps: - uses: actions/checkout@v4 @@ -119,7 +119,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up JDK uses: actions/setup-java@v3 with: diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 3a1ba40114..ca1345ed8a 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - node-version: [16.x, 18.x, 20.x] + node-version: ['16.x', '18.x', '20.x', '21.x'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/python-atom-tests.yml b/.github/workflows/python-atom-tests.yml index 9f62f84c54..d207f02cba 100644 --- a/.github/workflows/python-atom-tests.yml +++ b/.github/workflows/python-atom-tests.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [18.x] + node-version: ['21.x'] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index c5b9505175..19d9cca21b 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [18.x] + node-version: ['21.x'] os: ['ubuntu-latest', 'windows-latest'] runs-on: ${{ matrix.os }} steps: @@ -148,6 +148,10 @@ jobs: with: repository: 'hoolicorp/java-sec-code' path: 'repotests/java-sec-code' + - uses: actions/checkout@v3 + with: + repository: 'DefectDojo/django-DefectDojo' + path: 'repotests/django-DefectDojo' - uses: dtolnay/rust-toolchain@stable - name: repotests run: | @@ -160,6 +164,8 @@ jobs: bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --only spring bin/cdxgen.js -p -t java repotests/java-sec-code -o repotests/java-sec-code/bom.json --deep node bin/evinse.js -i repotests/java-sec-code/bom.json -o bomresults/java-sec-code.evinse.json -l java --with-reachables -p repotests/java-sec-code + bin/cdxgen.js -t python repotests/django-DefectDojo -o repotests/django-DefectDojo/bom.json --deep + node bin/evinse.js -i repotests/django-DefectDojo/bom.json -o bomresults/django-DefectDojo.evinse.json -l python --with-reachables repotests/django-DefectDojo bin/cdxgen.js -p -r -t java repotests/shiftleft-java-example -o bomresults/bom-java.json --generate-key-and-sign node bin/evinse.js -i bomresults/bom-java.json -o bomresults/bom-java.evinse.json -l java --with-data-flow -p repotests/shiftleft-java-example SBOM_SIGN_ALGORITHM=RS512 SBOM_SIGN_PRIVATE_KEY=bomresults/private.key SBOM_SIGN_PUBLIC_KEY=bomresults/public.key bin/cdxgen.js -p -r -t github repotests/shiftleft-java-example -o bomresults/bom-github.json diff --git a/bin/cdxgen.js b/bin/cdxgen.js index fe5c2d8e90..263d6f5055 100755 --- a/bin/cdxgen.js +++ b/bin/cdxgen.js @@ -148,6 +148,9 @@ const args = yargs(hideBin(process.argv)) default: false, description: "Generate SBOM with evidence for supported languages. WIP" }) + .option("deps-slices-file", { + description: "Path for the parsedeps slice file created by atom." + }) .option("usages-slices-file", { description: "Path for the usages slice file created by atom." }) diff --git a/data/frameworks-list.json b/data/frameworks-list.json index b40f4d7188..c1ebfe6fa0 100644 --- a/data/frameworks-list.json +++ b/data/frameworks-list.json @@ -4,8 +4,8 @@ "System.ServiceModel", "System.Data", "spring", - "flask", - "django", + "pkg:pypi/flask", + "pkg:pypi/django", "beego", "chi", "echo", @@ -30,15 +30,28 @@ "express", "knex", "vue", - "aiohttp", - "bottle", - "cherrypy", - "drt", - "falcon", - "hug", - "pyramid", - "sanic", - "tornado", + "pkg:pypi/aiohttp", + "pkg:pypi/bottle", + "pkg:pypi/cherrypy", + "pkg:pypi/drt", + "pkg:pypi/falcon", + "pkg:pypi/hug", + "pkg:pypi/pyramid", + "pkg:pypi/sanic", + "pkg:pypi/tornado", + "pkg:pypi/fastapi", + "pkg:pypi/pyqt", + "pkg:pypi/tkinter", + "pkg:pypi/kivy", + "pkg:pypi/pyside", + "pkg:pypi/scikit", + "pkg:pypi/tensorflow", + "pkg:pypi/pytorch", + "pkg:pypi/keras", + "pkg:pypi/numpy", + "pkg:pypi/scipy", + "pkg:pypi/pandas", + "pkg:pypi/matplotlib", "vibora", "koa", "-sdk", diff --git a/data/pypi-pkg-aliases.json b/data/pypi-pkg-aliases.json index 974e0cfc2b..11916e16df 100644 --- a/data/pypi-pkg-aliases.json +++ b/data/pypi-pkg-aliases.json @@ -553,6 +553,7 @@ "creole": "python-creole", "creoleparser": "creoleparser", "crispy-forms": "django-crispy-forms", + "crum": "django-crum", "cronlog": "python-crontab", "crontab": "python-crontab", "crypto": "pycryptodome", @@ -589,6 +590,7 @@ "djcelery": "django-celery", "djkombu": "django-kombu", "djorm-pgarray": "djorm-ext-pgarray", + "django-filters": "filters-django", "dns": "dnspython", "docgen": "ansible-docgenerator", "docker": "docker-py", @@ -631,6 +633,7 @@ "fdpexpect": "pexpect", "fedora": "python-fedora", "fias": "ailove-django-fias", + "fieldsignals": "django-fieldsignals", "fiftyone-degrees": "51degrees-mobile-detector", "fiftyonedegrees": "51degrees-mobile-detector-v3-wrapper", "five": "five.customerize", @@ -709,6 +712,7 @@ "igraph": "python-igraph", "imdb": "imdbpy", "impala": "impyla", + "imagekit": "django-imagekit", "impersonate": "django-impersonate", "inmemorystorage": "ambition-inmemorystorage", "ipaddress": "backport-ipaddress", @@ -845,6 +849,7 @@ "path": "path.py", "patricia": "patricia-trie", "paver": "paver", + "packageurl": "packageurl-python", "peak": "proxytypes", "picasso": "anderson.picasso", "picklefield": "django-picklefield", @@ -1057,6 +1062,7 @@ "slugify": "unicode-slugify", "smarkets": "smk-python-sdk", "snappy": "ctypes-snappy", + "social-core": "social-auth-core", "social-django": "social-auth-app-django", "socketio": "python-socketio", "socketserver": "pies2overrides", diff --git a/evinser.js b/evinser.js index f7ce97d790..9f5d6c36c7 100644 --- a/evinser.js +++ b/evinser.js @@ -7,7 +7,7 @@ import { collectMvnDependencies } from "./utils.js"; import { tmpdir } from "node:os"; -import path from "node:path"; +import path, { basename } from "node:path"; import fs from "node:fs"; import * as db from "./db.js"; import { PackageURL } from "packageurl-js"; @@ -54,8 +54,6 @@ export const prepareDB = async (options) => { if ((!usagesSlice && !namespaceSlice) || options.force) { if (comp.purl.startsWith("pkg:maven")) { hasMavenPkgs = true; - } else if (isSlicingRequired(comp.purl)) { - purlsToSlice[comp.purl] = true; } } } @@ -250,17 +248,19 @@ export const initFromSbom = (components) => { const purlLocationMap = {}; const purlImportsMap = {}; for (const comp of components) { - if (!comp || !comp.evidence || !comp.evidence.occurrences) { + if (!comp || !comp.evidence) { continue; } - purlLocationMap[comp.purl] = new Set( - comp.evidence.occurrences.map((v) => v.location) - ); (comp.properties || []) .filter((v) => v.name === "ImportedModules") .forEach((v) => { purlImportsMap[comp.purl] = (v.value || "").split(","); }); + if (comp.evidence.occurrences) { + purlLocationMap[comp.purl] = new Set( + comp.evidence.occurrences.map((v) => v.location) + ); + } } return { purlLocationMap, @@ -412,7 +412,8 @@ export const parseObjectSlices = async ( if ( !slice.fileName || !slice.fileName.trim().length || - slice.fileName === "" + slice.fileName === "" || + slice.fileName === "" ) { continue; } @@ -426,6 +427,7 @@ export const parseObjectSlices = async ( ); detectServicesFromUsages(language, slice, servicesMap); } + detectServicesFromUDT(language, usageSlice.userDefinedTypes, servicesMap); return { purlLocationMap, servicesMap, @@ -475,10 +477,13 @@ export const parseSliceUsages = async ( atype[0] !== false && !isFilterableType(language, userDefinedTypesMap, atype[1]) ) { - if (!atype[1].includes("(")) { + if (!atype[1].includes("(") && !atype[1].includes(".py")) { typesToLookup.add(atype[1]); // Javascript calls can be resolved to a precise line number only from the call nodes - if (language == "javascript" && ausageLine) { + if ( + ["javascript", "js", "ts", "typescript"].includes(language) && + ausageLine + ) { if (atype[1].includes(":")) { typesToLookup.add(atype[1].split("::")[0].replace(/:/g, "/")); } @@ -503,7 +508,10 @@ export const parseSliceUsages = async ( if ( !isFilterableType(language, userDefinedTypesMap, acall?.resolvedMethod) ) { - if (!acall?.resolvedMethod.includes("(")) { + if ( + !acall?.resolvedMethod.includes("(") && + !acall?.resolvedMethod.includes(".py") + ) { typesToLookup.add(acall?.resolvedMethod); // Javascript calls can be resolved to a precise line number only from the call nodes if (acall.lineNumber) { @@ -531,7 +539,7 @@ export const parseSliceUsages = async ( } for (const aparamType of acall?.paramTypes || []) { if (!isFilterableType(language, userDefinedTypesMap, aparamType)) { - if (!aparamType.includes("(")) { + if (!aparamType.includes("(") && !aparamType.includes(".py")) { typesToLookup.add(aparamType); if (acall.lineNumber) { if (aparamType.includes(":")) { @@ -580,16 +588,17 @@ export const parseSliceUsages = async ( } } else { // Check the namespaces db - const nsHits = - typePurlsCache[atype] || - (await dbObjMap.Namespaces.findAll({ + let nsHits = typePurlsCache[atype]; + if (["java", "jar"].includes(language)) { + nsHits = await dbObjMap.Namespaces.findAll({ attributes: ["purl"], where: { data: { [Op.like]: `%${atype}%` } } - })); + }); + } if (nsHits && nsHits.length) { for (const ns of nsHits) { if (!purlLocationMap[ns.purl]) { @@ -612,16 +621,21 @@ export const isFilterableType = ( ) => { if ( !typeFullName || - ["ANY", "UNKNOWN", "VOID"].includes(typeFullName.toUpperCase()) + ["ANY", "UNKNOWN", "VOID", "IMPORT"].includes(typeFullName.toUpperCase()) ) { return true; } - if ( - typeFullName.startsWith("") || @@ -645,13 +659,20 @@ export const isFilterableType = ( typeFullName.startsWith("{ ") || typeFullName.startsWith("JSON") || typeFullName.startsWith("void:") || - typeFullName.startsWith("LAMBDA") || - typeFullName.startsWith("../") || typeFullName.startsWith("node:") ) { return true; } } + if (["python", "py"].includes(language)) { + if ( + typeFullName.startsWith("tmp") || + typeFullName.startsWith("self.") || + typeFullName.startsWith("_") + ) { + return true; + } + } if (userDefinedTypesMap[typeFullName]) { return true; } @@ -715,6 +736,61 @@ export const detectServicesFromUsages = (language, slice, servicesMap = {}) => { } }; +/** + * Method to detect services from user defined types in the usage slice + * + * @param {string} language Application language + * @param {array} userDefinedTypes User defined types + * @param {object} servicesMap Existing service map + */ +export const detectServicesFromUDT = ( + language, + userDefinedTypes, + servicesMap +) => { + if ( + ["python", "py"].includes(language) && + userDefinedTypes && + userDefinedTypes.length + ) { + for (const audt of userDefinedTypes) { + if ( + audt.name.includes("route") || + audt.name.includes("path") || + audt.name.includes("url") + ) { + const fields = audt.fields || []; + if ( + fields.length && + fields[0] && + fields[0].name && + fields[0].name.length > 1 + ) { + const endpoints = extractEndpoints(language, fields[0].name); + let serviceName = "service"; + if (audt.fileName) { + serviceName = `${basename( + audt.fileName.replace(".py", "") + )}-service`; + } + if (!servicesMap[serviceName]) { + servicesMap[serviceName] = { + endpoints: new Set(), + authenticated: false, + xTrustBoundary: undefined + }; + } + if (endpoints) { + for (const endpoint of endpoints) { + servicesMap[serviceName].endpoints.add(endpoint); + } + } + } + } + } + } +}; + export const constructServiceName = (language, slice) => { let serviceName = "service"; if (slice?.fullName) { @@ -753,7 +829,10 @@ export const extractEndpoints = (language, code) => { ); } break; + case "js": + case "ts": case "javascript": + case "typescript": if (code.includes("app.") || code.includes("route")) { const matches = code.match(/['"](.*?)['"]/gi) || []; endpoints = matches @@ -769,24 +848,18 @@ export const extractEndpoints = (language, code) => { ); } break; + case "py": + case "python": + endpoints = (code.match(/['"](.*?)['"]/gi) || []) + .map((v) => v.replace(/["']/g, "").replace("\n", "")) + .filter((v) => v.length > 2); + break; default: break; } return endpoints; }; -/** - * Function to determine if slicing is required for the given language's dependencies. - * For performance reasons, we make java operate only with namespaces - * - * @param {string} purl - * @returns - */ -export const isSlicingRequired = (purl) => { - const language = purlToLanguage(purl); - return ["python"].includes(language); -}; - /** * Method to create the SBOM with evidence file called evinse file. * @@ -945,7 +1018,10 @@ export const collectDataFlowFrames = async ( continue; } let typeFullName = theNode.typeFullName; - if (language === "javascript" && typeFullName == "ANY") { + if ( + ["javascript", "js", "ts", "typescript"].includes(language) && + typeFullName == "ANY" + ) { if ( theNode.code && (theNode.code.startsWith("new ") || @@ -971,16 +1047,17 @@ export const collectDataFlowFrames = async ( } } else { // Check the namespaces db - const nsHits = - typePurlsCache[typeFullName] || - (await dbObjMap.Namespaces.findAll({ + let nsHits = typePurlsCache[typeFullName]; + if (["java", "jar"].includes(language)) { + nsHits = await dbObjMap.Namespaces.findAll({ attributes: ["purl"], where: { data: { [Op.like]: `%${typeFullName}%` } } - })); + }); + } if (nsHits && nsHits.length) { for (const ns of nsHits) { referredPurls.add(ns.purl); @@ -1099,7 +1176,7 @@ export const getClassTypeFromSignature = (language, typeFullName) => { const tmpA = typeFullName.split("."); tmpA.pop(); typeFullName = tmpA.join("."); - } else if (language === "javascript") { + } else if (["javascript", "js", "ts", "typescript"].includes(language)) { typeFullName = typeFullName.replace("new: ", "").replace("await ", ""); if (typeFullName.includes(":")) { const tmpA = typeFullName.split("::")[0].replace(/:/g, "/").split("/"); @@ -1108,6 +1185,15 @@ export const getClassTypeFromSignature = (language, typeFullName) => { } typeFullName = tmpA.join("/"); } + } else if (["python", "py"].includes(language)) { + typeFullName = typeFullName + .replace(".py:", "") + .replace(/\//g, ".") + .replace(".", "") + .replace(".", "") + .replace(".", "") + .replace(".__iter__", "") + .replace(".__init__", ""); } if ( typeFullName.startsWith(" { // Get the imported modules and a dedupe list of packages const parentDependsOn = new Set(); const retMap = await getPyModules(path, pkgList, options); + // We need to patch the existing package list to add ImportedModules for evinse to work + if (retMap.modList && retMap.modList.length) { + const iSymbolsMap = {}; + retMap.modList.forEach((v) => { + iSymbolsMap[v.name] = v.importedSymbols; + }); + for (const apkg of pkgList) { + if (iSymbolsMap[apkg.name]) { + apkg.properties = apkg.properties || []; + apkg.properties.push({ + name: "ImportedModules", + value: iSymbolsMap[apkg.name] + }); + } + } + } if (retMap.pkgList && retMap.pkgList.length) { pkgList = pkgList.concat(retMap.pkgList); for (const p of retMap.pkgList) { diff --git a/package-lock.json b/package-lock.json index d2e0393303..f150748853 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cyclonedx/cdxgen", - "version": "9.9.0", + "version": "9.9.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@cyclonedx/cdxgen", - "version": "9.9.0", + "version": "9.9.1", "license": "Apache-2.0", "dependencies": { "@babel/parser": "^7.23.0", @@ -56,7 +56,7 @@ "node": ">=16" }, "optionalDependencies": { - "@appthreat/atom": "1.5.2", + "@appthreat/atom": "1.5.3", "@cyclonedx/cdxgen-plugins-bin": "^1.4.0", "@cyclonedx/cdxgen-plugins-bin-arm64": "^1.4.0", "@cyclonedx/cdxgen-plugins-bin-ppc64": "^1.4.0", @@ -91,9 +91,9 @@ } }, "node_modules/@appthreat/atom": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/@appthreat/atom/-/atom-1.5.2.tgz", - "integrity": "sha512-PhL/xuuFB10+R7ErNEPgMdmoQX8B0RAQHIH3CPXe95PfiGKYh3NUea0Wt+/qjBnUxRofRVlRdns0s25sn0bsGQ==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@appthreat/atom/-/atom-1.5.3.tgz", + "integrity": "sha512-Dxt0iMdsTX2FcEx8nTlcSMt0pptt2e4swoljxJl+gxjpG0tuyh7SOO4eGPtdP7uSLnAvH7kzh23H2BzjCeVA1A==", "optional": true, "dependencies": { "@babel/parser": "^7.23.0", diff --git a/package.json b/package.json index f8c7bd05db..078d3109c8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cyclonedx/cdxgen", - "version": "9.9.0", + "version": "9.9.1", "description": "Creates CycloneDX Software Bill of Materials (SBOM) from source or container image", "homepage": "http://github.com/cyclonedx/cdxgen", "author": "Prabhu Subramanian ", @@ -83,7 +83,7 @@ "yargs": "^17.7.2" }, "optionalDependencies": { - "@appthreat/atom": "1.5.2", + "@appthreat/atom": "1.5.3", "@cyclonedx/cdxgen-plugins-bin": "^1.4.0", "@cyclonedx/cdxgen-plugins-bin-arm64": "^1.4.0", "@cyclonedx/cdxgen-plugins-bin-ppc64": "^1.4.0", diff --git a/utils.js b/utils.js index da33de703e..6e384a4dff 100644 --- a/utils.js +++ b/utils.js @@ -116,6 +116,8 @@ const MAX_LICENSE_ID_LENGTH = 100; let PYTHON_CMD = "python"; if (process.env.PYTHON_CMD) { PYTHON_CMD = process.env.PYTHON_CMD; +} else if (process.env.CONDA_PYTHON_EXE) { + PYTHON_CMD = process.env.CONDA_PYTHON_EXE; } // Custom user-agent for cdxgen @@ -2430,9 +2432,18 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) { if (p.name.includes("[")) { p.name = p.name.split("[")[0]; } - const res = await cdxgenAgent.get(PYPI_URL + p.name + "/json", { - responseType: "json" - }); + let res = undefined; + try { + res = await cdxgenAgent.get(PYPI_URL + p.name + "/json", { + responseType: "json" + }); + } catch (err) { + // retry by prefixing django- to the package name + res = await cdxgenAgent.get(PYPI_URL + "django-" + p.name + "/json", { + responseType: "json" + }); + p.name = "django-" + p.name; + } const body = res.body; if (body.info.author && body.info.author.trim() !== "") { if (body.info.author_email && body.info.author_email.trim() !== "") { @@ -2551,6 +2562,16 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) { p._integrity = "md5-" + digest["md5"]; } } + const purlString = new PackageURL( + "pypi", + "", + p.name, + p.version, + null, + null + ).toString(); + p.purl = purlString; + p["bom-ref"] = decodeURIComponent(purlString); cdepList.push(p); } catch (err) { if (DEBUG_MODE) { @@ -2584,6 +2605,16 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) { } }; } + const purlString = new PackageURL( + "pypi", + "", + p.name, + p.version, + null, + null + ).toString(); + p.purl = purlString; + p["bom-ref"] = decodeURIComponent(purlString); cdepList.push(p); } } @@ -2973,35 +3004,29 @@ export const getPyModules = async (src, epkgList, options) => { const allImports = {}; const dependenciesList = []; let modList = []; + const slicesFile = resolve( + options.depsSlicesFile || options.usagesSlicesFile + ); // Issue: 615 fix. Reuse existing slices file - // FIXME: The argument is called usagesSlicesFile while the atom command used is parsedeps. - // This logic could be rewritten while implementing evinse for python to that the analysis works for either type of slice - if (options.usagesSlicesFile && existsSync(options.usagesSlicesFile)) { - const slicesData = JSON.parse( - readFileSync(options.usagesSlicesFile, "utf-8") - ); + if (slicesFile && existsSync(slicesFile)) { + const slicesData = JSON.parse(readFileSync(slicesFile, "utf-8")); if (slicesData && Object.keys(slicesData) && slicesData.modules) { modList = slicesData.modules; } else { modList = slicesData; } } else { - modList = findAppModules( - src, - "python", - "parsedeps", - options.usagesSlicesFile - ); + modList = findAppModules(src, "python", "parsedeps", slicesFile); } const pyDefaultModules = new Set(PYTHON_STD_MODULES); - const filteredModList = modList.filter( + modList = modList.filter( (x) => !pyDefaultModules.has(x.name.toLowerCase()) && !x.name.startsWith("_") && !x.name.startsWith(".") ); - let pkgList = filteredModList.map((p) => { - return { + let pkgList = modList.map((p) => { + const apkg = { name: PYPI_MODULE_PACKAGE_MAPPING[p.name.toLowerCase()] || PYPI_MODULE_PACKAGE_MAPPING[p.name.replace(/_/g, "-").toLowerCase()] || @@ -3015,6 +3040,13 @@ export const getPyModules = async (src, epkgList, options) => { } ] }; + if (p.importedSymbols) { + apkg.properties.push({ + name: "ImportedModules", + value: p.importedSymbols + }); + } + return apkg; }); pkgList = pkgList.filter( (obj, index) => pkgList.findIndex((i) => i.name === obj.name) === index @@ -3038,7 +3070,7 @@ export const getPyModules = async (src, epkgList, options) => { }); } } - return { allImports, pkgList, dependenciesList }; + return { allImports, pkgList, dependenciesList, modList }; }; /** @@ -6862,9 +6894,19 @@ const flattenDeps = (dependenciesMap, pkgList, reqOrSetupFile, t) => { if (!dependenciesMap[pkgRef]) { dependenciesMap[pkgRef] = []; } + const purlString = new PackageURL( + "pypi", + "", + d.name, + d.version, + null, + null + ).toString(); pkgList.push({ name: d.name, version: d.version, + purl: purlString, + "bom-ref": decodeURIComponent(purlString), properties: [ { name: "SrcFile", @@ -6874,11 +6916,11 @@ const flattenDeps = (dependenciesMap, pkgList, reqOrSetupFile, t) => { evidence: { identity: { field: "purl", - confidence: 1, + confidence: 0.8, methods: [ { technique: "manifest-analysis", - confidence: 1, + confidence: 0.8, value: reqOrSetupFile } ] @@ -6918,6 +6960,7 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => { */ if ( !process.env.VIRTUAL_ENV && + !process.env.CONDA_PREFIX && reqOrSetupFile && !reqOrSetupFile.endsWith("poetry.lock") ) { @@ -7106,7 +7149,10 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => { } } // Bug #375. Attempt pip freeze on existing and new virtual environments - if (env.VIRTUAL_ENV && env.VIRTUAL_ENV.length) { + if ( + (env.VIRTUAL_ENV && env.VIRTUAL_ENV.length) || + (env.CONDA_PREFIX && env.CONDA_PREFIX.length) + ) { /** * At this point, the previous attempt to do a pip install might have failed and we might have an unclean virtual environment with an incomplete list * The position taken by cdxgen is "Some SBOM is better than no SBOM", so we proceed to collecting the dependencies that got installed with pip freeze @@ -7136,9 +7182,19 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => { const version = t.version; let exclude = ["pip", "setuptools", "wheel"]; if (!exclude.includes(name)) { + const purlString = new PackageURL( + "pypi", + "", + name, + version, + null, + null + ).toString(); pkgList.push({ name, version, + purl: purlString, + "bom-ref": decodeURIComponent(purlString), evidence: { identity: { field: "purl", @@ -7147,7 +7203,7 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => { { technique: "instrumentation", confidence: 1, - value: env.VIRTUAL_ENV + value: env.VIRTUAL_ENV || env.CONDA_PREFIX } ] }