Merge branch 'master' into yk/update-request-promise-native

.prettierignore

@@ -0,0 +1,5 @@
+.github/workflows/*
+.vscode/launch.json
+schemas/*.json
+test/**/*.json
+test/**/*.yaml

.vscode/launch.json

@@ -14,12 +14,16 @@
       "name": "ClearlyDefined",
       "program": "${workspaceRoot}/index.js",
       "cwd": "${workspaceRoot}",
-      "runtimeArgs": ["--nolazy"],
+      "runtimeArgs": [
+        "--nolazy"
+      ],
       "env": {
         "NODE_ENV": "localhost",
         "PORT": "5000"
       },
-      "skipFiles": ["<node_internals>/**/*.js"],
+      "skipFiles": [
+        "<node_internals>/**/*.js"
+      ],
       "showAsyncStacks": true
     },
     {
@@ -28,15 +32,23 @@
       "name": "Mocha",
       "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
       "stopOnEntry": false,
-      "args": ["${workspaceRoot}/test/**/*.js", "--timeout", "999999"],
+      "args": [
+        "${workspaceRoot}/test/**/*.js",
+        "--timeout",
+        "999999"
+      ],
       "cwd": "${workspaceRoot}",
       "runtimeExecutable": null,
-      "runtimeArgs": ["--nolazy"],
+      "runtimeArgs": [
+        "--nolazy"
+      ],
       "env": {
         "NODE_ENV": "localhost"
       },
       "console": "internalConsole",
-      "skipFiles": ["<node_internals>/**/*.js"],
+      "skipFiles": [
+        "<node_internals>/**/*.js"
+      ],
       "showAsyncStacks": true
     }
   ]

DevDockerfile

@@ -16,12 +16,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends --no-install-su
   gem install bundler -v 2.5.4 --no-document
 
 # Scancode
-ARG SCANCODE_VERSION="30.1.0"
+ARG SCANCODE_VERSION="32.1.0"
 RUN pip3 install --upgrade pip setuptools wheel && \
   curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt && \
   pip3 install --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION && \
   rm requirements.txt && \
-  scancode --reindex-licenses && \
+  scancode-reindex-licenses && \
   scancode --version
 
 ENV SCANCODE_HOME=/usr/local/bin

Dockerfile

@@ -1,14 +1,8 @@
 # Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 # SPDX-License-Identifier: MIT
 
-#FROM fossology/fossology:3.4.0 as fossology
-#COPY fossology_init.sh fossology_init.sh
-#RUN ./fossology_init.sh
-
 FROM node:18-bullseye
 ENV APPDIR=/opt/service
-#RUN apk update && apk upgrade && \
-#    apk add --no-cache bash git openssh
 
 ARG BUILD_NUMBER=0
 ENV CRAWLER_BUILD_NUMBER=$BUILD_NUMBER
@@ -22,12 +16,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends --no-install-su
   gem install bundler -v 2.5.4 --no-document
 
 # Scancode
-ARG SCANCODE_VERSION="30.1.0"
+ARG SCANCODE_VERSION="32.1.0"
 RUN pip3 install --upgrade pip setuptools wheel && \
   curl -Os https://raw.githubusercontent.com/nexB/scancode-toolkit/v$SCANCODE_VERSION/requirements.txt && \
   pip3 install --constraint requirements.txt scancode-toolkit==$SCANCODE_VERSION && \
   rm requirements.txt && \
-  scancode --reindex-licenses && \
+  scancode-reindex-licenses && \
   scancode --version
 
 ENV SCANCODE_HOME=/usr/local/bin
@@ -45,32 +39,6 @@ RUN gem install nokogiri:1.16.0 --no-document && \
 RUN pip3 install setuptools
 RUN pip3 install reuse==3.0.1
 
-# FOSSology
-# WORKDIR /opt
-# RUN git clone https://github.com/fossology/fossology.git
-# RUN cd fossology && git checkout -b clearlydefined tags/3.4.0
-
-# See https://github.com/fossology/fossology/blob/faaaeedb9d08f00def00f9b8a68a5cffc5eaa657/utils/fo-installdeps#L103-L105
-# Additional libjsoncpp-dev https://github.com/fossology/fossology/blob/261d1a3e663b5fd20652a05b2d6360f4b31a17cb/src/copyright/mod_deps#L79-L80
-# RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests \
-#  libmxml-dev curl libxml2-dev libcunit1-dev libjsoncpp-dev \
-#  build-essential libtext-template-perl subversion rpm librpm-dev libmagic-dev libglib2.0 libboost-regex-dev libboost-program-options-dev
-
-# WORKDIR /opt/fossology/src/nomos/agent
-# RUN make -f Makefile.sa
-# RUN echo $(./nomossa -V)
-
-# NOTE: must build copyright before Monk to cause libfossology to be built
-# WORKDIR /opt/fossology/src/copyright/agent
-# RUN make
-
-# WORKDIR /opt/fossology/src/monk/agent
-# RUN make
-# RUN echo $(./monk -V)
-# COPY --from=fossology /tmp/monk_knowledgebase .
-
-# ENV FOSSOLOGY_HOME=/opt/fossology/src
-
 # Crawler config
 ENV CRAWLER_DEADLETTER_PROVIDER=cd(azblob)
 ENV CRAWLER_NAME=cdcrawlerprod
@@ -84,7 +52,7 @@ RUN git config --global --add safe.directory '*'
 
 COPY package*.json /tmp/
 COPY patches /tmp/patches
-RUN cd /tmp && npm install --production
+RUN cd /tmp && npm install --omit=dev
 RUN mkdir -p "${APPDIR}" && cp -a /tmp/node_modules "${APPDIR}"
 
 WORKDIR "${APPDIR}"

SECURITY.md

@@ -0,0 +1,71 @@
+# Vulnerability Disclosure and Embargo Policy
+
+The ClearlyDefined project welcomes the responsible disclosure of vulnerabilities, including those discovered in:
+
+- [ClearlyDefined website](https://github.com/clearlydefined/website/security/advisories/new)
+- [ClearlyDefined service](https://github.com/clearlydefined/service/security/advisories/new)
+- [ClearlyDefined crawler](https://github.com/clearlydefined/crawler/security/advisories/new)
+- [ClearlyDefined documentation](https://github.com/clearlydefined/clearlydefined/security/advisories/new)
+
+## Initial Contact
+
+All security bugs in ClearlyDefined should be reported to the security team.
+To do so, please reach out in the form of a
+[Github Security Advisory](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities).
+
+You will be invited to join this private area to discuss specifics. Doing so
+allows us to start with a high level of confidentiality and relax it if the
+issue is less critical, moving to work on the fix in the open.
+
+Your initial contact will be acknowledged within 48 hours, and you’ll receive
+a more detailed response within 96 hours indicating the next steps in handling
+your report.
+
+After the initial reply to your report, the security team will endeavor to
+keep you informed of the progress being made towards a fix and full
+announcement. As recommended by
+[RFPolicy](https://dl.packetstormsecurity.net/papers/general/rfpolicy-2.0.txt),
+these updates will be sent at least every five working days.
+
+## Disclosure Policy
+
+The ClearlyDefined project has a 5 step disclosure process.
+
+1. Contact is established, a private channel created, and the security report
+   is received and is assigned a primary handler. This person will coordinate
+   the fix and release process.
+2. The problem is confirmed and a list of all affected versions is determined.
+   If an embargo is needed (see below), details of the embargo are decided.
+3. Code is audited to find any potential similar problems.
+4. Fixes are prepared for all releases which are still under maintenance. In
+   case of embargo, these fixes are not committed to the public repository but
+   rather held in a private fork pending the announcement.
+5. The changes are pushed to the public repository and new builds are deployed.
+
+This process can take some time, especially when coordination is required
+with maintainers of other projects. Every effort will be made to handle the bug
+in as timely a manner as possible, however it is important that we follow the
+release process above to ensure that the disclosure is handled in a consistent
+manner.
+
+## Embargoes
+
+While the ClearlyDefined project aims to follow the highest standards of
+transparency and openness, handling some security issues may pose such an
+immediate threat to various stakeholders and require coordination between
+various actors that it cannot be made immediately public.
+
+In this case, security issues will fall under an embargo.
+
+An embargo can be called for in various cases:
+
+- when disclosing the issue without simultaneously providing a mitigation
+  would seriously endanger users,
+- when producing a fix requires coordinating between multiple actors (such as
+  upstream or downstream/dependency projects), or simply
+- when proper analysis of the issue and its ramifications demands time.
+
+If we determine that an issue you report requires an embargo, we will discuss
+this with you and try to find a reasonable expiry date (aka “embargo
+completion date”), as well as who should be included in the list of
+need-to-know people.

config/cdConfig.js

@@ -77,20 +77,21 @@ module.exports = {
       installDir: config.get('SCANCODE_HOME'),
       options: [
         '--copyright',
-        '--license',
         '--info',
-        '--license-text',
-        '--is-license-text',
         '--package',
-        '--license-text-diagnostics',
         '--strip-root',
         '--email',
         '--url',
-        '--license-clarity-score',
         '--classify',
         '--generated',
+        '--license',
+        '--license-clarity-score',
+        '--license-references',
+        '--license-text',
+        '--license-text-diagnostics',
         '--summary',
-        '--summary-key-files'
+        '--tallies',
+        '--tallies-key-files'
         // '--quiet'
       ],
       timeout: 1000,