From 085c2641df840b2dbfcd95f3ad373ee15f9c6e5e Mon Sep 17 00:00:00 2001
From: barshaul <barshaul@amazon.com>
Date: Mon, 5 Feb 2024 10:09:12 +0000
Subject: [PATCH] Added ORT workflow

---
 .../workflows/build-node-wrapper/action.yml   |  34 +---
 .../node-create-package-file/action.yml       |  66 +++++++
 .github/workflows/ort.yml                     | 182 ++++++++++++++++++
 .github/workflows/run-ort-tools/action.yml    |  22 +++
 4 files changed, 278 insertions(+), 26 deletions(-)
 create mode 100644 .github/workflows/node-create-package-file/action.yml
 create mode 100644 .github/workflows/ort.yml
 create mode 100644 .github/workflows/run-ort-tools/action.yml

diff --git a/.github/workflows/build-node-wrapper/action.yml b/.github/workflows/build-node-wrapper/action.yml
index 19e21ad4b0..0e0e08fcaa 100644
--- a/.github/workflows/build-node-wrapper/action.yml
+++ b/.github/workflows/build-node-wrapper/action.yml
@@ -38,7 +38,7 @@ inputs:
         description: "The NPM scope"
         required: false
         type: string
-        default: ""
+        default: "@aws"
     github-token:
         description: "GITHUB_TOKEN, GitHub App installation access token"
         required: true
@@ -58,31 +58,13 @@ runs:
               github-token: ${{ inputs.github-token }}
 
         - name: Create package.json file
-          shell: bash
-          working-directory: ./node
-          run: |
-              # set the package name
-              name="glide-for-redis"
-              # derive the OS and architecture from the inputs
-              export node_os="${{ inputs.named_os }}"
-              export node_arch="${{ inputs.arch }}"
-              # set the version
-              export package_version="${{ env.RELEASE_VERSION }}"
-              # set the package name
-              export pkg_name="${name}-${node_os}-${node_arch}"
-              # set the scope
-              export scope=`if [ "${{ inputs.npm_scope }}" != ''  ]; then echo "${{ inputs.npm_scope }}/"; fi`
-              # set the registry scope
-              export registry_scope=`if [ "${{ inputs.npm_scope }}" != ''  ]; then echo "${{ inputs.npm_scope }}:"; fi`
-              # remove the current name section
-              SED_FOR_MACOS=`if [[ "${{ inputs.os }}" =~ .*"macos".*  ]]; then echo "''"; fi`
-              sed -i $SED_FOR_MACOS '/"name":/d' ./package.json
-              # Remove all `///` occurrences to enable the commented out sections
-              sed -i -e 's|///||g' package.json
-              # generate package.json from the template
-              mv package.json package.json.tmpl
-              envsubst < package.json.tmpl > "package.json"
-              cat package.json
+          uses: .github/workflows/node-create-package-file
+          with:
+            release_version: ${{ env.RELEASE_VERSION }}
+            os: ${{ inputs.os }}
+            named_os: ${{ inputs.named_os }}
+            arch: ${{ inputs.arch }}
+            npm_scope: ${{ inputs.npm_scope }}
 
         - name: npm install
           shell: bash
diff --git a/.github/workflows/node-create-package-file/action.yml b/.github/workflows/node-create-package-file/action.yml
new file mode 100644
index 0000000000..bdadeeb8ba
--- /dev/null
+++ b/.github/workflows/node-create-package-file/action.yml
@@ -0,0 +1,66 @@
+name: Create the package.json main file for the Node wrapper
+
+inputs:
+    release_version:
+        description: "The package release version"
+        required: true
+        type: string
+    os:
+        description: "The current operating system"
+        required: true
+        type: string
+        options:
+            - amazon-linux
+            - macos-latest
+            - ubuntu-latest
+    named_os:
+        description: "The name of the current operating system"
+        required: false
+        default: "linux"
+        type: string
+        options:
+            - linux
+            - darwin
+    arch:
+        description: "The current architecture"
+        required: false
+        default: "x64"
+        type: string
+        options:
+            - x64
+            - arm64
+    npm_scope:
+        description: "The NPM scope"
+        required: false
+        type: string
+        default: "@aws"
+
+runs:
+    using: "composite"
+    steps:
+        - name: Create package.json file
+          shell: bash
+          working-directory: ./node
+          run: |
+              # set the package name
+              name="glide-for-redis"
+              # derive the OS and architecture from the inputs
+              export node_os="${{ inputs.named_os }}"
+              export node_arch="${{ inputs.arch }}"
+              # set the version
+              export package_version="${{ inputs.release_version }}"
+              # set the package name
+              export pkg_name="${name}-${node_os}-${node_arch}"
+              # set the scope
+              export scope=`if [ "${{ inputs.npm_scope }}" != ''  ]; then echo "${{ inputs.npm_scope }}/"; fi`
+              # set the registry scope
+              export registry_scope=`if [ "${{ inputs.npm_scope }}" != ''  ]; then echo "${{ inputs.npm_scope }}:"; fi`
+              # remove the current name section
+              SED_FOR_MACOS=`if [[ "${{ inputs.os }}" =~ .*"macos".*  ]]; then echo "''"; fi`
+              sed -i $SED_FOR_MACOS '/"name":/d' ./package.json
+              # Remove all `///` occurrences to enable the commented out sections
+              sed -i -e 's|///||g' package.json
+              # generate package.json from the template
+              mv package.json package.json.tmpl
+              envsubst < package.json.tmpl > "package.json"
+              cat package.json
diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml
new file mode 100644
index 0000000000..456fbb750b
--- /dev/null
+++ b/.github/workflows/ort.yml
@@ -0,0 +1,182 @@
+
+name: The OSS Review Toolkit (ORT)
+
+on:
+    schedule:
+      - cron: "0 0 * * *"
+    push:
+        tags:
+            - "ort-for-v*.*"
+jobs:
+    run-ort:
+        name: Create attribution files
+        runs-on: ubuntu-latest
+        strategy:
+          fail-fast: false
+        env: 
+          PYTHON_ATTRIBUIOTNS: "python/THIRD_PARTY_LICENSES_PYTHON"
+          NODE_ATTRIBUIOTNS: "node/THIRD_PARTY_LICENSES_NODE"
+          RUST_ATTRIBUIOTNS: "glide-core/THIRD_PARTY_LICENSES_RUST"
+        steps:
+            - name: Set the release version
+              shell: bash
+              run: |
+                  export version=`if ${{ github.event_name == 'schedule' }}; then echo '255.255.255'; else echo ${GITHUB_REF:19}; fi`
+                  echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV
+              
+            - name: Set the base branch
+              run: |
+                export BASE_BRANCH=`if ${{ github.event_name == 'schedule' }}; then echo "main"; else echo "v${{ env.RELEASE_VERSION }}"; fi`
+                echo "Base branch is: ${BASE_BRANCH}"
+                echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV
+
+            - name: Checkout
+              uses: actions/checkout@v4
+              with:
+                  submodules: "true"
+
+            - name: Set up JDK 11 for the ORT package
+              uses: actions/setup-java@v3
+              with:
+                  distribution: "temurin"
+                  java-version: 11
+
+            - name: Cache ORT and Gradle packages
+              uses: actions/cache@v4
+              id: cache-ort
+              with:
+                path: |
+                  ./ort
+                  ~/.gradle/caches
+                  ~/.gradle/wrapper
+                key: ${{ runner.os }}-ort
+
+            - name: Checkout ORT Repository
+              if: steps.cache-ort.outputs.cache-hit != 'true'
+              uses: actions/checkout@v4
+              with: 
+                  repository: "oss-review-toolkit/ort"
+                  path: "./ort"
+                  ref: main
+                  submodules: recursive
+
+            - name: Checkout ORT latest release tag
+              if: steps.cache-ort.outputs.cache-hit != 'true'
+              working-directory: ./ort/
+              run: |
+                # Get new tags from remote
+                git fetch --tags
+                # Get latest tag name
+                LATEST_TAG=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+                # Checkout latest tag
+                git checkout $LATEST_TAG
+
+            - name: Install ORT
+              if: steps.cache-ort.outputs.cache-hit != 'true'
+              working-directory: ./ort/
+              run: |
+                export JAVA_OPTS="$JAVA_OPTS -Xmx8g"
+                ./gradlew installDist
+
+            - name: Create ORT config file
+              run: |
+                mkdir -p ~/.ort/config
+                cat << EOF > ~/.ort/config/config.yml
+                ort:
+                  analyzer:
+                    allowDynamicVersions: true
+                    enabledPackageManagers: [Cargo, NPM, PIP]
+                EOF
+                cat ~/.ort/config/config.yml
+
+          ### NODE ###
+
+            - name: Set up Node.js 16.x
+              uses: actions/setup-node@v3
+              with:
+                  node-version: 16.x
+
+            - name: Create package.json file for the Node wrapper 
+              uses: ./.github/workflows/node-create-package-file
+              with:
+                release_version: ${{ env.RELEASE_VERSION }}
+                os: "ubuntu-latest"
+
+            - name: Fix Node base NPM package.json file for ORT
+              working-directory: ./node/npm/glide
+              run: |
+                # Remove the glide-rs dependency to avoid duplication 
+                sed -i '/ "glide-rs":/d' ../../package.json
+                export pkg_name=glide-for-redis-base
+                export package_version="${{ env.RELEASE_VERSION }}"
+                export scope=`if [ "$NPM_SCOPE" != ''  ]; then echo "$NPM_SCOPE/"; fi`
+                mv package.json package.json.tmpl
+                envsubst < package.json.tmpl > "package.json"
+                cat package.json
+            
+            - name: Run ORT tools for Node
+              uses: ./.github/workflows/run-ort-tools
+              with:
+                folder_name: "node"
+            
+          ### Python ###
+
+            - name: Set up Python 3.10
+              uses: actions/setup-python@v4
+              with:
+                  python-version: "3.10"
+
+            - name: Install python-inspector
+              working-directory: ./python
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install git+https://github.com/nexB/python-inspector
+
+            - name: Run ORT tools for Python
+              uses: ./.github/workflows/run-ort-tools
+              with:
+                folder_name: "python"
+
+            ### Rust ###
+
+            - name: Run ORT tools for Rust
+              uses: ./.github/workflows/run-ort-tools
+              with:
+                folder_name: "glide-core"
+
+            - name: Check for diff
+              run: |
+                cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUIOTNS
+                cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUIOTNS
+                cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUIOTNS
+                GIT_DIFF=`git diff $PYTHON_ATTRIBUIOTNS $NODE_ATTRIBUIOTNS $RUST_ATTRIBUIOTNS`
+                if [ -n "$GIT_DIFF" ]; then
+                echo "FOUND_DIFF=true" >> $GITHUB_ENV
+                else
+                echo "FOUND_DIFF=false" >> $GITHUB_ENV
+                fi
+                echo "FOUND_DIFF == ${{ env.FOUND_DIFF }}"
+
+            - name: Retrieve licenses list
+              working-directory: ./utils
+              run: |
+                LICENSES_LIST=`python3 ort/get_licenses_from_ort.py`
+                echo "LICENSES_LIST=$LICENSES_LIST" >> $GITHUB_ENV
+                
+            ### Create PR ###
+
+            - name: create pull request
+              if: ${{ env.FOUND_DIFF  == 'true' }}
+              run: |
+                export BRANCH_NAME=`if ${{ github.event_name == 'schedule' }}  ]; then echo "scheduled-ort"; else echo "ort-${{ env.RELEASE_VERSION }}"; fi`
+                echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}"
+                git config --global user.email "barshaul@amazon.com"
+                git config --global user.name "barshaul"
+                git checkout -b ${BRANCH_NAME}
+
+                git add $PYTHON_ATTRIBUIOTNS $NODE_ATTRIBUIOTNS $RUST_ATTRIBUIOTNS
+                git commit -m "Updated attribution files"
+                git push --set-upstream origin ${BRANCH_NAME}
+                gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title 'Updated attribution files for ${BRANCH_NAME}' --body 'Created by Github action.\n${{env.LICENSES_LIST}}'
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/run-ort-tools/action.yml b/.github/workflows/run-ort-tools/action.yml
new file mode 100644
index 0000000000..0239b1ad14
--- /dev/null
+++ b/.github/workflows/run-ort-tools/action.yml
@@ -0,0 +1,22 @@
+name: Run the OSS review tool
+
+inputs:
+    folder_name:
+        description: "The folder to run the ORT tool in"
+        required: true
+        type: string
+
+runs:
+    using: "composite"
+    steps:
+        - name: Run ORT tools
+        working-directory: ./ort/
+        run: |
+          echo "Running ORT tools for ${{ inputs.folder_name }}"
+          FOLDER=${{ inputs.folder_name }}
+          mkdir $GITHUB_WORKSPACE/$FOLDER/ort_results
+          # Analyzer (analyzer-result.json)
+          ./gradlew cli:run --args="analyze -i $GITHUB_WORKSPACE/$FOLDER -o $GITHUB_WORKSPACE/$FOLDER/ort_results -f JSON"
+          
+          # NOTICE DEFAULT
+          ./gradlew cli:run --args="report -i $GITHUB_WORKSPACE/$FOLDER/ort_results/analyzer-result.json -o $GITHUB_WORKSPACE/$FOLDER/ort_results/ -f PlainTextTemplate"