From a16c3a1211512426a5917258316cf6570b5342dc Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 16:20:34 -0700 Subject: [PATCH 1/9] Adding CodeQL workflow item to detect vulnerabilities. --- .github/workflows/codeql.yml | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..c601c67b --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,100 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# For more information see: +# https://nasa.github.io/scrub/ +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [main, develop] + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + schedule: + # default branch on sundays at 5a + - cron: '0 5 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + # CodeQL supports ['cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby'] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + language: ['python'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + #config-file: ./.github/workflows/codeql/codeql-config.yml + languages: ${{ matrix.language }} + queries: security-and-quality, security-extended + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + - name: Post-Process Output + run: | + python3 -m pip install nasa-scrub + + results_dir=`realpath ${{ github.workspace }}/../results` + sarif_files=`find $results_dir -name '*.sarif'` + + for sarif_file in $sarif_files + do + output_file="$results_dir/$(basename $sarif_file .sarif).scrub" + + python3 -m scrub.tools.parsers.translate_results $sarif_file $output_file ${{ github.workspace }} scrub + done + + python3 -m scrub.tools.parsers.csv_parser $results_dir + + echo "RESULTS_DIR=$results_dir" >> $GITHUB_ENV + + + - name: Upload CodeQL Artifacts + uses: actions/upload-artifact@v4 + with: + name: codeql-artifacts + path: ${{ env.RESULTS_DIR }} + if-no-files-found: error + overwrite: true + retention-days: 15 \ No newline at end of file From 9a2954caa3107bacd423710ed4e1e4ead1ef6d48 Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 16:25:46 -0700 Subject: [PATCH 2/9] Added Ruff as a linter to clean up and detect errors. --- .github/workflows/ruff.yml | 69 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 00000000..a3f4ffaf --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,69 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to disable certain Ruff checks in the +# "Analyze" configuration block below. +# +# For more information see: +# https://nasa-ammos.github.io/slim/docs/guides/software-lifecycle/application-starter-kits/python-starter-kit/ +# +# ******** NOTE ******** +# Ruff is a Python-based linter that works to evaluate Python code. +# +name: "Ruff" + +on: + push: + branches: [main, develop] + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + security-events: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Upgrade tooling + run: | + python3 -m pip install --upgrade pip + pip3 install --upgrade build importlib_metadata setuptools setuptools_scm wheel + pip3 install ruff + - name: Install dependencies + run: | + pip3 install -r requirements.txt + pip3 install -e . + - name: Prepare PYTHONPATH + run: | + src_paths=`find ${PWD} -type f -maxdepth 3 -mindepth 2 -name "*.py" -exec dirname {} + | uniq` + pythonpathplus="" + for p in $src_paths + do + pythonpathplus="${pythonpathplus:+:${pythonpathplus}}:$p" + done + echo "PYTHONPATH=${PYTHONPATH:+:${PYTHONPATH}}${pythonpathplus}:." >> $GITHUB_ENV + - name: Analyze + # Refer to https://docs.astral.sh/ruff/configuration/#command-line-interface + # to add extra rules or a configuration file. + run: ruff check --output-file ruff_report.txt . || true + continue-on-error: true + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: ruff_report + path: ruff_report.txt + if-no-files-found: error + overwrite: true + retention-days: 15 \ No newline at end of file From 0be604eb8f08b3029239aee845800db01f75a1b4 Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 18:56:46 -0700 Subject: [PATCH 3/9] Testing release on Testpypi --- .github/workflows/codeql.yml | 7 ++----- .github/workflows/codeql/codeql-config.yml | 5 +++++ .github/workflows/python-publish.yml | 2 +- .github/workflows/ruff.yml | 12 +++++++----- exotic/version.py | 2 +- 5 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/codeql/codeql-config.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c601c67b..9a3d75ae 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -15,10 +15,7 @@ name: "CodeQL" on: - push: - branches: [main, develop] pull_request: - # The branches below must be a subset of the branches above branches: [develop] schedule: # default branch on sundays at 5a @@ -49,9 +46,9 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: - #config-file: ./.github/workflows/codeql/codeql-config.yml + config-file: ./.github/workflows/codeql/codeql-config.yml languages: ${{ matrix.language }} - queries: security-and-quality, security-extended + # queries: security-and-quality, security-extended # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. diff --git a/.github/workflows/codeql/codeql-config.yml b/.github/workflows/codeql/codeql-config.yml new file mode 100644 index 00000000..f1491c24 --- /dev/null +++ b/.github/workflows/codeql/codeql-config.yml @@ -0,0 +1,5 @@ +name: "CodeQL config" +queries: + - name: Run custom queries + - uses: security-extended + - uses: security-and-quality \ No newline at end of file diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 1d71cc3e..8bad4eed 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -58,4 +58,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: print-hash: true - repository-url: https://pypi.org/p/exotic/ # for testing sub https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index a3f4ffaf..55c32317 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -13,10 +13,7 @@ name: "Ruff" on: - push: - branches: [main, develop] pull_request: - # The branches below must be a subset of the branches above branches: [develop] jobs: @@ -24,27 +21,30 @@ jobs: name: Analyze runs-on: ubuntu-latest permissions: - actions: write contents: read - security-events: write + steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.10' + - name: Upgrade tooling run: | python3 -m pip install --upgrade pip pip3 install --upgrade build importlib_metadata setuptools setuptools_scm wheel pip3 install ruff + - name: Install dependencies run: | pip3 install -r requirements.txt pip3 install -e . + - name: Prepare PYTHONPATH run: | src_paths=`find ${PWD} -type f -maxdepth 3 -mindepth 2 -name "*.py" -exec dirname {} + | uniq` @@ -54,11 +54,13 @@ jobs: pythonpathplus="${pythonpathplus:+:${pythonpathplus}}:$p" done echo "PYTHONPATH=${PYTHONPATH:+:${PYTHONPATH}}${pythonpathplus}:." >> $GITHUB_ENV + - name: Analyze # Refer to https://docs.astral.sh/ruff/configuration/#command-line-interface # to add extra rules or a configuration file. run: ruff check --output-file ruff_report.txt . || true continue-on-error: true + - name: Upload Artifact uses: actions/upload-artifact@v4 with: diff --git a/exotic/version.py b/exotic/version.py index fa721b49..47cbba72 100644 --- a/exotic/version.py +++ b/exotic/version.py @@ -1 +1 @@ -__version__ = '4.1.0' +__version__ = '4.1.1' From 1d3f6eba3969f2be95f4c62c2397034bc84ae51b Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 19:14:07 -0700 Subject: [PATCH 4/9] Revert back to Pypi for publishing. --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 8bad4eed..1d71cc3e 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -58,4 +58,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: print-hash: true - repository-url: https://test.pypi.org/legacy/ + repository-url: https://pypi.org/p/exotic/ # for testing sub https://test.pypi.org/legacy/ From 92067a68a159f0cbf50ac8d3cb2589ba9debd528 Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 22:50:27 -0700 Subject: [PATCH 5/9] Try to publish on TestPyPi --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 1d71cc3e..8bad4eed 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -58,4 +58,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: print-hash: true - repository-url: https://pypi.org/p/exotic/ # for testing sub https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ From 7f5774a5a9f3c9010196a5ac4815940c4e7b8c0b Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 23:15:11 -0700 Subject: [PATCH 6/9] temp check --- .github/workflows/python-publish.yml | 46 +++++++++++----------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 8bad4eed..c5062f38 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,61 +1,51 @@ -# This workflows will upload a Python Package when a release is created -# For more information see: -# https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: Upload Python Package +name: Publish Package to TestPyPi on: release: types: [ published ] jobs: - build: runs-on: ubuntu-latest + steps: - name: Checkout code uses: actions/checkout@v4 - with: - fetch-depth: 0 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' - - name: Upgrade tooling + python-version: "3.11" + + - name: Install pypa/build run: | python3 -m pip install --upgrade pip - pip3 install --upgrade build setuptools setuptools_scm[toml] twine wheel + pip3 install --upgrade build --user + - name: Build package - run: | - python3 -m build --wheel - python3 setup.py sdist --format=zip - - name: Verify package - run: | - twine check dist/* - - name: Store package + run: python3 -m build + + - name: Store packages uses: actions/upload-artifact@v4 with: - name: python-package-distribution - path: | - dist/*.whl - dist/*.zip - if-no-files-found: error + name: python-package-distributions + path: dist/ release: runs-on: ubuntu-latest needs: build environment: - name: release + name: test-pypi permissions: - id-token: write # mandatory for trusted publishing + id-token: write # IMPORTANT: mandatory for trusted publishing steps: - name: Retrieve package uses: actions/download-artifact@v4 with: - name: python-package-distribution + name: python-package-distributions path: dist/ - - name: Publish package (PyPi) + + - name: Publish package to TestPyPi uses: pypa/gh-action-pypi-publish@release/v1 with: - print-hash: true repository-url: https://test.pypi.org/legacy/ From 628d159da6ade4fc07232951004e331fa5184fe8 Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 23:23:15 -0700 Subject: [PATCH 7/9] temp check --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index c5062f38..ec6e7420 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest needs: build environment: - name: test-pypi + name: release permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: From e4f6101bbd62ddd95ceed90a92e7864889579ebd Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 23:54:09 -0700 Subject: [PATCH 8/9] temp check --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index ec6e7420..95de04d5 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest needs: build environment: - name: release + name: rele permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: From dc7bc7cbc14ed2cc752d283819d2747c964e89bd Mon Sep 17 00:00:00 2001 From: Tamim Fatahi Date: Fri, 27 Sep 2024 23:55:51 -0700 Subject: [PATCH 9/9] temp check --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 95de04d5..ec6e7420 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest needs: build environment: - name: rele + name: release permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: