From d5035fa0fa18b5bd47bd3e3c35f896ef1cfa5390 Mon Sep 17 00:00:00 2001 From: Jack Kavanagh Date: Thu, 23 Jan 2025 12:34:07 +0100 Subject: [PATCH] Feature/ins-4917-set-up-linux-arm-build-to-support-flatpak (#82) * add target arch flag * fix cache * fix target_arch * fix electron jobs * skip failing exception tests * add arm to publish action * remove test skip and change url on easy.spec.ts * revert concurrency group setting ref to head_ref * Update test/curl/easy.spec.ts --------- Co-authored-by: Filipe Freire --- .github/workflows/build-and-release.yaml | 20 ++++++------ .github/workflows/build-lint-test.yaml | 39 ++++++++++++++---------- .github/workflows/new-version.yml | 2 +- .github/workflows/publish.yaml | 2 +- .github/workflows/sast.yml | 2 +- scripts/ci/build.sh | 12 +++++--- test/curl/easy.spec.ts | 3 +- 7 files changed, 46 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 93bd94808..103de1394 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -22,7 +22,7 @@ jobs: permissions: packages: write contents: write # publish sbom to GH releases/tag assets - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -49,6 +49,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node-libcurl-cpp-std: @@ -82,16 +83,16 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork + key: v4-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork restore-keys: | - v4-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork + v4-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork - name: 'Set GIT_TAG' if: startsWith(github.ref, 'refs/tags') run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: 'Publish Binary' run: | pip install packaging - GIT_COMMIT=${{ github.sha }} GIT_TAG=$GIT_TAG RUNNER_OS=${{ runner.os }} ./scripts/ci/build.sh + GIT_COMMIT=${{ github.sha }} GIT_TAG=$GIT_TAG RUNNER_OS=${{ matrix.os }} ./scripts/ci/build.sh - name: Upload artifacts if: always() uses: actions/upload-artifact@v4 @@ -109,6 +110,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node: @@ -139,10 +141,10 @@ jobs: uses: actions/cache@v4 with: path: ~/Library/Caches/electron - key: v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} + key: v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} restore-keys: | - v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} - v1-${{ runner.os }}-electron-cache- + v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} + v1-${{ matrix.os }}-electron-cache- - name: Restore libcurl deps cache uses: actions/cache@v4 id: libcurl-deps-cache @@ -150,9 +152,9 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork + key: v4-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork restore-keys: | - v4-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork + v4-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork - name: 'Set GIT_TAG' if: startsWith(github.ref, 'refs/tags') run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV diff --git a/.github/workflows/build-lint-test.yaml b/.github/workflows/build-lint-test.yaml index bb5759afb..0e024e3f6 100644 --- a/.github/workflows/build-lint-test.yaml +++ b/.github/workflows/build-lint-test.yaml @@ -1,7 +1,6 @@ name: build-lint-test -on: - pull_request: +on: [push, workflow_dispatch] env: LIBCURL_RELEASE: 'LATEST' @@ -17,7 +16,7 @@ jobs: permissions: packages: write contents: write # publish sbom to GH releases/tag assets - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -42,6 +41,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node-libcurl-cpp-std: @@ -50,7 +50,7 @@ jobs: - 20.18.0 include: # Lint - - os: ubuntu-latest + - os: ubuntu-22.04 node: 20.18.0 node-libcurl-cpp-std: c++20 libcurl-release: 7.86.0 @@ -94,15 +94,16 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }} + key: v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }} restore-keys: | - v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }} + v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }} - name: Build node-libcurl ${{ matrix.node }} if: steps.run_result.outputs.run_result != 'success' run: | RUN_TESTS=false \ RUN_PREGYP_CLEAN=false \ PUBLISH_BINARY=false \ + TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm64' || 'x64' }} \ ./scripts/ci/build.sh - name: 'Run lint' if: matrix.run-lint-and-tsc && steps.run_result.outputs.run_result != 'success' @@ -140,6 +141,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node: @@ -164,11 +166,12 @@ jobs: ${{ github.run_id }}-${{ github.job }}- - id: run_result run: cat run_result 2>/dev/null || echo 'default' - - name: Set up Homebrew - id: set-up-homebrew - uses: Homebrew/actions/setup-homebrew@master - - name: Install Needed packages + - if: runner.os == 'macOS' + name: Install Needed packages on macOS run: brew install coreutils wget automake libtool cmake gnu-sed m4 autoconf groff + - if: runner.os == 'Linux' + name: Install Needed packages on Linux + run: sudo apt-get install -y cmake - name: Checkout uses: actions/checkout@v4 - name: Setup Node @@ -180,11 +183,13 @@ jobs: - name: Restore Electron Cache uses: actions/cache@v4 with: - path: ~/Library/Caches/electron - key: v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} + path: | + ~/Library/Caches/electron + ~/.cache/electron + key: v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} restore-keys: | - v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} - v1-${{ runner.os }}-electron-cache- + v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} + v1-${{ matrix.os }}-electron-cache- - name: Restore libcurl deps cache uses: actions/cache@v4 id: libcurl-deps-cache @@ -192,9 +197,9 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} + key: v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} restore-keys: | - v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} + v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} - name: 'Set GIT_TAG' if: startsWith(github.ref, 'refs/tags') run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV @@ -204,6 +209,8 @@ jobs: RUN_TESTS=true \ RUN_PREGYP_CLEAN=false \ PUBLISH_BINARY=false \ + PUBLISH_BINARY=false \ + TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm64' || 'x64' }} \ ./scripts/ci/build.sh - name: Upload artifacts if: always() && steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/new-version.yml b/.github/workflows/new-version.yml index 9f994590f..c13602bd5 100644 --- a/.github/workflows/new-version.yml +++ b/.github/workflows/new-version.yml @@ -14,7 +14,7 @@ env: jobs: release: permissions: write-all - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 0f0fecd9d..d945003bb 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -7,7 +7,7 @@ on: jobs: publish: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: contents: write id-token: write # For using token to sign images diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index 9aaaa8dbd..071ea3427 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -13,7 +13,7 @@ on: jobs: semgrep: name: Semgrep SAST - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: # required for all workflows security-events: write diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index e997adf01..d23b62753 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -60,6 +60,12 @@ LOGS_FOLDER=${BUILD_LOGS_FOLDER:-./logs} mkdir -p $LOGS_FOLDER +# ia32, x64, armv7, etc +target_arch=${TARGET_ARCH:-"x64"} +# the build system of some dependencies (e.g. zstd and openldap) seems to read +# and use this variable, so unset it here to avoid making them fail +unset TARGET_ARCH + # The following two, libunistring and libidn2, are only necessary if building libcurl >= 7.53 # However we are going to build then anyway, they are not that slow to build. @@ -243,8 +249,8 @@ fi LIBCURL_DEST_FOLDER=$PREFIX_DIR/deps/libcurl echo "Building libcurl v$LIBCURL_RELEASE - Latest is v$LATEST_LIBCURL_RELEASE" ./scripts/ci/build-libcurl.sh $LIBCURL_RELEASE $LIBCURL_DEST_FOLDER || (echo "libcurl failed build log:" && cat $LIBCURL_DEST_FOLDER/source/$LIBCURL_RELEASE/config.log && exit 1) -echo "libcurl successful build log:" -cat $LIBCURL_DEST_FOLDER/source/$LIBCURL_RELEASE/config.log +# echo "libcurl successful build log:" +# cat $LIBCURL_DEST_FOLDER/source/$LIBCURL_RELEASE/config.log export LIBCURL_BUILD_FOLDER=$LIBCURL_DEST_FOLDER/build/$LIBCURL_RELEASE ls -al $LIBCURL_BUILD_FOLDER/lib @@ -307,8 +313,6 @@ else fi target=`echo $target | sed 's/^v//'` -# ia32, x64, armv7, etc -target_arch=${TARGET_ARCH:-"x64"} NODE_LIBCURL_CPP_STD=${NODE_LIBCURL_CPP_STD:-$(node $curr_dirname/../cpp-std.js)} diff --git a/test/curl/easy.spec.ts b/test/curl/easy.spec.ts index 6667a1d3f..a2665444a 100644 --- a/test/curl/easy.spec.ts +++ b/test/curl/easy.spec.ts @@ -8,7 +8,7 @@ import 'should' import { Curl, CurlCode, Easy } from '../../lib' -const url = 'http://example.com/' +const url = 'http://httpbin.org/put' // This is the only test that does not uses a express server // It makes a request to a live server, which can cause issues if there are network problems @@ -66,7 +66,6 @@ describe('easy', () => { 'Return value from the HEADER callback must be an integer.', ) }) - it('READFUNCTION - should rethrow error', () => { curl.setOpt('UPLOAD', true) // @ts-ignore