From d7521f00826d22b09a55a92d435444839487b3db Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 11:55:27 +0200 Subject: [PATCH 01/10] [ci] Use parity-large runners instead ubuntu-latest-16-cores --- .github/workflows/build-nodes.yml | 2 +- .github/workflows/rust.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-nodes.yml b/.github/workflows/build-nodes.yml index c48f2fc351..d1af8ade3e 100644 --- a/.github/workflows/build-nodes.yml +++ b/.github/workflows/build-nodes.yml @@ -10,7 +10,7 @@ on: jobs: tests: name: Build Substrate and Polkadot Binaries - runs-on: ubuntu-latest-16-cores + runs-on: parity-large steps: - name: checkout polkadot-sdk uses: actions/checkout@v4 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ae092449f9..06ec733633 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -268,7 +268,7 @@ jobs: tests: name: "Test (Native)" - runs-on: ubuntu-latest-16-cores + runs-on: parity-large needs: [clippy, wasm_clippy, check, wasm_check, docs] timeout-minutes: 30 steps: @@ -302,7 +302,7 @@ jobs: unstable_backend_tests: name: "Test chainhead backend" - runs-on: ubuntu-latest-16-cores + runs-on: parity-large needs: [clippy, wasm_clippy, check, wasm_check, docs] timeout-minutes: 30 steps: From 74cd6c5dc1e3c60fe1037e4f5850655719478710 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 12:21:08 +0200 Subject: [PATCH 02/10] isntall curl, move clippy, check, docs to parity-large --- .github/workflows/actions/use-nodes/action.yml | 4 ++++ .github/workflows/rust.yml | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions/use-nodes/action.yml b/.github/workflows/actions/use-nodes/action.yml index ab7c34264b..661bc53d7c 100644 --- a/.github/workflows/actions/use-nodes/action.yml +++ b/.github/workflows/actions/use-nodes/action.yml @@ -3,6 +3,10 @@ description: Downloads and configures the substrate and polkadot binaries built runs: using: composite steps: + - name: Install curl + shell: bash + run: sudo apt-get update && sudo apt-get install -y curl + - name: Download substrate-node binary id: download-substrate-binary uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 06ec733633..fb2de1e067 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -82,7 +82,7 @@ jobs: clippy: name: Cargo clippy - runs-on: ubuntu-latest + runs-on: parity-large needs: [fmt, machete] steps: - name: Checkout sources @@ -145,7 +145,7 @@ jobs: check: name: Cargo check - runs-on: ubuntu-latest + runs-on: parity-large needs: [fmt, machete] steps: - name: Checkout sources @@ -235,7 +235,7 @@ jobs: docs: name: Check documentation and run doc tests - runs-on: ubuntu-latest + runs-on: parity-large needs: [fmt, machete] steps: - name: Checkout sources From 52d49ad5e2e2e81a45cd41dd615472740b7c83fa Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 12:27:46 +0200 Subject: [PATCH 03/10] install gcc, make, clang --- .github/workflows/actions/use-nodes/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions/use-nodes/action.yml b/.github/workflows/actions/use-nodes/action.yml index 661bc53d7c..ac61936f0b 100644 --- a/.github/workflows/actions/use-nodes/action.yml +++ b/.github/workflows/actions/use-nodes/action.yml @@ -5,7 +5,7 @@ runs: steps: - name: Install curl shell: bash - run: sudo apt-get update && sudo apt-get install -y curl + run: sudo apt-get update && sudo apt-get install -y curl gcc make clang - name: Download substrate-node binary id: download-substrate-binary From fd2d9578d5877fc9a61594cf90fc99bbc0f1d7ad Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 12:30:44 +0200 Subject: [PATCH 04/10] isntall cmake --- .github/workflows/actions/use-nodes/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions/use-nodes/action.yml b/.github/workflows/actions/use-nodes/action.yml index ac61936f0b..6c6372bf78 100644 --- a/.github/workflows/actions/use-nodes/action.yml +++ b/.github/workflows/actions/use-nodes/action.yml @@ -5,7 +5,7 @@ runs: steps: - name: Install curl shell: bash - run: sudo apt-get update && sudo apt-get install -y curl gcc make clang + run: sudo apt-get update && sudo apt-get install -y curl gcc make clang cmake - name: Download substrate-node binary id: download-substrate-binary From 524387ce5ee7a3236217391478168c295cc07248 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 12:42:43 +0200 Subject: [PATCH 05/10] mv artifacts to /usr/local/bin --- .github/workflows/actions/use-nodes/action.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/actions/use-nodes/action.yml b/.github/workflows/actions/use-nodes/action.yml index 6c6372bf78..bd1872c22a 100644 --- a/.github/workflows/actions/use-nodes/action.yml +++ b/.github/workflows/actions/use-nodes/action.yml @@ -36,9 +36,8 @@ runs: chmod u+x ./polkadot-prepare-worker ./substrate-node --version ./polkadot --version - mkdir -p ~/.local/bin - mv ./substrate-node ~/.local/bin - mv ./polkadot ~/.local/bin - mv ./polkadot-execute-worker ~/.local/bin - mv ./polkadot-prepare-worker ~/.local/bin + mv ./substrate-node /usr/local/bin + mv ./polkadot /usr/local/bin + mv ./polkadot-execute-worker /usr/local/bin + mv ./polkadot-prepare-worker /usr/local/bin rm ./polkadot.tar.gz From c4a54ad608a778879ba088fb84de6e6a083e60a0 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 12:45:15 +0200 Subject: [PATCH 06/10] sudo mv --- .github/workflows/actions/use-nodes/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/actions/use-nodes/action.yml b/.github/workflows/actions/use-nodes/action.yml index bd1872c22a..52753bdb20 100644 --- a/.github/workflows/actions/use-nodes/action.yml +++ b/.github/workflows/actions/use-nodes/action.yml @@ -36,8 +36,8 @@ runs: chmod u+x ./polkadot-prepare-worker ./substrate-node --version ./polkadot --version - mv ./substrate-node /usr/local/bin - mv ./polkadot /usr/local/bin - mv ./polkadot-execute-worker /usr/local/bin - mv ./polkadot-prepare-worker /usr/local/bin + sudo mv ./substrate-node /usr/local/bin + sudo mv ./polkadot /usr/local/bin + sudo mv ./polkadot-execute-worker /usr/local/bin + sudo mv ./polkadot-prepare-worker /usr/local/bin rm ./polkadot.tar.gz From 34d2db7f12984cb53ab398dfbcffe992b9a9150b Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 13:24:46 +0200 Subject: [PATCH 07/10] check build-nodes --- .../workflows/actions/use-nodes/action.yml | 2 +- .github/workflows/build-nodes.yml | 58 ++++++++++++------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/.github/workflows/actions/use-nodes/action.yml b/.github/workflows/actions/use-nodes/action.yml index 52753bdb20..9d45b2649d 100644 --- a/.github/workflows/actions/use-nodes/action.yml +++ b/.github/workflows/actions/use-nodes/action.yml @@ -3,7 +3,7 @@ description: Downloads and configures the substrate and polkadot binaries built runs: using: composite steps: - - name: Install curl + - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y curl gcc make clang cmake diff --git a/.github/workflows/build-nodes.yml b/.github/workflows/build-nodes.yml index d1af8ade3e..1acdc6ed3e 100644 --- a/.github/workflows/build-nodes.yml +++ b/.github/workflows/build-nodes.yml @@ -6,6 +6,12 @@ on: # Run at 2am every day for nightly builds. schedule: - cron: "0 2 * * *" + #remove me, temporary to check self-hosted runners + pull_request: + # Run jobs for any external PR that wants + # to merge to master, too: + branches: + - master jobs: tests: @@ -18,13 +24,21 @@ jobs: repository: paritytech/polkadot-sdk - name: Install dependencies - run: sudo apt-get install -y protobuf-compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler curl gcc make clang cmake - - name: Install WASM toolchain - run: rustup target add wasm32-unknown-unknown + # - name: Install WASM toolchain + # run: rustup target add wasm32-unknown-unknown - - name: Install WASM toolchain - run: rustup component add rust-src + # - name: Install WASM toolchain + # run: rustup component add rust-src + + - name: Install Rust stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rust-src + target: wasm32-unknown-unknown - name: Rust Cache uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 @@ -46,26 +60,26 @@ jobs: cargo install cargo-strip cargo strip - - name: upload substrate binary - uses: actions/upload-artifact@v4 - with: - name: nightly-substrate-binary - path: target/release/substrate-node - retention-days: 2 - if-no-files-found: error + # - name: upload substrate binary + # uses: actions/upload-artifact@v4 + # with: + # name: nightly-substrate-binary + # path: target/release/substrate-node + # retention-days: 2 + # if-no-files-found: error # Note: Uncompressed polkadot binary is ~124MB -> too large for git (max 100MB) without git lfs. Compressed it is only ~45MB - name: compress polkadot binary run: | tar -zcvf target/release/polkadot.tar.gz target/release/polkadot - - name: upload polkadot binary - uses: actions/upload-artifact@v4 - with: - name: nightly-polkadot-binary - path: | - target/release/polkadot.tar.gz - target/release/polkadot-execute-worker - target/release/polkadot-prepare-worker - retention-days: 2 - if-no-files-found: error + # - name: upload polkadot binary + # uses: actions/upload-artifact@v4 + # with: + # name: nightly-polkadot-binary + # path: | + # target/release/polkadot.tar.gz + # target/release/polkadot-execute-worker + # target/release/polkadot-prepare-worker + # retention-days: 2 + # if-no-files-found: error From 50f77219f1e76f845877746e05aa45a21575abff Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 13:56:08 +0200 Subject: [PATCH 08/10] uncomment upload --- .github/workflows/build-nodes.yml | 46 ++++++++++++------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-nodes.yml b/.github/workflows/build-nodes.yml index 1acdc6ed3e..6ec594c026 100644 --- a/.github/workflows/build-nodes.yml +++ b/.github/workflows/build-nodes.yml @@ -6,12 +6,6 @@ on: # Run at 2am every day for nightly builds. schedule: - cron: "0 2 * * *" - #remove me, temporary to check self-hosted runners - pull_request: - # Run jobs for any external PR that wants - # to merge to master, too: - branches: - - master jobs: tests: @@ -26,12 +20,6 @@ jobs: - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y protobuf-compiler curl gcc make clang cmake - # - name: Install WASM toolchain - # run: rustup target add wasm32-unknown-unknown - - # - name: Install WASM toolchain - # run: rustup component add rust-src - - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1 with: @@ -60,26 +48,26 @@ jobs: cargo install cargo-strip cargo strip - # - name: upload substrate binary - # uses: actions/upload-artifact@v4 - # with: - # name: nightly-substrate-binary - # path: target/release/substrate-node - # retention-days: 2 - # if-no-files-found: error + - name: upload substrate binary + uses: actions/upload-artifact@v4 + with: + name: nightly-substrate-binary + path: target/release/substrate-node + retention-days: 2 + if-no-files-found: error # Note: Uncompressed polkadot binary is ~124MB -> too large for git (max 100MB) without git lfs. Compressed it is only ~45MB - name: compress polkadot binary run: | tar -zcvf target/release/polkadot.tar.gz target/release/polkadot - # - name: upload polkadot binary - # uses: actions/upload-artifact@v4 - # with: - # name: nightly-polkadot-binary - # path: | - # target/release/polkadot.tar.gz - # target/release/polkadot-execute-worker - # target/release/polkadot-prepare-worker - # retention-days: 2 - # if-no-files-found: error + - name: upload polkadot binary + uses: actions/upload-artifact@v4 + with: + name: nightly-polkadot-binary + path: | + target/release/polkadot.tar.gz + target/release/polkadot-execute-worker + target/release/polkadot-prepare-worker + retention-days: 2 + if-no-files-found: error From 35371c4c86daefd54921dd12d2117bb25847ab32 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 14:00:27 +0200 Subject: [PATCH 09/10] temporary run build-nodes on pull request to make CI green --- .github/workflows/build-nodes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-nodes.yml b/.github/workflows/build-nodes.yml index 6ec594c026..bac5f3fd25 100644 --- a/.github/workflows/build-nodes.yml +++ b/.github/workflows/build-nodes.yml @@ -6,6 +6,7 @@ on: # Run at 2am every day for nightly builds. schedule: - cron: "0 2 * * *" + pull_request: jobs: tests: From fc24f60ec82d297814e8f646132c3e797d4805af Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 9 Oct 2024 14:19:31 +0200 Subject: [PATCH 10/10] rm on: pr from build-nodes --- .github/workflows/build-nodes.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-nodes.yml b/.github/workflows/build-nodes.yml index bac5f3fd25..6ec594c026 100644 --- a/.github/workflows/build-nodes.yml +++ b/.github/workflows/build-nodes.yml @@ -6,7 +6,6 @@ on: # Run at 2am every day for nightly builds. schedule: - cron: "0 2 * * *" - pull_request: jobs: tests: