chore(ci): Speedup workflows (#8448)

cube-js · Jul 11, 2024 · 20c8e89 · 20c8e89
1 parent c2f39ff
commit 20c8e89
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 98 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   npm:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     permissions:
       contents: write
@@ -39,8 +39,6 @@ jobs:
             ${{ runner.os }}-yarn-
       - name: Set Yarn version
         run: yarn policies set-version v1.22.19
-      - name: Copy yarn.lock file
-        run: cp yarn.lock packages/cubejs-docker
       - name: Yarn install
         uses: nick-fields/retry@v3
         env:
@@ -106,19 +104,9 @@ jobs:
         run: npm install -g yarn
       - name: Set Yarn version
         run: yarn policies set-version v1.22.19
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
-        shell: bash
-      - name: Restore yarn cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-      - name: Copy yarn.lock file
-        run: cp yarn.lock packages/cubejs-docker
+        # We don't need to install all yarn deps to build native
+      - name: Install cargo-cp-artifact
+        run: npm install -g [email protected]
       - name: Yarn install
         uses: nick-fields/retry@v3
         env:
@@ -219,21 +207,11 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
-        shell: bash
-      - name: Restore yarn cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
       - name: Set Yarn version
         run: yarn policies set-version v1.22.19
-      - name: Copy yarn.lock file
-        run: cp yarn.lock packages/cubejs-docker
+        # We don't need to install all yarn deps to build native
+      - name: Install cargo-cp-artifact
+        run: npm install -g [email protected]
       - name: Yarn install
         uses: nick-fields/retry@v3
         env:
@@ -315,21 +293,11 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
-        shell: bash
-      - name: Restore yarn cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
       - name: Set Yarn version
         run: yarn policies set-version v1.22.19
-      - name: Copy yarn.lock file
-        run: cp yarn.lock packages/cubejs-docker
+        # We don't need to install all yarn deps to build native
+      - name: Install cargo-cp-artifact
+        run: npm install -g [email protected]
       - name: Yarn install
         uses: nick-fields/retry@v3
         env:
@@ -455,7 +423,7 @@ jobs:
   docker-debian-jdk:
     needs: [npm, cubestore_linux, native_linux]
     name: Debian with jdk docker image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     permissions:
       contents: write
@@ -536,7 +504,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-20.04
+          - os: ubuntu-24.04
             target: x86_64-unknown-linux-gnu
             platforms: linux/amd64
             build-args: WITH_AVX2=1
@@ -549,7 +517,7 @@ jobs:
             postfix: "-arm64v8"
             tag: "arm64v8"
           # Non AVX build
-          - os: ubuntu-20.04
+          - os: ubuntu-24.04
             target: x86_64-unknown-linux-gnu
             platforms: linux/amd64
             build-args: WITH_AVX2=0
@@ -766,7 +734,7 @@ jobs:
         run: vcpkg integrate install; vcpkg install openssl:x64-windows
       - name: Instal LLVM for Windows
         if: ${{ startsWith(matrix.os, 'windows') }}
-        run: choco install -y --force llvm --version 18.1.2
+        run: choco install -y --force llvm --version 18.1.6
       - name: Set Env Variables for Windows
         uses: allenevans/[email protected]
         if: ${{ startsWith(matrix.os, 'windows') }}

diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml
@@ -142,16 +142,7 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: v0.9.1
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-${{ matrix.target }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.target }}-buildx-
+        uses: docker/setup-buildx-action@v3
       - name: Push to Docker Hub
         uses: docker/build-push-action@v3
         with:
@@ -161,8 +152,8 @@ jobs:
           build-args: ${{ matrix.build-args }}
           push: true
           tags: ${{ steps.prep.outputs.tags }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           labels: |
             org.opencontainers.image.title=${{ fromJson(steps.repo.outputs.result).name }}
             org.opencontainers.image.description=${{ fromJson(steps.repo.outputs.result).description }}
@@ -223,7 +214,7 @@ jobs:
         run: vcpkg integrate install; vcpkg install openssl:x64-windows
       - name: Instal LLVM for Windows
         if: ${{ startsWith(matrix.os, 'windows') }}
-        run: choco install -y --force llvm --version 18.1.2
+        run: choco install -y --force llvm --version 18.1.6
       - name: Set Env Variables for Windows
         uses: allenevans/[email protected]
         if: ${{ startsWith(matrix.os, 'windows') }}

diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml
@@ -90,16 +90,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: v0.9.1
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-${{ matrix.target }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.target }}-buildx-
+        uses: docker/setup-buildx-action@v3
       - name: Build only
         uses: docker/build-push-action@v3
         with:
@@ -108,8 +99,8 @@ jobs:
           platforms: ${{ matrix.platforms }}
           build-args: ${{ matrix.build-args }}
           push: false
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
   cubestore:
     runs-on: ${{ matrix.os }}
@@ -154,7 +145,7 @@ jobs:
         run: vcpkg integrate install; vcpkg install openssl:x64-windows
       - name: Instal LLVM for Windows
         if: ${{ startsWith(matrix.os, 'windows') }}
-        run: choco install -y --force llvm --version 18.1.2
+        run: choco install -y --force llvm --version 18.1.6
       - name: Set Env Variables for Windows
         uses: allenevans/[email protected]
         if: ${{ startsWith(matrix.os, 'windows') }}

diff --git a/rust/cubestore/Dockerfile b/rust/cubestore/Dockerfile
@@ -1,28 +1,4 @@
-FROM rust:1-bookworm AS builder
-
-RUN rustup update && \
-    rustup default nightly-2024-01-10 && \
-    rustup component add --toolchain nightly-2024-01-10 rustfmt;
-
-RUN apt update \
-    && apt upgrade -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common pkg-config wget gnupg git apt-transport-https ca-certificates \
-    && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
-    # https://github.com/llvm/llvm-project/issues/62475 \
-    # add it twice to workaround:
-    && add-apt-repository --yes "deb https://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-18 main" \
-    && add-apt-repository --yes "deb https://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-18 main" \
-    && sleep 5 \
-    && apt update \
-    && apt install -y git llvm-18 clang-18 libclang-18-dev clang-18 cmake \
-    && rm -rf /var/lib/apt/lists/*;
-
-RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
-RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100
-RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-18 100
-RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-18 100
-
-WORKDIR /usr/src
+FROM cubejs/rust-builder:bookworm-llvm-18 AS builder
 
 WORKDIR /build/cubestore
 COPY Cargo.toml .

diff --git a/rust/cubestore/builder.Dockerfile b/rust/cubestore/builder.Dockerfile
@@ -0,0 +1,29 @@
+ARG OS_NAME=bookworm-slim
+
+FROM rust:$OS_NAME
+
+ARG LLVM_VERSION=18
+
+RUN rustup update && \
+    rustup default nightly-2024-01-29 && \
+    rustup component add --toolchain nightly-2024-01-29 rustfmt clippy;
+
+RUN apt update \
+    && apt upgrade -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common libssl-dev pkg-config wget gnupg git apt-transport-https ca-certificates \
+    && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
+    # https://github.com/llvm/llvm-project/issues/62475 \
+    # add it twice to workaround:
+    && add-apt-repository --yes "deb https://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-$LLVM_VERSION main" \
+    && add-apt-repository --yes "deb https://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-$LLVM_VERSION main" \
+    && sleep 5 \
+    && apt update \
+    && apt install -y git llvm-$LLVM_VERSION clang-$LLVM_VERSION libclang-$LLVM_VERSION-dev clang-$LLVM_VERSION cmake \
+    && rm -rf /var/lib/apt/lists/*;
+
+RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION 100
+RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$LLVM_VERSION 100
+RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-$LLVM_VERSION 100
+RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-$LLVM_VERSION 100
+
+WORKDIR /usr/src
diff --git a/rust/cubestore/docker-bake.hcl b/rust/cubestore/docker-bake.hcl
@@ -0,0 +1,10 @@
+target "rust-builder-bookworm" {
+  context = "."
+  dockerfile = "builder.Dockerfile"
+  args = {
+    OS_NAME = "1-slim-bookworm"
+    LLVM_VERSION = "18"
+  }
+  tags = ["cubejs/rust-builder:bookworm-llvm-18"]
+  platforms = ["linux/amd64", "linux/arm64"]
+}
diff --git a/rust/cubestore/rust-toolchain.toml b/rust/cubestore/rust-toolchain.toml
@@ -1,4 +1,4 @@
 [toolchain]
 channel = "nightly-2024-01-29"
-components = ["rustfmt", "rustc-dev", "clippy"]
+components = ["rustfmt", "clippy"]
 profile = "minimal"