Make nightly build pipeline faster

.dockerignore

@@ -0,0 +1 @@
+rust/target

.github/workflows/ci.yaml

@@ -143,9 +143,42 @@ jobs:
       - run: make lint
       - run: git diff --exit-code
 
+  build-rust-amd64:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: actions-rust-lang/[email protected]
+        with:
+          rustflags: ''
+      - name: Configure sccache
+        run: |
+          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+          echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+      - name: Run sccache-cache
+        uses: mozilla-actions/[email protected]
+      - name: Install dependencies
+        run: sudo apt-get install -y protobuf-compiler
+      - name: Build binary
+        run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu
+      - name: Rename binary
+        run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow ./numaflow-rs-linux-amd64
+      - name: List files
+        run: pwd && ls -al
+      - name: Upload numaflow binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: numaflow-rs-linux-amd64
+          path: rust/numaflow-rs-linux-amd64
+          if-no-files-found: error
+
   e2e-tests:
     name: E2E Tests
     runs-on: ubuntu-latest
+    needs: [ build-rust-amd64 ]
     timeout-minutes: 20
     strategy:
       fail-fast: false
@@ -185,6 +218,10 @@ jobs:
         with:
           path: ui/node_modules
           key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }}
+      - name: Download Rust amd64 binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: numaflow-rs-linux-amd64
       - name: Install k3d
         run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash &
       - name: Create a cluster

.github/workflows/nightly-build.yml

@@ -15,7 +15,7 @@ defaults:
     shell: bash
 
 jobs:
-  build-binaries:
+  build-go-binaries:
     runs-on: ubuntu-20.04
     if: github.repository == 'numaproj/numaflow'
     name: Build binaries
@@ -40,10 +40,71 @@ jobs:
           name: binaries
           path: dist
 
+  build-rust-amd64:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: actions-rust-lang/[email protected]
+        with:
+          rustflags: ''
+      - name: Configure sccache
+        run: |
+          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+          echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+      - name: Run sccache-cache
+        uses: mozilla-actions/[email protected]
+      - name: Install dependencies
+        run: sudo apt-get install -y protobuf-compiler
+      - name: Build binary
+        run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu
+      - name: Rename binary
+        run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64
+      - name: Upload numaflow binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: numaflow-rs-linux-amd64
+          path: rust/numaflow-rs-linux-amd64
+
+  build-rust-arm64:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Update Rust Toolchain Target
+        run: |
+          echo "targets = ['aarch64-unknown-linux-gnu']" >> rust-toolchain.toml
+      - name: Setup Rust toolchain
+        uses: actions-rust-lang/[email protected]
+        with:
+          rustflags: ''
+      - name: Configure sccache
+        run: |
+          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+          echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+      - name: Run sccache-cache
+        uses: mozilla-actions/[email protected]
+      - name: Install dependenices
+        run: sudo apt-get install -y gcc-aarch64-linux-gnu protobuf-compiler
+      - name: Build binary
+        run: export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER='aarch64-linux-gnu-gcc' && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu
+      - name: Rename binary
+        run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64
+      - name: Upload numaflow binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: numaflow-rs-linux-arm64
+          path: rust/numaflow-rs-linux-arm64
+
   build-push-linux-multi:
     name: Build & push linux/amd64 and linux/arm64
-    needs: [ build-binaries ]
-    runs-on: ubuntu-20.04
+    needs: [ build-go-binaries,  build-rust-amd64, build-rust-arm64]
+    runs-on: ubuntu-latest
     if: github.repository == 'numaproj/numaflow'
     strategy:
       matrix:
@@ -61,14 +122,26 @@ jobs:
           key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
-      - name: Download binaries
+      - name: Download Go binaries
         uses: actions/download-artifact@v3
         with:
           name: binaries
           path: dist/
 
+      - name: Download Rust amd64 binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: numaflow-rs-linux-amd64
+          path: dist/numaflow-rs-linux-amd64
+
+      - name: Download Rust arm64 binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: numaflow-rs-linux-arm64
+          path: dist/numaflow-rs-linux-arm64
+
       - name: Registry Login
         uses: docker/login-action@v2
         with:

Dockerfile

@@ -3,84 +3,71 @@ ARG ARCH=$TARGETARCH
 ####################################################################################################
 # base
 ####################################################################################################
-FROM debian:bullseye as base
+FROM debian:bullseye AS base
 ARG ARCH
 
 COPY dist/numaflow-linux-${ARCH} /bin/numaflow
 
 RUN chmod +x /bin/numaflow
 
 ####################################################################################################
-# extension base
+# Rust binary
 ####################################################################################################
-FROM rust:1.79-bookworm as extension-base
-
-RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
-
-RUN apt-get update
-RUN apt-get install protobuf-compiler -y
-
-RUN cargo new numaflow
-# Create a new empty shell project
+FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
+ARG TARGETPLATFORM
+RUN apt-get update && apt-get install -y protobuf-compiler clang curl
 WORKDIR /numaflow
 
-RUN cargo new servesink
-COPY ./rust/servesink/Cargo.toml ./servesink/
-
-RUN cargo new backoff
-COPY ./rust/backoff/Cargo.toml ./backoff/
-
-RUN cargo new numaflow-models
-COPY ./rust/numaflow-models/Cargo.toml ./numaflow-models/
-
-RUN cargo new monovertex
-COPY ./rust/monovertex/Cargo.toml ./monovertex/
-
-RUN cargo new serving
-COPY ./rust/serving/Cargo.toml ./serving/Cargo.toml
+FROM chef AS planner
+COPY ./rust/ .
+RUN cargo chef prepare --recipe-path recipe.json
 
-# Copy all Cargo.toml and Cargo.lock files for caching dependencies
-COPY ./rust/Cargo.toml ./rust/Cargo.lock ./
-
-# Build to cache dependencies
-RUN mkdir -p src/bin && echo "fn main() {}" > src/bin/main.rs && \
-    cargo build --workspace --all --release
-
-# Copy the actual source code files of the main project and the subprojects
-COPY ./rust/src ./src
-COPY ./rust/servesink/src ./servesink/src
-COPY ./rust/backoff/src ./backoff/src
-COPY ./rust/numaflow-models/src ./numaflow-models/src
-COPY ./rust/serving/src ./serving/src
-COPY ./rust/monovertex/src ./monovertex/src
-COPY ./rust/monovertex/build.rs ./monovertex/build.rs
-COPY ./rust/monovertex/proto ./monovertex/proto
-
-# Build the real binaries
-RUN touch src/bin/main.rs && \
-    cargo build --workspace --all --release
+FROM chef AS builder
+ARG TARGETPLATFORM
+ARG ARCH
+COPY --from=planner /numaflow/recipe.json recipe.json
+# Build dependencies - this is the caching Docker layer!
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    case ${TARGETPLATFORM} in \
+      "linux/amd64") TARGET="x86_64-unknown-linux-gnu" ;; \
+      "linux/arm64") TARGET="aarch64-unknown-linux-gnu" ;; \
+      *) echo "Unsupported platform: ${TARGETPLATFORM}" && exit 1 ;; \
+    esac && \
+    RUSTFLAGS='-C target-feature=+crt-static' cargo chef cook --release --target ${TARGET} --recipe-path recipe.json
+
+# Build application
+COPY ./rust/ .
+# RUN --mount=type=cache,target=/usr/local/cargo/registry RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    case ${TARGETPLATFORM} in \
+      "linux/amd64") TARGET="x86_64-unknown-linux-gnu" ;; \
+      "linux/arm64") TARGET="aarch64-unknown-linux-gnu" ;; \
+      *) echo "Unsupported platform: ${TARGETPLATFORM}" && exit 1 ;; \
+    esac && \
+    RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target ${TARGET} && \
+    cp -pv target/${TARGET}/release/numaflow /root/numaflow-rs-linux-${ARCH}
 
 ####################################################################################################
 # numaflow
 ####################################################################################################
 ARG BASE_IMAGE
-FROM debian:bookworm as numaflow
-
-# Install necessary libraries
-RUN apt-get update && apt-get install -y libssl3
+FROM gcr.io/distroless/static-debian12 AS numaflow
+ARG ARCH
 
-COPY --from=base /bin/numaflow /bin/numaflow
+COPY dist/numaflow-linux-${ARCH} /bin/numaflow
+COPY dist/numaflow-rs-linux-${ARCH} /bin/numaflow-rs
 COPY ui/build /ui/build
 
-COPY --from=extension-base /numaflow/target/release/numaflow /bin/numaflow-rs
 COPY ./rust/serving/config config
 
 ENTRYPOINT [ "/bin/numaflow" ]
 
 ####################################################################################################
 # testbase
 ####################################################################################################
-FROM alpine:3.17 as testbase
+FROM alpine:3.17 AS testbase
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata

Makefile

@@ -2,6 +2,7 @@ SHELL:=/bin/bash
 
 PACKAGE=github.com/numaproj/numaflow
 CURRENT_DIR=$(shell pwd)
+LOCAL_ARCH=$(shell arch)
 DIST_DIR=${CURRENT_DIR}/dist
 BINARY_NAME:=numaflow
 DOCKERFILE:=Dockerfile
@@ -68,6 +69,7 @@ dist/$(BINARY_NAME)-%.gz: dist/$(BINARY_NAME)-%
 	@[[ -e dist/$(BINARY_NAME)-$*.gz ]] || gzip -k dist/$(BINARY_NAME)-$*
 
 dist/$(BINARY_NAME): GOARGS = GOOS= GOARCH=
+dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH): GOARGS = GOOS=linux
 dist/$(BINARY_NAME)-linux-amd64: GOARGS = GOOS=linux GOARCH=amd64
 dist/$(BINARY_NAME)-linux-arm64: GOARGS = GOOS=linux GOARCH=arm64
 dist/$(BINARY_NAME)-linux-arm: GOARGS = GOOS=linux GOARCH=arm
@@ -78,7 +80,7 @@ dist/$(BINARY_NAME):
 	go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/$(BINARY_NAME) ./cmd
 
 dist/e2eapi:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/e2eapi ./test/e2e-api
+	CGO_ENABLED=0 GOOS=linux go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/e2eapi ./test/e2e-api
 
 dist/$(BINARY_NAME)-%:
 	CGO_ENABLED=0 $(GOARGS) go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/$(BINARY_NAME)-$* ./cmd
@@ -162,14 +164,38 @@ ui-test: ui-build
 	./hack/test-ui.sh
 
 .PHONY: image
-image: clean ui-build dist/$(BINARY_NAME)-linux-amd64
-	DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "ARCH=amd64" --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) -f $(DOCKERFILE) .
+image: clean ui-build dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH)
+ifdef GITHUB_ACTIONS
+	cp -pv numaflow-rs-linux-amd64 dist/numaflow-rs-linux-amd64
+	cp -pv dist/numaflow-linux-x86_64 dist/numaflow-linux-amd64
+else
+	DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . --load
+	$(eval RUST_BUILDER := $(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)))
+	docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-$(LOCAL_ARCH) dist/numaflow-rs-linux-$(LOCAL_ARCH)
+	docker rm $(RUST_BUILDER)
+endif
+	DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . --load
 	@if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION); fi
 ifdef IMAGE_IMPORT_CMD
 	$(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION)
 endif
 
+.PHONY: build-rust-in-docker
+build-rust-in-docker:
+	mkdir -p dist
+	DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-amd64 --platform linux/amd64 --target builder -f $(DOCKERFILE) .
+	$(eval RUST_BUILDER=$(shell docker create --platform linux/amd64 $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-amd64))
+	docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-amd64 dist/numaflow-rs-linux-amd64
+	docker rm $(RUST_BUILDER)
+	DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-arm64 --platform linux/arm64 --target builder -f $(DOCKERFILE) .
+	$(eval RUST_BUILDER=$(shell docker create --platform linux/arm64 $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-arm64))
+	docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-arm64 dist/numaflow-rs-linux-arm64
+	docker rm $(RUST_BUILDER)
+
 image-multi: ui-build set-qemu dist/$(BINARY_NAME)-linux-arm64.gz dist/$(BINARY_NAME)-linux-amd64.gz
+ifndef GITHUB_ACTIONS
+image-multi: build-rust-in-docker
+endif
 	$(DOCKER) buildx build --sbom=false --provenance=false --build-arg "BASE_IMAGE=$(RELEASE_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) --platform linux/amd64,linux/arm64 --file $(DOCKERFILE) ${PUSH_OPTION} .
 
 set-qemu:
@@ -232,7 +258,7 @@ start: image
 
 .PHONY: e2eapi-image
 e2eapi-image: clean dist/e2eapi
-	DOCKER_BUILDKIT=1 $(DOCKER) build . --build-arg "ARCH=amd64" --target e2eapi --tag $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) --build-arg VERSION="$(VERSION)"
+	DOCKER_BUILDKIT=1 $(DOCKER) build . --target e2eapi --tag $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) --build-arg VERSION="$(VERSION)" --load
 	@if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/e2eapi:$(VERSION); fi
 ifdef IMAGE_IMPORT_CMD
 	$(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/e2eapi:$(VERSION)