diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e27feac027..eb48e60d61 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -143,9 +143,42 @@ jobs: - run: make lint - run: git diff --exit-code + build-rust-amd64: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: ./rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.5 + - name: Install dependencies + run: sudo apt-get install -y protobuf-compiler + - name: Build binary + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu + - name: Rename binary + run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow ./numaflow-rs-linux-amd64 + - name: List files + run: pwd && ls -al && file ./numaflow-rs-linux-amd64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v4 + with: + name: numaflow-rs-linux-amd64 + path: rust/numaflow-rs-linux-amd64 + if-no-files-found: error + e2e-tests: name: E2E Tests runs-on: ubuntu-latest + needs: [ build-rust-amd64 ] timeout-minutes: 20 strategy: fail-fast: false @@ -185,6 +218,10 @@ jobs: with: path: ui/node_modules key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} + - name: Download Rust amd64 binaries + uses: actions/download-artifact@v4 + with: + name: numaflow-rs-linux-amd64 - name: Install k3d run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash & - name: Create a cluster diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 0b1a6353d1..3a671b2275 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -15,7 +15,7 @@ defaults: shell: bash jobs: - build-binaries: + build-go-binaries: runs-on: ubuntu-20.04 if: github.repository == 'numaproj/numaflow' name: Build binaries @@ -40,9 +40,70 @@ jobs: name: binaries path: dist + build-rust-amd64: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: ./rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.5 + - name: Install dependencies + run: sudo apt-get install -y protobuf-compiler + - name: Build binary + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu + - name: Rename binary + run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v3 + with: + name: numaflow-rs-linux-amd64 + path: rust/numaflow-rs-linux-amd64 + + build-rust-arm64: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: ./rust + steps: + - uses: actions/checkout@v4 + - name: Update Rust Toolchain Target + run: | + echo "targets = ['aarch64-unknown-linux-gnu']" >> rust-toolchain.toml + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.5 + - name: Install dependenices + run: sudo apt-get install -y gcc-aarch64-linux-gnu protobuf-compiler + - name: Build binary + run: RUSTFLAGS='-C target-feature=+crt-static -C linker=aarch64-linux-gnu-gcc' cargo build --release --target aarch64-unknown-linux-gnu + - name: Rename binary + run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v3 + with: + name: numaflow-rs-linux-arm64 + path: rust/numaflow-rs-linux-arm64 + build-push-linux-multi: name: Build & push linux/amd64 and linux/arm64 - needs: [ build-binaries ] + needs: [ build-go-binaries, build-rust-amd64, build-rust-arm64] runs-on: ubuntu-20.04 if: github.repository == 'numaproj/numaflow' strategy: @@ -63,12 +124,24 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Download binaries + - name: Download Go binaries uses: actions/download-artifact@v3 with: name: binaries path: dist/ + - name: Download Rust amd64 binaries + uses: actions/download-artifact@v3 + with: + name: numaflow-rs-linux-amd64 + path: dist/numaflow-rs-linux-amd64 + + - name: Download Rust arm64 binaries + uses: actions/download-artifact@v3 + with: + name: numaflow-rs-linux-arm64 + path: dist/numaflow-rs-linux-arm64 + - name: Registry Login uses: docker/login-action@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5b597014d..5c17591fe8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ defaults: shell: bash jobs: - build-binaries: + build-go-binaries: runs-on: ubuntu-20.04 if: github.repository == 'numaproj/numaflow' name: Build binaries @@ -38,9 +38,58 @@ jobs: name: binaries path: dist + build-rust-amd64: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: ./rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Install dependencies + run: sudo apt-get install -y protobuf-compiler + - name: Build binary + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu + - name: Rename binary + run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v3 + with: + name: numaflow-rs-linux-amd64 + path: rust/numaflow-rs-linux-amd64 + + build-rust-arm64: + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: ./rust + steps: + - uses: actions/checkout@v4 + - name: Update Rust Toolchain Target + run: | + echo "targets = ['aarch64-unknown-linux-gnu']" >> rust-toolchain.toml + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Install dependenices + run: sudo apt-get install -y gcc-aarch64-linux-gnu protobuf-compiler + - name: Build binary + run: RUSTFLAGS='-C target-feature=+crt-static -C linker=aarch64-linux-gnu-gcc' cargo build --release --target aarch64-unknown-linux-gnu + - name: Rename binary + run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v3 + with: + name: numaflow-rs-linux-arm64 + path: rust/numaflow-rs-linux-arm64 + build-push-linux-multi: name: Build & push linux/amd64 and linux/arm64 - needs: [ build-binaries ] + needs: [ build-go-binaries, build-rust-amd64, build-rust-arm64] runs-on: ubuntu-20.04 if: github.repository == 'numaproj/numaflow' strategy: @@ -61,12 +110,24 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Download binaries + - name: Download Go binaries uses: actions/download-artifact@v3 with: name: binaries path: dist/ + - name: Download Rust amd64 binaries + uses: actions/download-artifact@v3 + with: + name: numaflow-rs-linux-amd64 + path: dist/numaflow-rs-linux-amd64 + + - name: Download Rust arm64 binaries + uses: actions/download-artifact@v3 + with: + name: numaflow-rs-linux-arm64 + path: dist/numaflow-rs-linux-arm64 + - name: Registry Login uses: docker/login-action@v2 with: diff --git a/Dockerfile b/Dockerfile index c5ac53cbb1..efee5a8fd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,41 +8,30 @@ ARG ARCH RUN apk update && apk upgrade && \ apk add ca-certificates && \ apk --no-cache add tzdata -ARG ARCH COPY dist/numaflow-linux-${ARCH} /bin/numaflow +COPY dist/numaflow-rs-linux-${ARCH} /bin/numaflow-rs RUN chmod +x /bin/numaflow +RUN chmod +x /bin/numaflow-rs #################################################################################################### -# extension base +# Rust binary #################################################################################################### -FROM rust:1.80-bookworm AS extension-base +FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef ARG TARGETPLATFORM - -RUN apt-get update && apt-get install protobuf-compiler -y - -RUN cargo new numaflow -# Create a new empty shell project WORKDIR /numaflow +RUN apt-get update && apt-get install -y protobuf-compiler -RUN cargo new servesink -COPY ./rust/servesink/Cargo.toml ./servesink/ - -RUN cargo new backoff -COPY ./rust/backoff/Cargo.toml ./backoff/ - -RUN cargo new numaflow-models -COPY ./rust/numaflow-models/Cargo.toml ./numaflow-models/ -RUN cargo new monovertex -COPY ./rust/monovertex/Cargo.toml ./monovertex/ +FROM chef AS planner +COPY ./rust/ . +RUN cargo chef prepare --recipe-path recipe.json -RUN cargo new serving -COPY ./rust/serving/Cargo.toml ./serving/Cargo.toml - -# Copy all Cargo.toml and Cargo.lock files for caching dependencies -COPY ./rust/Cargo.toml ./rust/Cargo.lock ./ +FROM chef AS rust-builder +ARG TARGETPLATFORM +ARG ARCH +COPY --from=planner /numaflow/recipe.json recipe.json # Build to cache dependencies RUN --mount=type=cache,target=/usr/local/cargo/registry \ @@ -52,18 +41,10 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ "linux/arm64") TARGET="aarch64-unknown-linux-gnu" ;; \ *) echo "Unsupported platform: ${TARGETPLATFORM}" && exit 1 ;; \ esac && \ - mkdir -p src/bin && echo "fn main() {}" > src/bin/main.rs && \ - RUSTFLAGS='-C target-feature=+crt-static' cargo build --workspace --all --release --target ${TARGET} + RUSTFLAGS='-C target-feature=+crt-static' cargo chef cook --workspace --release --target ${TARGET} --recipe-path recipe.json # Copy the actual source code files of the main project and the subprojects -COPY ./rust/src ./src -COPY ./rust/servesink/src ./servesink/src -COPY ./rust/backoff/src ./backoff/src -COPY ./rust/numaflow-models/src ./numaflow-models/src -COPY ./rust/serving/src ./serving/src -COPY ./rust/monovertex/src ./monovertex/src -COPY ./rust/monovertex/build.rs ./monovertex/build.rs -COPY ./rust/monovertex/proto ./monovertex/proto +COPY ./rust/ . # Build the real binaries RUN --mount=type=cache,target=/usr/local/cargo/registry \ @@ -73,7 +54,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ "linux/arm64") TARGET="aarch64-unknown-linux-gnu" ;; \ *) echo "Unsupported platform: ${TARGETPLATFORM}" && exit 1 ;; \ esac && \ - touch src/bin/main.rs && \ RUSTFLAGS='-C target-feature=+crt-static' cargo build --workspace --all --release --target ${TARGET} && \ cp -pv target/${TARGET}/release/numaflow /root/numaflow @@ -82,13 +62,14 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ #################################################################################################### ARG BASE_IMAGE FROM ${BASE_IMAGE} AS numaflow +ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=base /bin/numaflow /bin/numaflow +COPY --from=base /bin/numaflow-rs /bin/numaflow-rs COPY ui/build /ui/build -COPY --from=extension-base /root/numaflow /bin/numaflow-rs COPY ./rust/serving/config config ENTRYPOINT [ "/bin/numaflow" ] diff --git a/Makefile b/Makefile index b616d50621..d31c9095e6 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,16 @@ SHELL:=/bin/bash PACKAGE=github.com/numaproj/numaflow CURRENT_DIR=$(shell pwd) + +HOST_ARCH=$(shell uname -m) +# Github actions instances are x86_64 +ifeq ($(HOST_ARCH),x86_64) + HOST_ARCH=amd64 +endif +ifeq ($(HOST_ARCH),aarch64) + HOST_ARCH=arm64 +endif + DIST_DIR=${CURRENT_DIR}/dist BINARY_NAME:=numaflow DOCKERFILE:=Dockerfile @@ -78,7 +88,7 @@ dist/$(BINARY_NAME): go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/$(BINARY_NAME) ./cmd dist/e2eapi: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/e2eapi ./test/e2e-api + CGO_ENABLED=0 GOOS=linux go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/e2eapi ./test/e2e-api dist/$(BINARY_NAME)-%: CGO_ENABLED=0 $(GOARGS) go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/$(BINARY_NAME)-$* ./cmd @@ -162,13 +172,27 @@ ui-test: ui-build ./hack/test-ui.sh .PHONY: image -image: clean ui-build dist/$(BINARY_NAME)-linux-amd64 - DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "ARCH=amd64" --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) -f $(DOCKERFILE) . +image: clean ui-build dist/$(BINARY_NAME)-linux-$(HOST_ARCH) +ifdef GITHUB_ACTIONS + # The binary will be built in a separate Github Actions job + cp -pv numaflow-rs-linux-amd64 dist/numaflow-rs-linux-amd64 +else + $(MAKE) build-rust-in-docker +endif + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) -f $(DOCKERFILE) . @if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION); fi ifdef IMAGE_IMPORT_CMD $(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) endif +.PHONY: build-rust-in-docker +build-rust-in-docker: + mkdir -p dist + -$(DOCKER) container ls --all --filter=ancestor='$(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)' --format "{{.ID}}" | xargs docker rm + -$(DOCKER) image rm $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target rust-builder -f $(DOCKERFILE) . + export CTR=$$(docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)) && $(DOCKER) cp $$CTR:/root/numaflow dist/numaflow-rs-linux-$(HOST_ARCH) && $(DOCKER) rm $$CTR && $(DOCKER) image rm $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) + image-multi: ui-build set-qemu dist/$(BINARY_NAME)-linux-arm64.gz dist/$(BINARY_NAME)-linux-amd64.gz $(DOCKER) buildx build --sbom=false --provenance=false --build-arg "BASE_IMAGE=$(RELEASE_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) --platform linux/amd64,linux/arm64 --file $(DOCKERFILE) ${PUSH_OPTION} . @@ -232,7 +256,7 @@ start: image .PHONY: e2eapi-image e2eapi-image: clean dist/e2eapi - DOCKER_BUILDKIT=1 $(DOCKER) build . --build-arg "ARCH=amd64" --target e2eapi --tag $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) --build-arg VERSION="$(VERSION)" + DOCKER_BUILDKIT=1 $(DOCKER) build . --target e2eapi --tag $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) --build-arg VERSION="$(VERSION)" @if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/e2eapi:$(VERSION); fi ifdef IMAGE_IMPORT_CMD $(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml new file mode 100644 index 0000000000..a5b1f06904 --- /dev/null +++ b/rust/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +profile = "default" +channel = "1.80"