From 7884eb2537be676a00ca592a8e63752e6fea81a3 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 11:41:55 +0530 Subject: [PATCH 01/20] Make nightly build pipeline faster Signed-off-by: Sreekanth --- .github/workflows/nightly-build.yml | 73 ++++++++++++++++++++++- Dockerfile | 91 +++++++++++++---------------- Makefile | 25 +++++++- 3 files changed, 132 insertions(+), 57 deletions(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 0b1a6353d1..3c0489f8a7 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -15,7 +15,7 @@ defaults: shell: bash jobs: - build-binaries: + build-go-binaries: runs-on: ubuntu-20.04 if: github.repository == 'numaproj/numaflow' name: Build binaries @@ -40,9 +40,64 @@ jobs: name: binaries path: dist + build-rust-amd64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.5 + - name: Install dependencies + run: sudo apt-get install -y protobuf-compiler + - name: Build binary + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu + - name: Rename binary + run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v4 + with: + name: numaflow-rs-linux-amd64 + path: numaflow-rs-linux-amd64 + + build-rust-arm64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Update Rust Toolchain Target + run: | + echo "targets = ['aarch64-unknown-linux-gnu']" >> rust-toolchain.toml + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.5 + - name: Install dependenices + run: sudo apt-get install -y gcc-aarch64-linux-gnu protobuf-compiler + - name: Build binary + run: export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER='aarch64-linux-gnu-gcc' && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu + - name: Rename binary + run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v4 + with: + name: numaflow-rs-linux-arm64 + path: numaflow-rs-linux-arm64 + build-push-linux-multi: name: Build & push linux/amd64 and linux/arm64 - needs: [ build-binaries ] + needs: [ build-go-binaries, build-rust-amd64, build-rust-arm64] runs-on: ubuntu-20.04 if: github.repository == 'numaproj/numaflow' strategy: @@ -63,12 +118,24 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Download binaries + - name: Download Go binaries uses: actions/download-artifact@v3 with: name: binaries path: dist/ + - name: Download Rust amd64 binaries + uses: actions/download-artifact@v3 + with: + name: numaflow-rs-linux-amd64 + path: dist/numaflow-rs-linux-amd64 + + - name: Download Rust arm64 binaries + uses: actions/download-artifact@v3 + with: + name: numaflow-rs-linux-arm64 + path: dist/numaflow-rs-linux-arm64 + - name: Registry Login uses: docker/login-action@v2 with: diff --git a/Dockerfile b/Dockerfile index 7796a35b81..fce05135f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG ARCH=$TARGETARCH #################################################################################################### # base #################################################################################################### -FROM debian:bullseye as base +FROM debian:bullseye AS base ARG ARCH COPY dist/numaflow-linux-${ARCH} /bin/numaflow @@ -11,68 +11,55 @@ COPY dist/numaflow-linux-${ARCH} /bin/numaflow RUN chmod +x /bin/numaflow #################################################################################################### -# extension base +# Rust binary #################################################################################################### -FROM rust:1.79-bookworm as extension-base - -RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash - -RUN apt-get update -RUN apt-get install protobuf-compiler -y - -RUN cargo new numaflow -# Create a new empty shell project +FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef +ARG TARGETPLATFORM +RUN apt-get update && apt-get install -y protobuf-compiler clang curl WORKDIR /numaflow -RUN cargo new servesink -COPY ./rust/servesink/Cargo.toml ./servesink/ - -RUN cargo new backoff -COPY ./rust/backoff/Cargo.toml ./backoff/ - -RUN cargo new numaflow-models -COPY ./rust/numaflow-models/Cargo.toml ./numaflow-models/ - -RUN cargo new monovertex -COPY ./rust/monovertex/Cargo.toml ./monovertex/ - -RUN cargo new serving -COPY ./rust/serving/Cargo.toml ./serving/Cargo.toml +FROM chef AS planner +COPY ./rust/ . +RUN cargo chef prepare --recipe-path recipe.json -# Copy all Cargo.toml and Cargo.lock files for caching dependencies -COPY ./rust/Cargo.toml ./rust/Cargo.lock ./ - -# Build to cache dependencies -RUN mkdir -p src/bin && echo "fn main() {}" > src/bin/main.rs && \ - cargo build --workspace --all --release - -# Copy the actual source code files of the main project and the subprojects -COPY ./rust/src ./src -COPY ./rust/servesink/src ./servesink/src -COPY ./rust/backoff/src ./backoff/src -COPY ./rust/numaflow-models/src ./numaflow-models/src -COPY ./rust/serving/src ./serving/src -COPY ./rust/monovertex/src ./monovertex/src -COPY ./rust/monovertex/build.rs ./monovertex/build.rs -COPY ./rust/monovertex/proto ./monovertex/proto - -# Build the real binaries -RUN touch src/bin/main.rs && \ - cargo build --workspace --all --release +FROM chef AS builder +ARG TARGETPLATFORM +ARG ARCH +COPY --from=planner /numaflow/recipe.json recipe.json +# Build dependencies - this is the caching Docker layer! +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + case ${TARGETPLATFORM} in \ + "linux/amd64") TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64") TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported platform: ${TARGETPLATFORM}" && exit 1 ;; \ + esac && \ + RUSTFLAGS='-C target-feature=+crt-static' cargo chef cook --release --target ${TARGET} --recipe-path recipe.json + +# Build application +COPY ./rust/ . +# RUN --mount=type=cache,target=/usr/local/cargo/registry RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + case ${TARGETPLATFORM} in \ + "linux/amd64") TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64") TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported platform: ${TARGETPLATFORM}" && exit 1 ;; \ + esac && \ + RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target ${TARGET} && \ + cp -pv target/${TARGET}/release/numaflow /root/numaflow-rs-linux-${ARCH} #################################################################################################### # numaflow #################################################################################################### ARG BASE_IMAGE -FROM debian:bookworm as numaflow - -# Install necessary libraries -RUN apt-get update && apt-get install -y libssl3 +FROM debian:bookworm AS numaflow +ARG ARCH -COPY --from=base /bin/numaflow /bin/numaflow +COPY dist/numaflow-linux-${ARCH} /bin/numaflow +COPY dist/numaflow-rs-linux-${ARCH} /bin/numaflow-rs COPY ui/build /ui/build -COPY --from=extension-base /numaflow/target/release/numaflow /bin/numaflow-rs COPY ./rust/serving/config config ENTRYPOINT [ "/bin/numaflow" ] @@ -80,7 +67,7 @@ ENTRYPOINT [ "/bin/numaflow" ] #################################################################################################### # testbase #################################################################################################### -FROM alpine:3.17 as testbase +FROM alpine:3.17 AS testbase RUN apk update && apk upgrade && \ apk add ca-certificates && \ apk --no-cache add tzdata diff --git a/Makefile b/Makefile index 2c6e30d68c..26d45e974e 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ SHELL:=/bin/bash PACKAGE=github.com/numaproj/numaflow CURRENT_DIR=$(shell pwd) +LOCAL_ARCH=$(shell arch) DIST_DIR=${CURRENT_DIR}/dist BINARY_NAME:=numaflow DOCKERFILE:=Dockerfile @@ -68,6 +69,7 @@ dist/$(BINARY_NAME)-%.gz: dist/$(BINARY_NAME)-% @[[ -e dist/$(BINARY_NAME)-$*.gz ]] || gzip -k dist/$(BINARY_NAME)-$* dist/$(BINARY_NAME): GOARGS = GOOS= GOARCH= +dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH): GOARGS = GOOS=linux dist/$(BINARY_NAME)-linux-amd64: GOARGS = GOOS=linux GOARCH=amd64 dist/$(BINARY_NAME)-linux-arm64: GOARGS = GOOS=linux GOARCH=arm64 dist/$(BINARY_NAME)-linux-arm: GOARGS = GOOS=linux GOARCH=arm @@ -162,14 +164,33 @@ ui-test: ui-build ./hack/test-ui.sh .PHONY: image -image: clean ui-build dist/$(BINARY_NAME)-linux-amd64 - DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "ARCH=amd64" --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) -f $(DOCKERFILE) . +image: clean ui-build dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH) + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . + $(eval RUST_BUILDER=$(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION))) + docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-$(LOCAL_ARCH) dist/numaflow-rs-linux-$(LOCAL_ARCH) + docker rm $(RUST_BUILDER) + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . @if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION); fi ifdef IMAGE_IMPORT_CMD $(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) endif +.PHONY: build-rust-in-docker +build-rust-in-docker: + mkdir -p dist + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-amd64 --platform linux/amd64 --target builder -f $(DOCKERFILE) . + $(eval RUST_BUILDER=$(shell docker create --platform linux/amd64 $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-amd64)) + docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-amd64 dist/numaflow-rs-linux-amd64 + docker rm $(RUST_BUILDER) + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-arm64 --platform linux/arm64 --target builder -f $(DOCKERFILE) . + $(eval RUST_BUILDER=$(shell docker create --platform linux/arm64 $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)-arm64)) + docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-arm64 dist/numaflow-rs-linux-arm64 + docker rm $(RUST_BUILDER) + image-multi: ui-build set-qemu dist/$(BINARY_NAME)-linux-arm64.gz dist/$(BINARY_NAME)-linux-amd64.gz +ifndef GITHUB_ACTIONS +image-multi: build-rust-in-docker +endif $(DOCKER) buildx build --sbom=false --provenance=false --build-arg "BASE_IMAGE=$(RELEASE_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) --platform linux/amd64,linux/arm64 --file $(DOCKERFILE) ${PUSH_OPTION} . set-qemu: From 97cda02ea72d85a7f3a6aec86b0e4c97a821a49a Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 12:14:05 +0530 Subject: [PATCH 02/20] Load buildx image to docker engine Signed-off-by: Sreekanth --- .github/workflows/nightly-build.yml | 8 ++++---- Makefile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 3c0489f8a7..57b95d61d2 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -57,9 +57,9 @@ jobs: - name: Install dependencies run: sudo apt-get install -y protobuf-compiler - name: Build binary - run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu + run: cd rust/ && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu - name: Rename binary - run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + run: cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 - name: Upload numaflow binary uses: actions/upload-artifact@v4 with: @@ -86,9 +86,9 @@ jobs: - name: Install dependenices run: sudo apt-get install -y gcc-aarch64-linux-gnu protobuf-compiler - name: Build binary - run: export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER='aarch64-linux-gnu-gcc' && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu + run: cd rust && export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER='aarch64-linux-gnu-gcc' && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu - name: Rename binary - run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 + run: cp -pv rust/target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 - name: Upload numaflow binary uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index 26d45e974e..eae95cc3b7 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,7 @@ ui-test: ui-build .PHONY: image image: clean ui-build dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH) DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . - $(eval RUST_BUILDER=$(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION))) + $(eval RUST_BUILDER=$(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION))) --load docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-$(LOCAL_ARCH) dist/numaflow-rs-linux-$(LOCAL_ARCH) docker rm $(RUST_BUILDER) DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . From 95a8258e48f00918e9053915a6cb11e7b0b8c04d Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 13:31:35 +0530 Subject: [PATCH 03/20] Specify rust toolchain Signed-off-by: Sreekanth --- .github/workflows/nightly-build.yml | 14 ++++++++++---- rust/rust-toolchain.toml | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 rust/rust-toolchain.toml diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 57b95d61d2..6db7aff486 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -42,6 +42,9 @@ jobs: build-rust-amd64: runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain @@ -57,7 +60,7 @@ jobs: - name: Install dependencies run: sudo apt-get install -y protobuf-compiler - name: Build binary - run: cd rust/ && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu - name: Rename binary run: cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 - name: Upload numaflow binary @@ -68,6 +71,9 @@ jobs: build-rust-arm64: runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust steps: - uses: actions/checkout@v4 - name: Update Rust Toolchain Target @@ -86,7 +92,7 @@ jobs: - name: Install dependenices run: sudo apt-get install -y gcc-aarch64-linux-gnu protobuf-compiler - name: Build binary - run: cd rust && export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER='aarch64-linux-gnu-gcc' && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu + run: export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER='aarch64-linux-gnu-gcc' && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu - name: Rename binary run: cp -pv rust/target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 - name: Upload numaflow binary @@ -98,7 +104,7 @@ jobs: build-push-linux-multi: name: Build & push linux/amd64 and linux/arm64 needs: [ build-go-binaries, build-rust-amd64, build-rust-arm64] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.repository == 'numaproj/numaflow' strategy: matrix: @@ -116,7 +122,7 @@ jobs: key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Download Go binaries uses: actions/download-artifact@v3 diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml new file mode 100644 index 0000000000..a5b1f06904 --- /dev/null +++ b/rust/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +profile = "default" +channel = "1.80" From 37d6463a6dfa6b10e81397e1d683fc906f7a9a49 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 13:43:17 +0530 Subject: [PATCH 04/20] Load buildx image to docker engine Signed-off-by: Sreekanth --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index eae95cc3b7..3bf7f4138c 100644 --- a/Makefile +++ b/Makefile @@ -165,8 +165,8 @@ ui-test: ui-build .PHONY: image image: clean ui-build dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH) - DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . - $(eval RUST_BUILDER=$(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION))) --load + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . --load + $(eval RUST_BUILDER=$(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION))) docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-$(LOCAL_ARCH) dist/numaflow-rs-linux-$(LOCAL_ARCH) docker rm $(RUST_BUILDER) DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . From d9e6073ada2ff3fc2abeffda7f00c3d136d5feed Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 14:56:32 +0530 Subject: [PATCH 05/20] Build Rust binary in a separate job in e2e tests Signed-off-by: Sreekanth --- .github/workflows/ci.yaml | 35 +++++++++++++++++++++++++++++++++++ Makefile | 6 +++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 064afdb38c..cb693a86f4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -143,9 +143,39 @@ jobs: - run: make lint - run: git diff --exit-code + build-rust-amd64: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + with: + rustflags: '' + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.5 + - name: Install dependencies + run: sudo apt-get install -y protobuf-compiler + - name: Build binary + run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu + - name: Rename binary + run: cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + - name: Upload numaflow binary + uses: actions/upload-artifact@v4 + with: + name: numaflow-rs-linux-amd64 + path: numaflow-rs-linux-amd64 + e2e-tests: name: E2E Tests runs-on: ubuntu-latest + needs: [ build-rust-amd64 ] timeout-minutes: 20 strategy: fail-fast: false @@ -185,6 +215,11 @@ jobs: with: path: ui/node_modules key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} + - name: Download Rust amd64 binaries + uses: actions/download-artifact@v3 + with: + name: numaflow-rs-linux-amd64 + path: numaflow-rs-linux-amd64 - name: Install k3d run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash & - name: Create a cluster diff --git a/Makefile b/Makefile index 3bf7f4138c..784a6442f8 100644 --- a/Makefile +++ b/Makefile @@ -165,12 +165,16 @@ ui-test: ui-build .PHONY: image image: clean ui-build dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH) +ifdef GITHUB_ACTIONS + cp -pv numaflow-rs-linux-amd64 dist/numaflow-rs-linux-amd64 +else DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . --load - $(eval RUST_BUILDER=$(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION))) + $(eval RUST_BUILDER := $(shell docker create quay.io/numaproj/numaflow-rust-builder:latest)) docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-$(LOCAL_ARCH) dist/numaflow-rs-linux-$(LOCAL_ARCH) docker rm $(RUST_BUILDER) DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . @if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION); fi +endif ifdef IMAGE_IMPORT_CMD $(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) endif From 681c311fc1351a24c33dcc16a01e7032faf07feb Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 15:16:03 +0530 Subject: [PATCH 06/20] Use rustls-tls Signed-off-by: Sreekanth --- rust/Cargo.lock | 169 +++++++++++++++++++------------------- rust/servesink/Cargo.toml | 6 +- 2 files changed, 88 insertions(+), 87 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 3b4bfaa19b..4c08182013 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -84,7 +84,7 @@ dependencies = [ "ring", "rustls-native-certs", "rustls-pemfile 2.1.3", - "rustls-webpki", + "rustls-webpki 0.102.6", "serde", "serde_json", "serde_nanos", @@ -92,7 +92,7 @@ dependencies = [ "thiserror", "time", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tracing", "tryhard", "url", @@ -252,11 +252,11 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "pin-project-lite", - "rustls", + "rustls 0.23.12", "rustls-pemfile 2.1.3", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower", "tower-service", ] @@ -1102,15 +1102,29 @@ dependencies = [ "headers", "http 1.1.0", "hyper 1.4.1", - "hyper-rustls", + "hyper-rustls 0.27.2", "hyper-util", "pin-project-lite", "rustls-native-certs", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.30", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.2" @@ -1122,11 +1136,11 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "log", - "rustls", + "rustls 0.23.12", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", ] @@ -1156,22 +1170,6 @@ dependencies = [ "tokio-native-tls", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.4.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.6" @@ -1360,14 +1358,14 @@ dependencies = [ "http-body-util", "hyper 1.4.1", "hyper-http-proxy", - "hyper-rustls", + "hyper-rustls 0.27.2", "hyper-timeout", "hyper-util", "jsonpath-rust", "k8s-openapi", "kube-core", "pem", - "rustls", + "rustls 0.23.12", "rustls-pemfile 2.1.3", "secrecy", "serde", @@ -1581,7 +1579,7 @@ dependencies = [ "prost", "prost-types", "rcgen", - "rustls", + "rustls 0.23.12", "semver", "serde", "serde_json", @@ -1750,7 +1748,7 @@ version = "0.0.0-pre" dependencies = [ "k8s-openapi", "kube", - "reqwest 0.11.27", + "reqwest", "serde", "serde_derive", "serde_json", @@ -2301,7 +2299,8 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", - "hyper-tls 0.5.0", + "hyper-rustls 0.24.2", + "hyper-tls", "ipnet", "js-sys", "log", @@ -2311,6 +2310,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls 0.21.12", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -2319,55 +2319,14 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg 0.50.0", -] - -[[package]] -name = "reqwest" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" -dependencies = [ - "base64 0.22.1", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.4.5", - "http 1.1.0", - "http-body 1.0.1", - "http-body-util", - "hyper 1.4.1", - "hyper-rustls", - "hyper-tls 0.6.0", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile 2.1.3", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 1.0.1", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg 0.52.0", + "webpki-roots", + "winreg", ] [[package]] @@ -2441,6 +2400,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.12" @@ -2452,7 +2423,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] @@ -2495,6 +2466,16 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.6" @@ -2534,6 +2515,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "secrecy" version = "0.8.0" @@ -2684,7 +2675,7 @@ name = "servesink" version = "0.1.0" dependencies = [ "numaflow 0.1.0 (git+https://github.com/numaproj/numaflow-rs.git?branch=main)", - "reqwest 0.12.5", + "reqwest", "tokio", "tonic", "tracing", @@ -3037,13 +3028,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.12", "rustls-pki-types", "tokio", ] @@ -3493,6 +3494,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "which" version = "4.4.2" @@ -3694,16 +3701,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/rust/servesink/Cargo.toml b/rust/servesink/Cargo.toml index 70fa8e55f5..b554dd691d 100644 --- a/rust/servesink/Cargo.toml +++ b/rust/servesink/Cargo.toml @@ -7,6 +7,10 @@ edition = "2021" tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" } -reqwest = "0.12.4" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } + +[dependencies.reqwest] +default-features = false +version = "^0.11" +features = ["json", "multipart", "rustls-tls"] \ No newline at end of file From acc8b250308eb2f1e27ad77c18a7f9277f651358 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 15:33:55 +0530 Subject: [PATCH 07/20] Override dependency value for numaflow-models Signed-off-by: Sreekanth --- rust/numaflow-models/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/numaflow-models/Cargo.toml b/rust/numaflow-models/Cargo.toml index 6a38e0cc01..546575cc50 100644 --- a/rust/numaflow-models/Cargo.toml +++ b/rust/numaflow-models/Cargo.toml @@ -14,6 +14,8 @@ serde_derive = "^1.0" serde_json = "^1.0" url = "^2.2" uuid = { version = "^1.0", features = ["serde", "v4"] } + [dependencies.reqwest] +default-features = false version = "^0.11" -features = ["json", "multipart"] +features = ["json", "multipart", "rustls-tls"] From f9829308b2b6c8155ed71080df121bf3cc42ecae Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 15:49:16 +0530 Subject: [PATCH 08/20] Fix openapi template generation Signed-off-by: Sreekanth --- .github/workflows/ci.yaml | 2 +- rust/Cargo.lock | 108 ------------------ rust/numaflow-models/Cargo.toml | 3 +- rust/numaflow-models/templates/Cargo.mustache | 6 +- 4 files changed, 6 insertions(+), 113 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cb693a86f4..24c4337257 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -165,7 +165,7 @@ jobs: - name: Build binary run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu - name: Rename binary - run: cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 - name: Upload numaflow binary uses: actions/upload-artifact@v4 with: diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4c08182013..94bce16b9a 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -716,21 +716,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1157,19 +1142,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper 0.14.30", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "hyper-util" version = "0.1.6" @@ -1603,23 +1575,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" -[[package]] -name = "native-tls" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nkeys" version = "0.4.3" @@ -1771,50 +1726,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "openssl" -version = "0.10.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" -dependencies = [ - "bitflags 2.6.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "ordered-float" version = "2.10.1" @@ -2008,12 +1925,6 @@ dependencies = [ "spki", ] -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - [[package]] name = "portable-atomic" version = "1.7.0" @@ -2300,13 +2211,11 @@ dependencies = [ "http-body 0.4.6", "hyper 0.14.30", "hyper-rustls 0.24.2", - "hyper-tls", "ipnet", "js-sys", "log", "mime", "mime_guess", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -2318,7 +2227,6 @@ dependencies = [ "sync_wrapper 0.1.2", "system-configuration", "tokio", - "tokio-native-tls", "tokio-rustls 0.24.1", "tower-service", "url", @@ -3007,16 +2915,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-retry" version = "0.3.0" @@ -3391,12 +3289,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" diff --git a/rust/numaflow-models/Cargo.toml b/rust/numaflow-models/Cargo.toml index 546575cc50..1e133e3b08 100644 --- a/rust/numaflow-models/Cargo.toml +++ b/rust/numaflow-models/Cargo.toml @@ -14,8 +14,7 @@ serde_derive = "^1.0" serde_json = "^1.0" url = "^2.2" uuid = { version = "^1.0", features = ["serde", "v4"] } - [dependencies.reqwest] -default-features = false version = "^0.11" +default-features = false features = ["json", "multipart", "rustls-tls"] diff --git a/rust/numaflow-models/templates/Cargo.mustache b/rust/numaflow-models/templates/Cargo.mustache index a4bbdfeae2..f7d1cdeb7c 100644 --- a/rust/numaflow-models/templates/Cargo.mustache +++ b/rust/numaflow-models/templates/Cargo.mustache @@ -50,7 +50,8 @@ secrecy = "0.8.0" {{^supportAsync}} [dependencies.reqwest] version = "^0.11" -features = ["json", "blocking", "multipart"] +default-features = false +features = ["json", "blocking", "multipart", "rustls-tls"] {{/supportAsync}} {{#supportAsync}} {{#supportMiddleware}} @@ -58,6 +59,7 @@ reqwest-middleware = "0.2.0" {{/supportMiddleware}} [dependencies.reqwest] version = "^0.11" -features = ["json", "multipart"] +default-features = false +features = ["json", "multipart", "rustls-tls"] {{/supportAsync}} {{/reqwest}} From d4d4fb2ad561466b5a827368e1740c527b590cdd Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 17:11:47 +0530 Subject: [PATCH 09/20] Debug CI build failure Signed-off-by: Sreekanth --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 24c4337257..7f4c1ff4f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -165,7 +165,9 @@ jobs: - name: Build binary run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu - name: Rename binary - run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow ./numaflow-rs-linux-amd64 + - name: List files + run: pwd && ls -al - name: Upload numaflow binary uses: actions/upload-artifact@v4 with: From d2e9a28f1885e248450110d8198ea59c2bb2b57f Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 17:19:03 +0530 Subject: [PATCH 10/20] Explicitly specify relative path for upload/download Signed-off-by: Sreekanth --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7f4c1ff4f0..dcdcc6b47d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -172,7 +172,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: numaflow-rs-linux-amd64 - path: numaflow-rs-linux-amd64 + path: ./numaflow-rs-linux-amd64 e2e-tests: name: E2E Tests @@ -221,7 +221,7 @@ jobs: uses: actions/download-artifact@v3 with: name: numaflow-rs-linux-amd64 - path: numaflow-rs-linux-amd64 + path: ./numaflow-rs-linux-amd64 - name: Install k3d run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash & - name: Create a cluster From 04f56dc96a99db098120a661c5a7e9c64497f5d5 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 17:25:43 +0530 Subject: [PATCH 11/20] Upload action doesn't use working-directory setting Signed-off-by: Sreekanth --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dcdcc6b47d..930aa41157 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -172,7 +172,8 @@ jobs: uses: actions/upload-artifact@v4 with: name: numaflow-rs-linux-amd64 - path: ./numaflow-rs-linux-amd64 + path: rust/numaflow-rs-linux-amd64 + if-no-files-found: error e2e-tests: name: E2E Tests @@ -221,7 +222,7 @@ jobs: uses: actions/download-artifact@v3 with: name: numaflow-rs-linux-amd64 - path: ./numaflow-rs-linux-amd64 + path: numaflow-rs-linux-amd64 - name: Install k3d run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash & - name: Create a cluster From 37a4ce04ff4301785c9d7bbc6c87b922beaef2bb Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 17:36:05 +0530 Subject: [PATCH 12/20] Fix download action Signed-off-by: Sreekanth --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 930aa41157..a46fa8cddf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -222,7 +222,6 @@ jobs: uses: actions/download-artifact@v3 with: name: numaflow-rs-linux-amd64 - path: numaflow-rs-linux-amd64 - name: Install k3d run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash & - name: Create a cluster From 50dce61426f325d7bec8352749d978c323cc6a3d Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 17:59:24 +0530 Subject: [PATCH 13/20] Use v4 for upload and download Signed-off-by: Sreekanth --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a46fa8cddf..cecb674b3f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -219,7 +219,7 @@ jobs: path: ui/node_modules key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} - name: Download Rust amd64 binaries - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: numaflow-rs-linux-amd64 - name: Install k3d From e76c88b9da328dc0acdc5df2eb33e8bf9c1649fe Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 18:12:26 +0530 Subject: [PATCH 14/20] Fix Makefile Signed-off-by: Sreekanth --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 784a6442f8..b8a681f5f9 100644 --- a/Makefile +++ b/Makefile @@ -172,9 +172,9 @@ else $(eval RUST_BUILDER := $(shell docker create quay.io/numaproj/numaflow-rust-builder:latest)) docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-$(LOCAL_ARCH) dist/numaflow-rs-linux-$(LOCAL_ARCH) docker rm $(RUST_BUILDER) +endif DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . @if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION); fi -endif ifdef IMAGE_IMPORT_CMD $(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) endif From 9e80b18eb929677d299a69f6ea69e8999bbe35a0 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 18:35:53 +0530 Subject: [PATCH 15/20] Fix Go binary name Signed-off-by: Sreekanth --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index b8a681f5f9..d4dde0c1cd 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,7 @@ ui-test: ui-build image: clean ui-build dist/$(BINARY_NAME)-linux-$(LOCAL_ARCH) ifdef GITHUB_ACTIONS cp -pv numaflow-rs-linux-amd64 dist/numaflow-rs-linux-amd64 + cp -pv dist/numaflow-linux-x86_64 dist/numaflow-linux-amd64 else DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . --load $(eval RUST_BUILDER := $(shell docker create quay.io/numaproj/numaflow-rust-builder:latest)) From 53d1316e7a4034106a183cc1533c046152f2ad5d Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 19:02:11 +0530 Subject: [PATCH 16/20] Use v3 for upload and download actions Signed-off-by: Sreekanth --- .github/workflows/nightly-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 6db7aff486..5a2b9a3d68 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -62,9 +62,9 @@ jobs: - name: Build binary run: RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu - name: Rename binary - run: cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 + run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64 - name: Upload numaflow binary - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: numaflow-rs-linux-amd64 path: numaflow-rs-linux-amd64 @@ -94,9 +94,9 @@ jobs: - name: Build binary run: export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER='aarch64-linux-gnu-gcc' && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target aarch64-unknown-linux-gnu - name: Rename binary - run: cp -pv rust/target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 + run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64 - name: Upload numaflow binary - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: numaflow-rs-linux-arm64 path: numaflow-rs-linux-arm64 From 59d47d25dfe157f2f9600529d868a2f11926b624 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 19:12:03 +0530 Subject: [PATCH 17/20] Fix path for upload action Signed-off-by: Sreekanth --- .github/workflows/nightly-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 5a2b9a3d68..4a28f45bdf 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -67,7 +67,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: numaflow-rs-linux-amd64 - path: numaflow-rs-linux-amd64 + path: rust/numaflow-rs-linux-amd64 build-rust-arm64: runs-on: ubuntu-latest @@ -99,7 +99,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: numaflow-rs-linux-arm64 - path: numaflow-rs-linux-arm64 + path: rust/numaflow-rs-linux-arm64 build-push-linux-multi: name: Build & push linux/amd64 and linux/arm64 From a1196b40ab2bef6b0d9ae08962995d56c0c96d06 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Fri, 16 Aug 2024 19:30:17 +0530 Subject: [PATCH 18/20] Use distroless image Signed-off-by: Sreekanth --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fce05135f0..96fafae3b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ # numaflow #################################################################################################### ARG BASE_IMAGE -FROM debian:bookworm AS numaflow +FROM gcr.io/distroless/static-debian12 AS numaflow ARG ARCH COPY dist/numaflow-linux-${ARCH} /bin/numaflow From 6b0ec2fcf5732da9ac647c0fe820611b0fc0fb00 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Sat, 17 Aug 2024 12:28:52 +0530 Subject: [PATCH 19/20] Fix image name in Makefile Signed-off-by: Sreekanth --- .dockerignore | 1 + Makefile | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..7a8cf80175 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +rust/target \ No newline at end of file diff --git a/Makefile b/Makefile index d4dde0c1cd..03aa2ed137 100644 --- a/Makefile +++ b/Makefile @@ -170,11 +170,11 @@ ifdef GITHUB_ACTIONS cp -pv dist/numaflow-linux-x86_64 dist/numaflow-linux-amd64 else DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target builder -f $(DOCKERFILE) . --load - $(eval RUST_BUILDER := $(shell docker create quay.io/numaproj/numaflow-rust-builder:latest)) + $(eval RUST_BUILDER := $(shell docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION))) docker cp $(RUST_BUILDER):/root/numaflow-rs-linux-$(LOCAL_ARCH) dist/numaflow-rs-linux-$(LOCAL_ARCH) docker rm $(RUST_BUILDER) endif - DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . + DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target numaflow -f $(DOCKERFILE) . --load @if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION); fi ifdef IMAGE_IMPORT_CMD $(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) @@ -258,7 +258,7 @@ start: image .PHONY: e2eapi-image e2eapi-image: clean dist/e2eapi - DOCKER_BUILDKIT=1 $(DOCKER) build . --build-arg "ARCH=amd64" --target e2eapi --tag $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) --build-arg VERSION="$(VERSION)" + DOCKER_BUILDKIT=1 $(DOCKER) build . --target e2eapi --tag $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) --build-arg VERSION="$(VERSION)" --load @if [[ "$(DOCKER_PUSH)" = "true" ]]; then $(DOCKER) push $(IMAGE_NAMESPACE)/e2eapi:$(VERSION); fi ifdef IMAGE_IMPORT_CMD $(IMAGE_IMPORT_CMD) $(IMAGE_NAMESPACE)/e2eapi:$(VERSION) From 58ecdd3f68da2f23fe44110371b86d443a91e559 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Sat, 17 Aug 2024 15:46:43 +0530 Subject: [PATCH 20/20] Debug print container logs Signed-off-by: Sreekanth --- Makefile | 2 +- test/fixtures/util.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 03aa2ed137..a537c659be 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ dist/$(BINARY_NAME): go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/$(BINARY_NAME) ./cmd dist/e2eapi: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/e2eapi ./test/e2e-api + CGO_ENABLED=0 GOOS=linux go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/e2eapi ./test/e2e-api dist/$(BINARY_NAME)-%: CGO_ENABLED=0 $(GOARGS) go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/$(BINARY_NAME)-$* ./cmd diff --git a/test/fixtures/util.go b/test/fixtures/util.go index 2c8c8a0ae1..1288f846b0 100644 --- a/test/fixtures/util.go +++ b/test/fixtures/util.go @@ -525,6 +525,7 @@ func podLogContains(ctx context.Context, client kubernetes.Interface, namespace, return s.Err() } data := s.Bytes() + log.Printf("pod=%s, container=%s msg=%s", podName, containerName, string(data)) if exp.Match(data) { result <- true }