diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml new file mode 100644 index 0000000..0eb2228 --- /dev/null +++ b/.github/workflows/build-push.yml @@ -0,0 +1,40 @@ +name: Docker Publish + +on: + push: + branches: [ main ] + tags: + - '*' + +jobs: + docker_publish: + # run workflow only on numaproj/numaflow-rs repository + if: ${{ github.repository }} == "numaproj/numaflow-rs" + name: Build, Tag, and Push Image + runs-on: ubuntu-latest + + strategy: + matrix: + dockerfile_paths: [ + "examples/map-cat", "examples/map-tickgen-serde", "examples/mapt-event-time-filter", + "examples/reduce-counter", "examples/sideinput", "examples/sideinput/udf", + "examples/simple-source", "examples/sink-log", "examples/source-transformer-now" + ] + + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Quay.io registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.NUMAIO_USERNAME }} + password: ${{ secrets.NUMAIO_PASSWORD }} + - name: Build, tag, and push images + run: ./hack/update_examples.sh --build-push-example ${{ matrix.dockerfile_paths }} diff --git a/examples/map-cat/Cargo.toml b/examples/map-cat/Cargo.toml index 0047934..c4b0484 100644 --- a/examples/map-cat/Cargo.toml +++ b/examples/map-cat/Cargo.toml @@ -11,4 +11,4 @@ path = "src/main.rs" [dependencies] tonic = "0.10.2" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" } +numaflow = { path = "../../" } diff --git a/examples/map-cat/Dockerfile b/examples/map-cat/Dockerfile index 2d364ff..3e9d7d5 100644 --- a/examples/map-cat/Dockerfile +++ b/examples/map-cat/Dockerfile @@ -1,25 +1,20 @@ -FROM rust:1.75-bookworm as build +FROM rust:1.75-bookworm AS build RUN apt-get update RUN apt-get install protobuf-compiler -y -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/map-cat # build for release RUN cargo build --release # our final base -FROM debian:bookworm +FROM debian:bookworm AS map-cat # copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . +COPY --from=build /numaflow-rs/examples/map-cat/target/release/server . # set the startup command to run your binary -CMD ["./server"] \ No newline at end of file +CMD ["./server"] diff --git a/examples/map-cat/Makefile b/examples/map-cat/Makefile new file mode 100644 index 0000000..0d4b653 --- /dev/null +++ b/examples/map-cat/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/map-cat:${TAG} +DOCKER_FILE_PATH = examples/map-cat/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/map-cat/manifests/simple-map-cat.yaml b/examples/map-cat/manifests/simple-map-cat.yaml index f1766c9..941bafb 100644 --- a/examples/map-cat/manifests/simple-map-cat.yaml +++ b/examples/map-cat/manifests/simple-map-cat.yaml @@ -17,7 +17,7 @@ spec: min: 1 udf: container: - image: quay.io/numaio/numaflow-rs/map-cat:v1 + image: quay.io/numaio/numaflow-rs/map-cat:stable - name: out sink: # A simple log printing sink diff --git a/examples/map-cat/src/main.rs b/examples/map-cat/src/main.rs index ab9c5e2..cb6467d 100644 --- a/examples/map-cat/src/main.rs +++ b/examples/map-cat/src/main.rs @@ -1,7 +1,7 @@ use numaflow::map; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> Result<(), Box> { map::Server::new(Cat).start().await } diff --git a/examples/map-tickgen-serde/Cargo.toml b/examples/map-tickgen-serde/Cargo.toml index 6453d69..294a0b1 100644 --- a/examples/map-tickgen-serde/Cargo.toml +++ b/examples/map-tickgen-serde/Cargo.toml @@ -13,4 +13,4 @@ tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } serde = { version = "1.0.103", features = ["derive"] } serde_json = "1.0.103" chrono = "0.4.26" -numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" } +numaflow = { path = "../../" } diff --git a/examples/map-tickgen-serde/Dockerfile b/examples/map-tickgen-serde/Dockerfile index 2d364ff..5c1c68c 100644 --- a/examples/map-tickgen-serde/Dockerfile +++ b/examples/map-tickgen-serde/Dockerfile @@ -1,25 +1,20 @@ -FROM rust:1.75-bookworm as build +FROM rust:1.75-bookworm AS build RUN apt-get update RUN apt-get install protobuf-compiler -y -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/map-tickgen-serde # build for release RUN cargo build --release # our final base -FROM debian:bookworm +FROM debian:bookworm AS map-tickgen-serde # copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . +COPY --from=build /numaflow-rs/examples/map-tickgen-serde/target/release/server . # set the startup command to run your binary CMD ["./server"] \ No newline at end of file diff --git a/examples/map-tickgen-serde/Makefile b/examples/map-tickgen-serde/Makefile new file mode 100644 index 0000000..b980eda --- /dev/null +++ b/examples/map-tickgen-serde/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/map-tickgen-serde:${TAG} +DOCKER_FILE_PATH = examples/map-tickgen-serde/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/map-tickgen-serde/manifests/simple-map-udf.yaml b/examples/map-tickgen-serde/manifests/simple-map-udf.yaml index 46b9a1a..c0b1c36 100644 --- a/examples/map-tickgen-serde/manifests/simple-map-udf.yaml +++ b/examples/map-tickgen-serde/manifests/simple-map-udf.yaml @@ -17,7 +17,7 @@ spec: min: 1 udf: container: - image: quay.io/numaio/numaflow-rs/map-tickgen-serde:v1 + image: quay.io/numaio/numaflow-rs/map-tickgen-serde:stable - name: out sink: # A simple log printing sink diff --git a/examples/source-transformer-event-time-filter/.dockerignore b/examples/mapt-event-time-filter/.dockerignore similarity index 100% rename from examples/source-transformer-event-time-filter/.dockerignore rename to examples/mapt-event-time-filter/.dockerignore diff --git a/examples/source-transformer-event-time-filter/Cargo.toml b/examples/mapt-event-time-filter/Cargo.toml similarity index 100% rename from examples/source-transformer-event-time-filter/Cargo.toml rename to examples/mapt-event-time-filter/Cargo.toml diff --git a/examples/mapt-event-time-filter/Dockerfile b/examples/mapt-event-time-filter/Dockerfile new file mode 100644 index 0000000..e896439 --- /dev/null +++ b/examples/mapt-event-time-filter/Dockerfile @@ -0,0 +1,20 @@ +FROM rust:1.75-bookworm AS build + +RUN apt-get update +RUN apt-get install protobuf-compiler -y + +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/mapt-event-time-filter + +# build for release +RUN cargo build --release + +# our final base +FROM debian:bookworm AS mapt-event-time-filter + +# copy the build artifact from the build stage +COPY --from=build /numaflow-rs/examples/mapt-event-time-filter/target/release/server . + +# set the startup command to run your binary +CMD ["./server"] diff --git a/examples/mapt-event-time-filter/Makefile b/examples/mapt-event-time-filter/Makefile new file mode 100644 index 0000000..1b0564c --- /dev/null +++ b/examples/mapt-event-time-filter/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/mapt-event-time-filter:${TAG} +DOCKER_FILE_PATH = examples/mapt-event-time-filter/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/source-transformer-event-time-filter/manifests/event-time-filter.yaml b/examples/mapt-event-time-filter/manifests/event-time-filter.yaml similarity index 84% rename from examples/source-transformer-event-time-filter/manifests/event-time-filter.yaml rename to examples/mapt-event-time-filter/manifests/event-time-filter.yaml index c9e2c6e..14c3bb2 100644 --- a/examples/source-transformer-event-time-filter/manifests/event-time-filter.yaml +++ b/examples/mapt-event-time-filter/manifests/event-time-filter.yaml @@ -13,7 +13,7 @@ spec: duration: 1s transformer: container: - image: quay.io/numaio/numaflow-rust/mapt-event-time-filter:stable + image: quay.io/numaio/numaflow-rs/mapt-event-time-filter:stable imagePullPolicy: Always - name: out scale: diff --git a/examples/source-transformer-event-time-filter/src/main.rs b/examples/mapt-event-time-filter/src/main.rs similarity index 100% rename from examples/source-transformer-event-time-filter/src/main.rs rename to examples/mapt-event-time-filter/src/main.rs diff --git a/examples/reduce-counter/Cargo.toml b/examples/reduce-counter/Cargo.toml index 0569082..aa6bf1b 100644 --- a/examples/reduce-counter/Cargo.toml +++ b/examples/reduce-counter/Cargo.toml @@ -10,4 +10,5 @@ path = "src/main.rs" [dependencies] tonic = "0.11.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow-rs = { path = "../../" } +numaflow = { path = "../../" } + diff --git a/examples/reduce-counter/Dockerfile b/examples/reduce-counter/Dockerfile index 5bc8583..4f1593b 100644 --- a/examples/reduce-counter/Dockerfile +++ b/examples/reduce-counter/Dockerfile @@ -1,25 +1,20 @@ -FROM rust:1.70 as build +FROM rust:1.70 AS build RUN apt-get update RUN apt-get install protobuf-compiler -y -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/reduce-counter # build for release RUN cargo build --release # our final base -FROM rust +FROM rust AS reduce-counter # copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . +COPY --from=build /numaflow-rs/examples/reduce-counter/target/release/server . # set the startup command to run your binary CMD ["./server"] \ No newline at end of file diff --git a/examples/reduce-counter/Makefile b/examples/reduce-counter/Makefile new file mode 100644 index 0000000..14c165a --- /dev/null +++ b/examples/reduce-counter/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/reduce-counter:${TAG} +DOCKER_FILE_PATH = examples/reduce-counter/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/reduce-counter/manifests/simple-reduce.yaml b/examples/reduce-counter/manifests/simple-reduce.yaml index 5a469bf..20a77dd 100644 --- a/examples/reduce-counter/manifests/simple-reduce.yaml +++ b/examples/reduce-counter/manifests/simple-reduce.yaml @@ -20,7 +20,7 @@ spec: udf: container: # count element - image: quay.io/numaio/numaflow-rs/reduce-counter:v1 + image: quay.io/numaio/numaflow-rs/reduce-counter:stable groupBy: window: fixed: diff --git a/examples/side-input/Dockerfile b/examples/side-input/Dockerfile deleted file mode 100644 index 5bc8583..0000000 --- a/examples/side-input/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM rust:1.70 as build - -RUN apt-get update -RUN apt-get install protobuf-compiler -y - -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock - -# build for release -RUN cargo build --release - -# our final base -FROM rust - -# copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . - -# set the startup command to run your binary -CMD ["./server"] \ No newline at end of file diff --git a/examples/side-input/Makefile b/examples/side-input/Makefile deleted file mode 100644 index 6cefe48..0000000 --- a/examples/side-input/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -.PHONY: build -build: - cargo build --release --target x86_64-unknown-linux-gnu - mkdir -p ./dist - cp ./target/x86_64-unknown-linux-gnu/release/side-input ./dist/ - -.PHONY: image -image: build - docker buildx build -t "quay.io/numaio/numaflow-rust/sideinput-example:v0.5.0" --platform linux/amd64,linux/arm64 --target sideinput . --push - -clean: - cargo clean - -rm -rf ./dist diff --git a/examples/sideinput-udf/Dockerfile b/examples/sideinput-udf/Dockerfile deleted file mode 100644 index 41fa222..0000000 --- a/examples/sideinput-udf/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM rust:1.75-bookworm as build - -RUN apt-get update -RUN apt-get install protobuf-compiler -y - -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock - -# build for release -RUN cargo build --release - -# our final base -FROM debian:bookworm - -# copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . - -RUN mkdir -p /var/numaflow/side-inputs - -# set the startup command to run your binary -CMD ["./server"] \ No newline at end of file diff --git a/examples/side-input/Cargo.toml b/examples/sideinput/Cargo.toml similarity index 74% rename from examples/side-input/Cargo.toml rename to examples/sideinput/Cargo.toml index 861cce3..03ca6a8 100644 --- a/examples/side-input/Cargo.toml +++ b/examples/sideinput/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "side-input" +name = "sideinput" version = "0.1.0" edition = "2021" @@ -11,5 +11,5 @@ path = "src/main.rs" [dependencies] tonic = "0.9" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" } +numaflow = { path = "../../" } chrono = "0.4.30" \ No newline at end of file diff --git a/examples/sideinput/Dockerfile b/examples/sideinput/Dockerfile new file mode 100644 index 0000000..1216611 --- /dev/null +++ b/examples/sideinput/Dockerfile @@ -0,0 +1,20 @@ +FROM rust:1.70 AS build + +RUN apt-get update +RUN apt-get install protobuf-compiler -y + +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/sideinput + +# build for release +RUN cargo build --release + +# our final base +FROM rust AS sideinput + +# copy the build artifact from the build stage +COPY --from=build /numaflow-rs/examples/sideinput/target/release/server . + +# set the startup command to run your binary +CMD ["./server"] \ No newline at end of file diff --git a/examples/sideinput/Makefile b/examples/sideinput/Makefile new file mode 100644 index 0000000..11734fc --- /dev/null +++ b/examples/sideinput/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/sideinput-example:${TAG} +DOCKER_FILE_PATH = examples/sideinput/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target \ No newline at end of file diff --git a/examples/side-input/manifests/simple-sideinput.yaml b/examples/sideinput/manifests/simple-sideinput.yaml similarity index 84% rename from examples/side-input/manifests/simple-sideinput.yaml rename to examples/sideinput/manifests/simple-sideinput.yaml index 6fb7008..34bb09c 100644 --- a/examples/side-input/manifests/simple-sideinput.yaml +++ b/examples/sideinput/manifests/simple-sideinput.yaml @@ -6,7 +6,7 @@ spec: sideInputs: - name: myticker container: - image: "quay.io/numaio/numaflow-rust/sideinput-example:v0.5.0" + image: quay.io/numaio/numaflow-rs/sideinput-example:stable imagePullPolicy: Always trigger: schedule: "*/2 * * * *" @@ -21,7 +21,7 @@ spec: - name: si-log udf: container: - image: "quay.io/numaio/numaflow-rust/udf-sideinput-example:v0.5.0" + image: quay.io/numaio/numaflow-rs/udf-sideinput-example:stable imagePullPolicy: Always containerTemplate: env: diff --git a/examples/side-input/src/main.rs b/examples/sideinput/src/main.rs similarity index 75% rename from examples/side-input/src/main.rs rename to examples/sideinput/src/main.rs index 7f842b7..448a152 100644 --- a/examples/side-input/src/main.rs +++ b/examples/sideinput/src/main.rs @@ -1,7 +1,8 @@ use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; +use numaflow::sideinput::{self, SideInputer}; + -use numaflow::sideinput::SideInputer; -use numaflow::sideinput::start_uds_server; use tonic::async_trait; struct SideInputHandler { @@ -35,8 +36,6 @@ impl SideInputer for SideInputHandler { } #[tokio::main] -async fn main() -> Result<(), Box> { - let side_input_handler = SideInputHandler::new(); - start_uds_server(side_input_handler).await?; - Ok(()) +async fn main() -> Result<(), Box> { + sideinput::Server::new(SideInputHandler::new()).start().await } diff --git a/examples/sideinput-udf/Cargo.toml b/examples/sideinput/udf/Cargo.toml similarity index 77% rename from examples/sideinput-udf/Cargo.toml rename to examples/sideinput/udf/Cargo.toml index ec95a61..cabeb5c 100644 --- a/examples/sideinput-udf/Cargo.toml +++ b/examples/sideinput/udf/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "udf" +name = "sideinput-udf" version = "0.1.0" edition = "2021" @@ -12,6 +12,6 @@ path = "src/main.rs" [dependencies] tonic = "0.10.2" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" } +numaflow = { path = "../../../" } chrono = "0.4.30" notify = "6.1.1" \ No newline at end of file diff --git a/examples/sideinput/udf/Dockerfile b/examples/sideinput/udf/Dockerfile new file mode 100644 index 0000000..a36ab4b --- /dev/null +++ b/examples/sideinput/udf/Dockerfile @@ -0,0 +1,22 @@ +FROM rust:1.75-bookworm AS build + +RUN apt-get update +RUN apt-get install protobuf-compiler -y + +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/sideinput/udf + +# build for release +RUN cargo build --release + +# our final base +FROM debian:bookworm AS sideinput-udf + +# copy the build artifact from the build stage +COPY --from=build /numaflow-rs/examples/sideinput/udf/target/release/server . + +RUN mkdir -p /var/numaflow/sideinputs + +# set the startup command to run your binary +CMD ["./server"] \ No newline at end of file diff --git a/examples/sideinput/udf/Makefile b/examples/sideinput/udf/Makefile new file mode 100644 index 0000000..d6932c9 --- /dev/null +++ b/examples/sideinput/udf/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/udf-sideinput-example:${TAG} +DOCKER_FILE_PATH = examples/sideinput/udf/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/sideinput-udf/src/main.rs b/examples/sideinput/udf/src/main.rs similarity index 90% rename from examples/sideinput-udf/src/main.rs rename to examples/sideinput/udf/src/main.rs index b823388..c998b76 100644 --- a/examples/sideinput-udf/src/main.rs +++ b/examples/sideinput/udf/src/main.rs @@ -5,7 +5,7 @@ use numaflow::map::{Mapper, MapRequest, Message, Server}; use tokio::spawn; use tonic::async_trait; -const DIR_PATH: &str = "/var/numaflow/side-inputs"; +const DIR_PATH: &str = "/var/numaflow/sideinputs"; struct UdfMapper; @@ -13,9 +13,9 @@ struct UdfMapper; impl Mapper for UdfMapper { async fn map(&self, _input: MapRequest) -> Vec { let message = Message { - keys: vec![], + keys: vec![].into(), value: b"some_value".to_vec(), - tags: vec![], + tags: vec![].into(), }; vec![message] } diff --git a/examples/simple-source/Dockerfile b/examples/simple-source/Dockerfile index 9ff65d8..99657cc 100644 --- a/examples/simple-source/Dockerfile +++ b/examples/simple-source/Dockerfile @@ -1,25 +1,20 @@ -FROM rust:1.76-bookworm as build +FROM rust:1.76-bookworm AS build RUN apt-get update RUN apt-get install protobuf-compiler -y -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/simple-source # build for release RUN cargo build --release # our final base -FROM debian:bookworm +FROM debian:bookworm AS simple-source # copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . +COPY --from=build /numaflow-rs/examples/simple-source/target/release/server . # set the startup command to run your binary CMD ["./server"] \ No newline at end of file diff --git a/examples/simple-source/Makefile b/examples/simple-source/Makefile new file mode 100644 index 0000000..b7769e8 --- /dev/null +++ b/examples/simple-source/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/simple-source:${TAG} +DOCKER_FILE_PATH = examples/simple-source/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/simple-source/manifests/simple-source.yaml b/examples/simple-source/manifests/simple-source.yaml index c2006e7..1084d1a 100644 --- a/examples/simple-source/manifests/simple-source.yaml +++ b/examples/simple-source/manifests/simple-source.yaml @@ -10,7 +10,7 @@ spec: source: udsource: container: - image: quay.io/numaio/numaflow-rs/simple-source:v1 + image: quay.io/numaio/numaflow-rs/simple-source:stable imagePullPolicy: Always - name: out scale: diff --git a/examples/simple-source/src/main.rs b/examples/simple-source/src/main.rs index b05cd10..f2f2b82 100644 --- a/examples/simple-source/src/main.rs +++ b/examples/simple-source/src/main.rs @@ -8,9 +8,6 @@ async fn main() -> Result<(), Box> { pub(crate) mod simple_source { use numaflow::source::{Message, Offset, SourceReadRequest, Sourcer}; - use std::collections::HashMap; - use std::sync::Arc; - use std::sync::Arc; use std::{ collections::HashMap, collections::HashSet, diff --git a/examples/sink-log/Cargo.toml b/examples/sink-log/Cargo.toml index 034cb88..d4dda92 100644 --- a/examples/sink-log/Cargo.toml +++ b/examples/sink-log/Cargo.toml @@ -10,4 +10,4 @@ path = "src/main.rs" [dependencies] tonic = "0.9" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" } +numaflow = { path = "../../" } diff --git a/examples/sink-log/Dockerfile b/examples/sink-log/Dockerfile index 5bc8583..0a41f3f 100644 --- a/examples/sink-log/Dockerfile +++ b/examples/sink-log/Dockerfile @@ -1,25 +1,20 @@ -FROM rust:1.70 as build +FROM rust:1.70 AS build RUN apt-get update RUN apt-get install protobuf-compiler -y -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/sink-log # build for release RUN cargo build --release # our final base -FROM rust +FROM rust AS sink-log # copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . +COPY --from=build /numaflow-rs/examples/sink-log/target/release/server . # set the startup command to run your binary CMD ["./server"] \ No newline at end of file diff --git a/examples/sink-log/Makefile b/examples/sink-log/Makefile new file mode 100644 index 0000000..5db536d --- /dev/null +++ b/examples/sink-log/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/sink-log:${TAG} +DOCKER_FILE_PATH = examples/sink-log/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/sink-log/manifests/simple-sink.yaml b/examples/sink-log/manifests/simple-sink.yaml index a7ddc68..419dfa1 100644 --- a/examples/sink-log/manifests/simple-sink.yaml +++ b/examples/sink-log/manifests/simple-sink.yaml @@ -16,7 +16,7 @@ spec: sink: udsink: container: - image: quay.io/numaio/numaflow-rs/sink-log:v1 + image: quay.io/numaio/numaflow-rs/sink-log:stable edges: - from: in to: out diff --git a/examples/sink-log/src/main.rs b/examples/sink-log/src/main.rs index 53dbcba..bd4c234 100644 --- a/examples/sink-log/src/main.rs +++ b/examples/sink-log/src/main.rs @@ -1,7 +1,7 @@ use numaflow::sink::{self, Response, SinkRequest}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> Result<(), Box> { sink::Server::new(Logger).start().await } diff --git a/examples/source-transformer-event-time-filter/Dockerfile b/examples/source-transformer-event-time-filter/Dockerfile deleted file mode 100644 index 6a1049e..0000000 --- a/examples/source-transformer-event-time-filter/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM rust:1.75-bookworm as build - -RUN apt-get update -RUN apt-get install protobuf-compiler -y - -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock - -# build for release -RUN cargo build --release - -# our final base -FROM debian:bookworm - -# copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . - -# set the startup command to run your binary -CMD ["./server"] - -#docker buildx build --platform linux/amd64,linux/arm64 -t quay.io/numaio/numaflow-rust/source-transformer:stable --push . \ No newline at end of file diff --git a/examples/source-transformer-now/Dockerfile b/examples/source-transformer-now/Dockerfile index 77cf80e..d079a4a 100644 --- a/examples/source-transformer-now/Dockerfile +++ b/examples/source-transformer-now/Dockerfile @@ -1,25 +1,20 @@ -FROM rust:1.75-bookworm as build +FROM rust:1.75-bookworm AS build RUN apt-get update RUN apt-get install protobuf-compiler -y -# create a new empty shell project -WORKDIR /examples - -# copy your source tree -COPY src ./src - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock +WORKDIR /numaflow-rs +COPY ./ ./ +WORKDIR /numaflow-rs/examples/source-transformer-now # build for release RUN cargo build --release # our final base -FROM debian:bookworm +FROM debian:bookworm AS source-transformer-now # copy the build artifact from the build stage -COPY --from=build /examples/target/release/server . +COPY --from=build /numaflow-rs/examples/source-transformer-now/target/release/server . # set the startup command to run your binary CMD ["./server"] diff --git a/examples/source-transformer-now/Makefile b/examples/source-transformer-now/Makefile new file mode 100644 index 0000000..ffdefcd --- /dev/null +++ b/examples/source-transformer-now/Makefile @@ -0,0 +1,20 @@ +TAG ?= stable +PUSH ?= false +IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/source-transformer-now:${TAG} +DOCKER_FILE_PATH = examples/source-transformer-now/Dockerfile + +.PHONY: update +update: + cargo check + cargo update + +.PHONY: image +image: update + cd ../../ && docker build \ + -f ${DOCKER_FILE_PATH} \ + -t ${IMAGE_REGISTRY} . + @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi + +.PHONY: clean +clean: + -rm -rf target diff --git a/examples/source-transformer-now/manifests/simple-source-transformer.yaml b/examples/source-transformer-now/manifests/simple-source-transformer.yaml index ab5ec08..7f7349f 100644 --- a/examples/source-transformer-now/manifests/simple-source-transformer.yaml +++ b/examples/source-transformer-now/manifests/simple-source-transformer.yaml @@ -14,7 +14,7 @@ spec: value: 5 transformer: container: - image: quay.io/numaio/numaflow-rs/source-transformer-now:v1 + image: quay.io/numaio/numaflow-rs/source-transformer-now:stable imagePullPolicy: Always - name: out sink: diff --git a/examples/source-transformer-now/src/main.rs b/examples/source-transformer-now/src/main.rs index b42454f..42f3273 100644 --- a/examples/source-transformer-now/src/main.rs +++ b/examples/source-transformer-now/src/main.rs @@ -3,7 +3,7 @@ use numaflow::sourcetransform; /// A simple source transformer which assigns event time to the current time in utc. #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> Result<(), Box> { sourcetransform::Server::new(NowCat).start().await } diff --git a/hack/update_examples.sh b/hack/update_examples.sh new file mode 100755 index 0000000..27ce941 --- /dev/null +++ b/hack/update_examples.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +function show_help () { + echo "Usage: $0 [-h|--help | -t|--tag ] (-bpe|--build-push-example )" + echo " -h, --help Display help message and exit" + echo " -bpe, --build-push-example Build the Dockerfile of the given example directory path, and push it to the quay.io registry" + echo " -t, --tag To be optionally used with -bpe. Specify the tag to build with. Default tag: stable" +} + +if [ $# -eq 0 ]; then + echo "Error: provide at least one argument" >&2 + show_help + exit 1 +fi + +usingHelp=0 +usingBuildPushExample=0 +usingTag=0 +directoryPath="" +tag="stable" + +function handle_options () { + while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usingHelp=1 + ;; + -bpe | --build-push-example) + if [ -z "$2" ]; then + echo "Directory path not specified." >&2 + show_help + exit 1 + fi + + usingBuildPushExample=1 + directoryPath=$2 + shift + ;; + -t | --tag) + if [ -z "$2" ]; then + echo "Tag not specified." >&2 + show_help + exit 1 + fi + + usingTag=1 + tag=$2 + shift + ;; + *) + echo "Invalid option: $1" >&2 + show_help + exit 1 + ;; + esac + shift + done +} + +handle_options "$@" + +if (( usingBuildPushExample + usingHelp > 1 )); then + echo "Only one of '-h', '-bpe', or '-u' is allowed at a time" >&2 + show_help + exit 1 +fi + +if (( (usingTag + usingHelp > 1) || (usingTag && usingBuildPushExample == 0) )); then + echo "Can only use -t with -bpe" >&2 + show_help + exit 1 +fi + +if [ -n "$tag" ] && (( ! usingHelp )); then + echo "Using tag: $tag" +fi + +if (( usingBuildPushExample )); then + cd "./$directoryPath" || exit + if ! make image TAG="$tag" PUSH=true; then + echo "Error: failed to run make image in $directoryPath" >&2 + exit 1 + fi +elif (( usingHelp )); then + show_help +fi