feat: automate docker build and push + fix broken examples

.github/workflows/build-push.yml

@@ -0,0 +1,40 @@
+name: Docker Publish
+
+on:
+  push:
+    branches: [ main ]
+    tags:
+      - '*'
+
+jobs:
+  docker_publish:
+    # run workflow only on numaproj/numaflow-rs repository
+    if: ${{ github.repository }} == "numaproj/numaflow-rs"
+    name: Build, Tag, and Push Image
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        dockerfile_paths: [
+          "examples/map-cat", "examples/map-tickgen-serde", "examples/mapt-event-time-filter",
+          "examples/reduce-counter", "examples/sideinput", "examples/sideinput/udf",
+          "examples/simple-source", "examples/sink-log", "examples/source-transformer-now"
+        ]
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Quay.io registry
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.NUMAIO_USERNAME }}
+          password: ${{ secrets.NUMAIO_PASSWORD }}
+      - name: Build, tag, and push images
+        run: ./hack/update_examples.sh --build-push-example ${{ matrix.dockerfile_paths }}

examples/map-cat/Cargo.toml

@@ -11,4 +11,4 @@ path = "src/main.rs"
 [dependencies]
 tonic = "0.10.2"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
-numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" }
+numaflow = { path = "../../" }

examples/map-cat/Dockerfile

@@ -1,25 +1,20 @@
-FROM rust:1.75-bookworm as build
+FROM rust:1.75-bookworm AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
 
-# create a new empty shell project
-WORKDIR /examples
-
-# copy your source tree
-COPY src ./src
-
-COPY ./Cargo.toml ./Cargo.toml
-COPY ./Cargo.lock ./Cargo.lock
+WORKDIR /numaflow-rs
+COPY ./ ./
+WORKDIR /numaflow-rs/examples/map-cat
 
 # build for release
 RUN cargo build --release
 
 # our final base
-FROM debian:bookworm
+FROM debian:bookworm AS map-cat
 
 # copy the build artifact from the build stage
-COPY --from=build /examples/target/release/server .
+COPY --from=build /numaflow-rs/examples/map-cat/target/release/server .
 
 # set the startup command to run your binary
-CMD ["./server"]
+CMD ["./server"]

examples/map-cat/Makefile

@@ -0,0 +1,20 @@
+TAG ?= stable
+PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/map-cat:${TAG}
+DOCKER_FILE_PATH = examples/map-cat/Dockerfile
+
+.PHONY: update
+update:
+	cargo check
+	cargo update
+
+.PHONY: image
+image: update
+	cd ../../ && docker build \
+	-f ${DOCKER_FILE_PATH}  \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi
+
+.PHONY: clean
+clean:
+	-rm -rf target

examples/map-cat/manifests/simple-map-cat.yaml

@@ -17,7 +17,7 @@ spec:
         min: 1
       udf:
         container:
-          image: quay.io/numaio/numaflow-rs/map-cat:v1
+          image: quay.io/numaio/numaflow-rs/map-cat:stable
     - name: out
       sink:
         # A simple log printing sink

examples/map-cat/src/main.rs

@@ -1,7 +1,7 @@
 use numaflow::map;
 
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
+async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     map::Server::new(Cat).start().await
 }
 

examples/map-tickgen-serde/Cargo.toml

@@ -13,4 +13,4 @@ tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
 serde = { version = "1.0.103", features = ["derive"] }
 serde_json = "1.0.103"
 chrono = "0.4.26"
-numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" }
+numaflow = { path = "../../" }

examples/map-tickgen-serde/Dockerfile

@@ -1,25 +1,20 @@
-FROM rust:1.75-bookworm as build
+FROM rust:1.75-bookworm AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
 
-# create a new empty shell project
-WORKDIR /examples
-
-# copy your source tree
-COPY src ./src
-
-COPY ./Cargo.toml ./Cargo.toml
-COPY ./Cargo.lock ./Cargo.lock
+WORKDIR /numaflow-rs
+COPY ./ ./
+WORKDIR /numaflow-rs/examples/map-tickgen-serde
 
 # build for release
 RUN cargo build --release
 
 # our final base
-FROM debian:bookworm
+FROM debian:bookworm AS map-tickgen-serde
 
 # copy the build artifact from the build stage
-COPY --from=build /examples/target/release/server .
+COPY --from=build /numaflow-rs/examples/map-tickgen-serde/target/release/server .
 
 # set the startup command to run your binary
 CMD ["./server"]

examples/map-tickgen-serde/Makefile

@@ -0,0 +1,20 @@
+TAG ?= stable
+PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/map-tickgen-serde:${TAG}
+DOCKER_FILE_PATH = examples/map-tickgen-serde/Dockerfile
+
+.PHONY: update
+update:
+	cargo check
+	cargo update
+
+.PHONY: image
+image: update
+	cd ../../ && docker build \
+	-f ${DOCKER_FILE_PATH} \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi
+
+.PHONY: clean
+clean:
+	-rm -rf target

examples/map-tickgen-serde/manifests/simple-map-udf.yaml

@@ -17,7 +17,7 @@ spec:
         min: 1
       udf:
         container:
-          image: quay.io/numaio/numaflow-rs/map-tickgen-serde:v1
+          image: quay.io/numaio/numaflow-rs/map-tickgen-serde:stable
     - name: out
       sink:
         # A simple log printing sink

examples/mapt-event-time-filter/Dockerfile

@@ -0,0 +1,20 @@
+FROM rust:1.75-bookworm AS build
+
+RUN apt-get update
+RUN apt-get install protobuf-compiler -y
+
+WORKDIR /numaflow-rs
+COPY ./ ./
+WORKDIR /numaflow-rs/examples/mapt-event-time-filter
+
+# build for release
+RUN cargo build --release
+
+# our final base
+FROM debian:bookworm AS mapt-event-time-filter
+
+# copy the build artifact from the build stage
+COPY --from=build /numaflow-rs/examples/mapt-event-time-filter/target/release/server .
+
+# set the startup command to run your binary
+CMD ["./server"]

examples/mapt-event-time-filter/Makefile

@@ -0,0 +1,20 @@
+TAG ?= stable
+PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/mapt-event-time-filter:${TAG}
+DOCKER_FILE_PATH = examples/mapt-event-time-filter/Dockerfile
+
+.PHONY: update
+update:
+	cargo check
+	cargo update
+
+.PHONY: image
+image: update
+	cd ../../ && docker build \
+	-f ${DOCKER_FILE_PATH}  \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi
+
+.PHONY: clean
+clean:
+	-rm -rf target

examples/source-transformer-event-time-filter/manifests/event-time-filter.yaml → examples/mapt-event-time-filter/manifests/event-time-filter.yaml

@@ -13,7 +13,7 @@ spec:
           duration: 1s
         transformer:
           container:
-            image: quay.io/numaio/numaflow-rust/mapt-event-time-filter:stable
+            image: quay.io/numaio/numaflow-rs/mapt-event-time-filter:stable
             imagePullPolicy: Always
     - name: out
       scale:

examples/reduce-counter/Cargo.toml

@@ -10,4 +10,5 @@ path = "src/main.rs"
 [dependencies]
 tonic = "0.11.0"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
-numaflow-rs = { path = "../../" }
+numaflow = { path = "../../" }
+

examples/reduce-counter/Dockerfile

@@ -1,25 +1,20 @@
-FROM rust:1.70 as build
+FROM rust:1.70 AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
 
-# create a new empty shell project
-WORKDIR /examples
-
-# copy your source tree
-COPY src ./src
-
-COPY ./Cargo.toml ./Cargo.toml
-COPY ./Cargo.lock ./Cargo.lock
+WORKDIR /numaflow-rs
+COPY ./ ./
+WORKDIR /numaflow-rs/examples/reduce-counter
 
 # build for release
 RUN cargo build --release
 
 # our final base
-FROM rust
+FROM rust AS reduce-counter
 
 # copy the build artifact from the build stage
-COPY --from=build /examples/target/release/server .
+COPY --from=build /numaflow-rs/examples/reduce-counter/target/release/server .
 
 # set the startup command to run your binary
 CMD ["./server"]

examples/reduce-counter/Makefile

@@ -0,0 +1,20 @@
+TAG ?= stable
+PUSH ?= false
+IMAGE_REGISTRY = quay.io/numaio/numaflow-rs/reduce-counter:${TAG}
+DOCKER_FILE_PATH = examples/reduce-counter/Dockerfile
+
+.PHONY: update
+update:
+	cargo check
+	cargo update
+
+.PHONY: image
+image: update
+	cd ../../ && docker build \
+	-f ${DOCKER_FILE_PATH}  \
+	-t ${IMAGE_REGISTRY} .
+	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi
+
+.PHONY: clean
+clean:
+	-rm -rf target

examples/reduce-counter/manifests/simple-reduce.yaml

@@ -20,7 +20,7 @@ spec:
       udf:
         container:
           # count element
-          image: quay.io/numaio/numaflow-rs/reduce-counter:v1
+          image: quay.io/numaio/numaflow-rs/reduce-counter:stable
         groupBy:
           window:
             fixed:

examples/side-input/Cargo.toml → examples/sideinput/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "side-input"
+name = "sideinput"
 version = "0.1.0"
 edition = "2021"
 
@@ -11,5 +11,5 @@ path = "src/main.rs"
 [dependencies]
 tonic = "0.9"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
-numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", branch = "main" }
+numaflow = { path = "../../" }
 chrono = "0.4.30"