diff --git a/.dockerignore b/.dockerignore index 1f5f855..2f7896d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1 @@ target/ -**/target/ \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ba0cbe6..432e04d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,17 +27,6 @@ jobs: with: cache-workspaces: | . -> target - ./examples/batchmap-cat -> target - ./examples/batchmap-flatmap -> target - ./examples/map-cat -> target - ./examples/map-tickgen-serde -> target - ./examples/mapt-event-time-filter -> target - ./examples/reduce-counter -> target - ./examples/sideinput -> target - ./examples/sideinput/udf -> target - ./examples/simple-source -> target - ./examples/sink-log -> target - ./examples/source-transformer-now -> target - name: Configure sccache run: | @@ -65,5 +54,6 @@ jobs: - name: Documentation generation run: RUSTFLAGS="-D warnings" cargo doc + # To view the list of files included in the package: cargo package -p numaflow --list - name: Dry run Cargo Publish - run: cargo publish --dry-run + run: cargo publish -p numaflow --dry-run diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6e898b5..77295b5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -42,5 +42,6 @@ jobs: - name: Documentation generation run: RUSTFLAGS="-D warnings" cargo doc + # To view the list of files included in the package: cargo package -p numaflow --list - name: Dry run Cargo Publish - run: CARGO_REGISTRY_TOKEN=${{ secrets.CARGO_PUBLISH }} cargo publish + run: CARGO_REGISTRY_TOKEN=${{ secrets.CARGO_PUBLISH }} cargo publish -p numaflow diff --git a/Cargo.toml b/Cargo.toml index 3cb8285..7e2e63b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,48 +1,8 @@ -[package] -name = "numaflow" -version = "0.2.0" -edition = "2021" -rust-version = "1.80" -description = "Rust SDK for Numaflow" -authors = ["Numaflow Developers"] -license = "Apache-2.0" -readme = "README.md" -homepage = "https://numaproj.github.io/numaflow/" -repository = "https://github.com/numaproj/numaflow-rs" -keywords = ["numaflow", "streaming", "messaging", "event-driven"] -categories = ["network-programming", "api-bindings"] -exclude = [ - ".github/*", - ".gitignore", - ".dockerignore", - "hack/*", - "Makefile", -] - - -[lib] -name = "numaflow" -path = "src/lib.rs" - -[dependencies] -tonic = "0.12.3" -prost = "0.13.3" -prost-types = "0.13.3" -tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "signal"] } -tokio-util = "0.7.12" -tokio-stream = { version = "0.1.16", features = ["net"] } -serde = { version = "1.0.210", features = ["derive"] } -chrono = "0.4.38" -serde_json = "1.0.128" -futures-util = "0.3.30" -tracing = "0.1.40" -uuid = { version = "1.10.0", features = ["v4"] } -thiserror = "1.0" -hyper-util = "0.1.7" - -[build-dependencies] -tonic-build = "0.12.3" - -[dev-dependencies] -tempfile = "3.9.0" -tower = { version = "0.5.0", features = ["util"] } +[workspace] +members = ["numaflow", "examples/*"] +# Only check / build main crate by default (check all with `--workspace`) +default-members = ["numaflow"] +resolver = "2" + +[workspace.package] +rust-version = "1.82" diff --git a/Makefile b/Makefile index 71ea7e9..9a7f3c6 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,18 @@ -# Description: Makefile for Rust projects +.PHONY: lint fmt test-fmt clippy test codegen -# perform a cargo fmt on all directories containing a Cargo.toml file -.PHONY: lint -# find all directories containing Cargo.toml files -DIRS := $(shell find . -type f -name Cargo.toml -not -path "./target/*" -exec dirname {} \; | sort -u) -$(info Included directories: $(DIRS)) fmt: - @for dir in $(DIRS); do \ - echo "Formatting code in $$dir"; \ - cargo fmt --all --manifest-path "$$dir/Cargo.toml"; \ - done + cargo fmt --all -# Check if all files are formatted and run clippy on all directories containing a Cargo.toml file .PHONY: lint lint: test-fmt clippy .PHONY: test-fmt test-fmt: - @for dir in $(DIRS); do \ - echo "Checking if code is formatted in directory: $$dir"; \ - cargo fmt --all --check --manifest-path "$$dir/Cargo.toml" || { echo "Code is not formatted in $$dir"; exit 1; }; \ - done + cargo fmt --all --check .PHONY: clippy clippy: - @for dir in $(DIRS); do \ - echo "Running clippy in directory: $$dir"; \ - cargo clippy --workspace --manifest-path "$$dir/Cargo.toml" -- -D warnings || { echo "Clippy warnings/errors found in $$dir"; exit 1; }; \ - done + cargo clippy --workspace -- -D warnings # run cargo test on the repository root .PHONY: test @@ -37,5 +22,4 @@ test: .PHONY: codegen codegen: # Change timestamps so that tonic_build code generation will always be triggered. - touch proto/* - PROTO_CODE_GEN=1 cargo build + cd numaflow && mkdir -p src/servers && touch proto/* && PROTO_CODE_GEN=1 cargo build diff --git a/README.md b/README.md deleted file mode 100644 index f324d9a..0000000 --- a/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Rust SDK for Numaflow - -This SDK provides the interface for writing [User Defined Sources](https://numaflow.numaproj.io/user-guide/sources/user-defined-sources/), -[User Defined Source Tranformers](https://numaflow.numaproj.io/user-guide/sources/transformer/), -[UDFs](https://numaflow.numaproj.io/user-guide/user-defined-functions/user-defined-functions/) -and [User Defined Sinks](https://numaflow.numaproj.io/user-guide/sinks/user-defined-sinks/) in [Rust](https://www.rust-lang.org/). - -> This rust crate is being actively developed and it supports -> most of the features. You may use this crate but there might be -> few minor changes in the upcoming releases. - -## Examples - -You may find examples in the [examples folder](./examples). \ No newline at end of file diff --git a/README.md b/README.md new file mode 120000 index 0000000..00d866d --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +numaflow/README.md \ No newline at end of file diff --git a/examples/batchmap-cat/.dockerignore b/examples/batchmap-cat/.dockerignore deleted file mode 100644 index 9f97022..0000000 --- a/examples/batchmap-cat/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ \ No newline at end of file diff --git a/examples/batchmap-cat/Cargo.toml b/examples/batchmap-cat/Cargo.toml index 781f868..1fcfff0 100644 --- a/examples/batchmap-cat/Cargo.toml +++ b/examples/batchmap-cat/Cargo.toml @@ -3,12 +3,7 @@ name = "batchmap-cat" version = "0.1.0" edition = "2021" - -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } diff --git a/examples/batchmap-cat/Dockerfile b/examples/batchmap-cat/Dockerfile index 41a64f1..df063ac 100644 --- a/examples/batchmap-cat/Dockerfile +++ b/examples/batchmap-cat/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/batchmap-cat RUN cargo build --release # our final base -FROM debian:bookworm AS batchmap-cat +FROM debian:bullseye AS batchmap-cat # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/batchmap-cat/target/release/server . +COPY --from=build /numaflow-rs/target/release/batchmap-cat . # set the startup command to run your binary -CMD ["./server"] +CMD ["./batchmap-cat"] diff --git a/examples/batchmap-cat/Makefile b/examples/batchmap-cat/Makefile index 8f9436a..97b1c92 100644 --- a/examples/batchmap-cat/Makefile +++ b/examples/batchmap-cat/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target diff --git a/examples/batchmap-flatmap/.dockerignore b/examples/batchmap-flatmap/.dockerignore deleted file mode 100644 index 9f97022..0000000 --- a/examples/batchmap-flatmap/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ \ No newline at end of file diff --git a/examples/batchmap-flatmap/Cargo.toml b/examples/batchmap-flatmap/Cargo.toml index 746364f..753d20f 100644 --- a/examples/batchmap-flatmap/Cargo.toml +++ b/examples/batchmap-flatmap/Cargo.toml @@ -11,4 +11,4 @@ path = "src/main.rs" [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } diff --git a/examples/batchmap-flatmap/Dockerfile b/examples/batchmap-flatmap/Dockerfile index 07b1c96..01b6afa 100644 --- a/examples/batchmap-flatmap/Dockerfile +++ b/examples/batchmap-flatmap/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/batchmap-flatmap RUN cargo build --release # our final base -FROM debian:bookworm AS batchmap-flatmap +FROM debian:bullseye AS batchmap-flatmap # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/batchmap-flatmap/target/release/server . +COPY --from=build /numaflow-rs/target/release/batchmap-flatmap . # set the startup command to run your binary -CMD ["./server"] +CMD ["./batchmap-flatmap"] diff --git a/examples/batchmap-flatmap/Makefile b/examples/batchmap-flatmap/Makefile index 14eed90..a740b8a 100644 --- a/examples/batchmap-flatmap/Makefile +++ b/examples/batchmap-flatmap/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target diff --git a/examples/map-cat/.dockerignore b/examples/map-cat/.dockerignore deleted file mode 100644 index 9f97022..0000000 --- a/examples/map-cat/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ \ No newline at end of file diff --git a/examples/map-cat/Cargo.toml b/examples/map-cat/Cargo.toml index 5fb39a3..4db0404 100644 --- a/examples/map-cat/Cargo.toml +++ b/examples/map-cat/Cargo.toml @@ -3,12 +3,7 @@ name = "map-cat" version = "0.1.0" edition = "2021" - -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } diff --git a/examples/map-cat/Dockerfile b/examples/map-cat/Dockerfile index 22307bd..277ae2f 100644 --- a/examples/map-cat/Dockerfile +++ b/examples/map-cat/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/map-cat RUN cargo build --release # our final base -FROM debian:bookworm AS map-cat +FROM debian:bullseye AS map-cat # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/map-cat/target/release/server . +COPY --from=build /numaflow-rs/target/release/map-cat . # set the startup command to run your binary -CMD ["./server"] +CMD ["./map-cat"] diff --git a/examples/map-tickgen-serde/.dockerignore b/examples/map-tickgen-serde/.dockerignore deleted file mode 100644 index 9f97022..0000000 --- a/examples/map-tickgen-serde/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ \ No newline at end of file diff --git a/examples/map-tickgen-serde/Cargo.toml b/examples/map-tickgen-serde/Cargo.toml index 6bf3088..08abf16 100644 --- a/examples/map-tickgen-serde/Cargo.toml +++ b/examples/map-tickgen-serde/Cargo.toml @@ -3,14 +3,10 @@ name = "map-tickgen-serde" version = "0.1.0" edition = "2021" -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } serde = { version = "1.0.103", features = ["derive"] } serde_json = "1.0.103" chrono = "0.4.26" -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } diff --git a/examples/map-tickgen-serde/Dockerfile b/examples/map-tickgen-serde/Dockerfile index 05d8cd1..cdcfd67 100644 --- a/examples/map-tickgen-serde/Dockerfile +++ b/examples/map-tickgen-serde/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/map-tickgen-serde RUN cargo build --release # our final base -FROM debian:bookworm AS map-tickgen-serde +FROM debian:bullseye AS map-tickgen-serde # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/map-tickgen-serde/target/release/server . +COPY --from=build /numaflow-rs/target/release/map-tickgen-serde . # set the startup command to run your binary -CMD ["./server"] +CMD ["./map-tickgen-serde"] diff --git a/examples/map-tickgen-serde/Makefile b/examples/map-tickgen-serde/Makefile index b980eda..acca073 100644 --- a/examples/map-tickgen-serde/Makefile +++ b/examples/map-tickgen-serde/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target diff --git a/examples/mapt-event-time-filter/Cargo.toml b/examples/mapt-event-time-filter/Cargo.toml index c37662d..87af6ec 100644 --- a/examples/mapt-event-time-filter/Cargo.toml +++ b/examples/mapt-event-time-filter/Cargo.toml @@ -3,12 +3,8 @@ name = "source-transformer-event-time-filter" version = "0.1.0" edition = "2021" -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } tokio = "1.38.0" chrono = "0.4.38" -tonic = "0.12.0" \ No newline at end of file +tonic = "0.12.0" diff --git a/examples/mapt-event-time-filter/Dockerfile b/examples/mapt-event-time-filter/Dockerfile index bafa9e5..a5f513c 100644 --- a/examples/mapt-event-time-filter/Dockerfile +++ b/examples/mapt-event-time-filter/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/mapt-event-time-filter RUN cargo build --release # our final base -FROM debian:bookworm AS mapt-event-time-filter +FROM debian:bullseye AS mapt-event-time-filter # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/mapt-event-time-filter/target/release/server . +COPY --from=build /numaflow-rs/target/release/mapt-event-time-filter . # set the startup command to run your binary -CMD ["./server"] +CMD ["./mapt-event-time-filter"] diff --git a/examples/mapt-event-time-filter/src/main.rs b/examples/mapt-event-time-filter/src/main.rs index 1144472..6e3cef0 100644 --- a/examples/mapt-event-time-filter/src/main.rs +++ b/examples/mapt-event-time-filter/src/main.rs @@ -43,7 +43,7 @@ mod filter_impl { mod tests { use crate::filter_impl::filter_event_time; use chrono::{TimeZone, Utc}; - use numaflow::sourcetransform::{Message, SourceTransformRequest}; + use numaflow::sourcetransform::SourceTransformRequest; /// Tests that events from 2022 are tagged as within the year 2022. #[test] fn test_filter_event_time_should_return_within_year_2022() { diff --git a/examples/reduce-counter/Cargo.toml b/examples/reduce-counter/Cargo.toml index a03f53b..393b172 100644 --- a/examples/reduce-counter/Cargo.toml +++ b/examples/reduce-counter/Cargo.toml @@ -3,12 +3,7 @@ name = "reduce-counter" version = "0.1.0" edition = "2021" -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } - +numaflow = { path = "../../numaflow" } diff --git a/examples/reduce-counter/Dockerfile b/examples/reduce-counter/Dockerfile index ee3768f..d48fb13 100644 --- a/examples/reduce-counter/Dockerfile +++ b/examples/reduce-counter/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80 AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/reduce-counter RUN cargo build --release # our final base -FROM rust AS reduce-counter +FROM debian:bullseye AS reduce-counter # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/reduce-counter/target/release/server . +COPY --from=build /numaflow-rs/target/release/reduce-counter . # set the startup command to run your binary -CMD ["./server"] +CMD ["./reduce-counter"] diff --git a/examples/reduce-counter/Makefile b/examples/reduce-counter/Makefile index 14c165a..3d90d23 100644 --- a/examples/reduce-counter/Makefile +++ b/examples/reduce-counter/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target diff --git a/examples/sideinput/Cargo.toml b/examples/sideinput/Cargo.toml index e206480..f587f9d 100644 --- a/examples/sideinput/Cargo.toml +++ b/examples/sideinput/Cargo.toml @@ -3,13 +3,8 @@ name = "sideinput" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } -chrono = "0.4.30" \ No newline at end of file +numaflow = { path = "../../numaflow" } +chrono = "0.4.30" diff --git a/examples/sideinput/Dockerfile b/examples/sideinput/Dockerfile index 99ed118..898bc32 100644 --- a/examples/sideinput/Dockerfile +++ b/examples/sideinput/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80 AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -14,7 +14,7 @@ RUN cargo build --release FROM rust AS sideinput # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/sideinput/target/release/server . +COPY --from=build /numaflow-rs/target/release/sideinput . # set the startup command to run your binary -CMD ["./server"] +CMD ["./sideinput"] diff --git a/examples/sideinput/Makefile b/examples/sideinput/Makefile index 11734fc..4db679f 100644 --- a/examples/sideinput/Makefile +++ b/examples/sideinput/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target \ No newline at end of file diff --git a/examples/simple-source/.dockerignore b/examples/simple-source/.dockerignore deleted file mode 100644 index 9f97022..0000000 --- a/examples/simple-source/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ \ No newline at end of file diff --git a/examples/simple-source/Cargo.toml b/examples/simple-source/Cargo.toml index 2d00975..692e079 100644 --- a/examples/simple-source/Cargo.toml +++ b/examples/simple-source/Cargo.toml @@ -3,13 +3,9 @@ name = "simple-source" version = "0.1.0" edition = "2021" -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } chrono = "0.4.38" -uuid = "1.2.0" \ No newline at end of file +uuid = "1.2.0" diff --git a/examples/simple-source/Dockerfile b/examples/simple-source/Dockerfile index c6ee8c0..06a2bc8 100644 --- a/examples/simple-source/Dockerfile +++ b/examples/simple-source/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/simple-source RUN cargo build --release # our final base -FROM debian:bookworm AS simple-source +FROM debian:bullseye AS simple-source # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/simple-source/target/release/server . +COPY --from=build /numaflow-rs/target/release/simple-source . # set the startup command to run your binary -CMD ["./server"] +CMD ["./simple-source"] diff --git a/examples/simple-source/Makefile b/examples/simple-source/Makefile index b7769e8..827767a 100644 --- a/examples/simple-source/Makefile +++ b/examples/simple-source/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target diff --git a/examples/sink-log/.dockerignore b/examples/sink-log/.dockerignore deleted file mode 100644 index 9f97022..0000000 --- a/examples/sink-log/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ \ No newline at end of file diff --git a/examples/sink-log/Cargo.toml b/examples/sink-log/Cargo.toml index eddcc83..6c2e002 100644 --- a/examples/sink-log/Cargo.toml +++ b/examples/sink-log/Cargo.toml @@ -10,4 +10,4 @@ path = "src/main.rs" [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } diff --git a/examples/sink-log/Dockerfile b/examples/sink-log/Dockerfile index 334b445..cb183c7 100644 --- a/examples/sink-log/Dockerfile +++ b/examples/sink-log/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/sink-log RUN cargo build --release # our final base -FROM debian:bookworm AS sink-log +FROM debian:bullseye AS sink-log # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/sink-log/target/release/server . +COPY --from=build /numaflow-rs/target/release/sink-log . # set the startup command to run your binary -CMD ["./server"] +CMD ["./sink-log"] diff --git a/examples/sink-log/Makefile b/examples/sink-log/Makefile index 5db536d..2e06fbf 100644 --- a/examples/sink-log/Makefile +++ b/examples/sink-log/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target diff --git a/examples/source-transformer-now/.dockerignore b/examples/source-transformer-now/.dockerignore deleted file mode 100644 index 9f97022..0000000 --- a/examples/source-transformer-now/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ \ No newline at end of file diff --git a/examples/source-transformer-now/Cargo.toml b/examples/source-transformer-now/Cargo.toml index 5e570b9..668444f 100644 --- a/examples/source-transformer-now/Cargo.toml +++ b/examples/source-transformer-now/Cargo.toml @@ -3,12 +3,8 @@ name = "source-transformer-now" version = "0.1.0" edition = "2021" -[[bin]] -name = "server" -path = "src/main.rs" - [dependencies] tonic = "0.12.0" tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -numaflow = { path = "../../" } +numaflow = { path = "../../numaflow" } chrono = "0.4.30" diff --git a/examples/source-transformer-now/Dockerfile b/examples/source-transformer-now/Dockerfile index c1a0d0c..4f14635 100644 --- a/examples/source-transformer-now/Dockerfile +++ b/examples/source-transformer-now/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80-bookworm AS build +FROM rust:1.82-bullseye AS build RUN apt-get update RUN apt-get install protobuf-compiler -y @@ -11,11 +11,10 @@ WORKDIR /numaflow-rs/examples/source-transformer-now RUN cargo build --release # our final base -FROM debian:bookworm AS source-transformer-now +FROM debian:bullseye AS source-transformer-now # copy the build artifact from the build stage -COPY --from=build /numaflow-rs/examples/source-transformer-now/target/release/server . +COPY --from=build /numaflow-rs/target/release/source-transformer-now . # set the startup command to run your binary -CMD ["./server"] - +CMD ["./source-transformer-now"] diff --git a/examples/source-transformer-now/Makefile b/examples/source-transformer-now/Makefile index ffdefcd..d45f083 100644 --- a/examples/source-transformer-now/Makefile +++ b/examples/source-transformer-now/Makefile @@ -14,7 +14,3 @@ image: update -f ${DOCKER_FILE_PATH} \ -t ${IMAGE_REGISTRY} . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi - -.PHONY: clean -clean: - -rm -rf target diff --git a/hack/update_examples.sh b/hack/update_examples.sh index 27ce941..63016de 100755 --- a/hack/update_examples.sh +++ b/hack/update_examples.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -euo pipefail function show_help () { echo "Usage: $0 [-h|--help | -t|--tag ] (-bpe|--build-push-example )" diff --git a/numaflow/Cargo.toml b/numaflow/Cargo.toml new file mode 100644 index 0000000..3cb8285 --- /dev/null +++ b/numaflow/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "numaflow" +version = "0.2.0" +edition = "2021" +rust-version = "1.80" +description = "Rust SDK for Numaflow" +authors = ["Numaflow Developers"] +license = "Apache-2.0" +readme = "README.md" +homepage = "https://numaproj.github.io/numaflow/" +repository = "https://github.com/numaproj/numaflow-rs" +keywords = ["numaflow", "streaming", "messaging", "event-driven"] +categories = ["network-programming", "api-bindings"] +exclude = [ + ".github/*", + ".gitignore", + ".dockerignore", + "hack/*", + "Makefile", +] + + +[lib] +name = "numaflow" +path = "src/lib.rs" + +[dependencies] +tonic = "0.12.3" +prost = "0.13.3" +prost-types = "0.13.3" +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "signal"] } +tokio-util = "0.7.12" +tokio-stream = { version = "0.1.16", features = ["net"] } +serde = { version = "1.0.210", features = ["derive"] } +chrono = "0.4.38" +serde_json = "1.0.128" +futures-util = "0.3.30" +tracing = "0.1.40" +uuid = { version = "1.10.0", features = ["v4"] } +thiserror = "1.0" +hyper-util = "0.1.7" + +[build-dependencies] +tonic-build = "0.12.3" + +[dev-dependencies] +tempfile = "3.9.0" +tower = { version = "0.5.0", features = ["util"] } diff --git a/numaflow/README.md b/numaflow/README.md new file mode 100644 index 0000000..f324d9a --- /dev/null +++ b/numaflow/README.md @@ -0,0 +1,14 @@ +# Rust SDK for Numaflow + +This SDK provides the interface for writing [User Defined Sources](https://numaflow.numaproj.io/user-guide/sources/user-defined-sources/), +[User Defined Source Tranformers](https://numaflow.numaproj.io/user-guide/sources/transformer/), +[UDFs](https://numaflow.numaproj.io/user-guide/user-defined-functions/user-defined-functions/) +and [User Defined Sinks](https://numaflow.numaproj.io/user-guide/sinks/user-defined-sinks/) in [Rust](https://www.rust-lang.org/). + +> This rust crate is being actively developed and it supports +> most of the features. You may use this crate but there might be +> few minor changes in the upcoming releases. + +## Examples + +You may find examples in the [examples folder](./examples). \ No newline at end of file diff --git a/build.rs b/numaflow/build.rs similarity index 100% rename from build.rs rename to numaflow/build.rs diff --git a/proto/map.proto b/numaflow/proto/map.proto similarity index 100% rename from proto/map.proto rename to numaflow/proto/map.proto diff --git a/proto/reduce.proto b/numaflow/proto/reduce.proto similarity index 100% rename from proto/reduce.proto rename to numaflow/proto/reduce.proto diff --git a/proto/sideinput.proto b/numaflow/proto/sideinput.proto similarity index 100% rename from proto/sideinput.proto rename to numaflow/proto/sideinput.proto diff --git a/proto/sink.proto b/numaflow/proto/sink.proto similarity index 100% rename from proto/sink.proto rename to numaflow/proto/sink.proto diff --git a/proto/source.proto b/numaflow/proto/source.proto similarity index 100% rename from proto/source.proto rename to numaflow/proto/source.proto diff --git a/proto/sourcetransform.proto b/numaflow/proto/sourcetransform.proto similarity index 100% rename from proto/sourcetransform.proto rename to numaflow/proto/sourcetransform.proto diff --git a/src/batchmap.rs b/numaflow/src/batchmap.rs similarity index 100% rename from src/batchmap.rs rename to numaflow/src/batchmap.rs diff --git a/src/error.rs b/numaflow/src/error.rs similarity index 100% rename from src/error.rs rename to numaflow/src/error.rs diff --git a/src/lib.rs b/numaflow/src/lib.rs similarity index 100% rename from src/lib.rs rename to numaflow/src/lib.rs diff --git a/src/map.rs b/numaflow/src/map.rs similarity index 100% rename from src/map.rs rename to numaflow/src/map.rs diff --git a/src/reduce.rs b/numaflow/src/reduce.rs similarity index 100% rename from src/reduce.rs rename to numaflow/src/reduce.rs diff --git a/src/servers.rs b/numaflow/src/servers.rs similarity index 100% rename from src/servers.rs rename to numaflow/src/servers.rs diff --git a/src/servers/map.v1.rs b/numaflow/src/servers/map.v1.rs similarity index 100% rename from src/servers/map.v1.rs rename to numaflow/src/servers/map.v1.rs diff --git a/src/servers/reduce.v1.rs b/numaflow/src/servers/reduce.v1.rs similarity index 100% rename from src/servers/reduce.v1.rs rename to numaflow/src/servers/reduce.v1.rs diff --git a/src/servers/sideinput.v1.rs b/numaflow/src/servers/sideinput.v1.rs similarity index 100% rename from src/servers/sideinput.v1.rs rename to numaflow/src/servers/sideinput.v1.rs diff --git a/src/servers/sink.v1.rs b/numaflow/src/servers/sink.v1.rs similarity index 100% rename from src/servers/sink.v1.rs rename to numaflow/src/servers/sink.v1.rs diff --git a/src/servers/source.v1.rs b/numaflow/src/servers/source.v1.rs similarity index 100% rename from src/servers/source.v1.rs rename to numaflow/src/servers/source.v1.rs diff --git a/src/servers/sourcetransformer.v1.rs b/numaflow/src/servers/sourcetransformer.v1.rs similarity index 100% rename from src/servers/sourcetransformer.v1.rs rename to numaflow/src/servers/sourcetransformer.v1.rs diff --git a/src/shared.rs b/numaflow/src/shared.rs similarity index 100% rename from src/shared.rs rename to numaflow/src/shared.rs diff --git a/src/sideinput.rs b/numaflow/src/sideinput.rs similarity index 100% rename from src/sideinput.rs rename to numaflow/src/sideinput.rs diff --git a/src/sink.rs b/numaflow/src/sink.rs similarity index 100% rename from src/sink.rs rename to numaflow/src/sink.rs diff --git a/src/source.rs b/numaflow/src/source.rs similarity index 100% rename from src/source.rs rename to numaflow/src/source.rs diff --git a/src/sourcetransform.rs b/numaflow/src/sourcetransform.rs similarity index 100% rename from src/sourcetransform.rs rename to numaflow/src/sourcetransform.rs