refactor: generate static gRPC clients

.codecov.yml

@@ -10,6 +10,7 @@ ignore:
 - "test/.*"
 - "rust/**/error.rs"
 - "rust/numaflow-models/**" # ignore generated files
+- "rust/numaflow-grpc/**" # ignore generated files
 coverage:
   status:
     patch: off

Dockerfile

@@ -23,7 +23,6 @@ ARG TARGETPLATFORM
 WORKDIR /numaflow
 RUN apt-get update && apt-get install -y protobuf-compiler
 
-
 FROM chef AS planner
 COPY ./rust/ .
 RUN cargo chef prepare --recipe-path recipe.json
@@ -90,4 +89,4 @@ RUN chmod +x /bin/e2eapi
 ####################################################################################################
 FROM scratch AS e2eapi
 COPY --from=testbase /bin/e2eapi .
-ENTRYPOINT ["/e2eapi"]
+ENTRYPOINT ["/e2eapi"]

Makefile

@@ -213,6 +213,10 @@ swagger:
 api/json-schema/schema.json: api/openapi-spec/swagger.json hack/json-schema/main.go
 	go run ./hack/json-schema
 
+.PHONY: rustgen
+rustgen:
+	$(MAKE) --directory rust generate
+
 .PHONY: codegen
 codegen:
 	./hack/generate-proto.sh
@@ -223,7 +227,7 @@ codegen:
 	$(MAKE) manifests
 	rm -rf ./vendor
 	go mod tidy
-	$(MAKE) --directory rust/numaflow-models generate
+	$(MAKE) rustgen
 
 clean:
 	-rm -rf ${CURRENT_DIR}/dist

hack/generate-proto.sh

@@ -14,33 +14,7 @@ export GOPATH="${FAKE_GOPATH}"
 export PATH="${GOPATH}/bin:${PATH}"
 cd "${FAKE_REPOPATH}"
 
-install-protobuf() {
-  # protobuf version
-  PROTOBUF_VERSION=27.2
-  PB_REL="https://github.com/protocolbuffers/protobuf/releases"
-  OS=$(uname_os)
-  ARCH=$(uname_arch)
-
-  echo "OS: $OS  ARCH: $ARCH"
-  if [[ "$ARCH" = "amd64" ]]; then
-    ARCH="x86_64"
-  elif [[ "$ARCH" = "arm64" ]]; then
-    ARCH="aarch_64"
-  fi
-  BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip
-  if [[ "$OS" = "darwin" ]]; then
-    BINARY_URL=$PB_REL/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-osx-universal_binary.zip
-  fi
-  echo "Downloading $BINARY_URL"
-
-  tmp=$(mktemp -d)
-  trap 'rm -rf ${tmp}' EXIT
-
-  curl -sL -o ${tmp}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip $BINARY_URL
-  unzip ${tmp}/protoc-${PROTOBUF_VERSION}-${OS}-${ARCH}.zip -d ${GOPATH}
-}
-
-install-protobuf
+install-protobuf --install-dir ${GOPATH}
 
 go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogo
 go install -mod=vendor ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast

hack/library.sh

@@ -59,7 +59,7 @@ uname_os() {
     mingw*) os="windows" ;;
     cygwin*) os="windows" ;;
     win*) os="windows" ;;
-    sunos) [ "$(uname -o)" = "illumos" ] && os=illumos ;;
+    sunos) [[ "$(uname -o)" = "illumos" ]] && os=illumos ;;
   esac
   echo "$os"
 }
@@ -80,3 +80,61 @@ uname_arch() {
   esac
   echo "${arch}"
 }
+
+install-protobuf() {
+  local install_dir=""
+  while [[ "$#" -gt 0 ]]; do
+    case "$1" in
+      "--install-dir")
+        install_dir="$2"
+        shift 2
+        ;;
+      *)
+        if [[ "$1" =~ ^-- ]]; then
+          echo "unknown argument: $1" >&2
+          return 1
+        fi
+        if [ -n "$install_dir" ]; then
+          echo "too many arguments: $1 (already have $install_dir)" >&2
+          return 1
+        fi
+        install_dir="$1"
+        shift
+        ;;
+    esac
+  done
+
+  if [[ -z "${install_dir}" ]]; then
+    echo "install-dir argument is required" >&2
+    return 1
+  fi
+
+  if [[ ! -d "${install_dir}" ]]; then
+    echo "${install_dir} does not exist" >&2
+    return 1
+  fi
+
+  # protobuf version
+  local protobuf_version=27.2
+  local pb_rel="https://github.com/protocolbuffers/protobuf/releases"
+  local os=$(uname_os)
+  local arch=$(uname_arch)
+
+  echo "OS: $os  ARCH: $arch"
+  if [[ "$arch" = "amd64" ]]; then
+    arch="x86_64"
+  elif [[ "$arch" = "arm64" ]]; then
+    arch="aarch_64"
+  fi
+  local binary_url=${pb_rel}/download/v${protobuf_version}/protoc-${protobuf_version}-${os}-${arch}.zip
+  if [[ "$os" = "darwin" ]]; then
+    binary_url=${pb_rel}/download/v${protobuf_version}/protoc-${protobuf_version}-osx-universal_binary.zip
+  fi
+  echo "Downloading $binary_url"
+
+  tmp=$(mktemp -d)
+  curl -sL -o ${tmp}/protoc-${protobuf_version}-${os}-${arch}.zip $binary_url
+  unzip ${tmp}/protoc-${protobuf_version}-${os}-${arch}.zip -d ${install_dir}
+  rm -rf ${tmp}
+}
+

hack/update-api-docs.sh

@@ -24,57 +24,78 @@ go run ${FAKE_REPOPATH}/vendor/github.com/ahmetb/gen-crd-api-reference-docs/main
  -template-dir "${FAKE_REPOPATH}/hack/api-docs-template"
 
 install-pandoc() {
-  # pandoc version
-  PANDOC_VERSION=3.2.1
-
-  if [[ "`command -v pandoc`" != "" ]]; then
-    if [[ "`pandoc -v | head -1 | awk '{print $2}'`" != "${PANDOC_VERSION}" ]]; then
-      warning "Existing pandoc version does not match the requirement (${PANDOC_VERSION}), will download a new one..."
-    else
-      PANDOC_BINARY="`command -v pandoc`"
-      return 
-    fi
+  local install_dir=""
+  while [[ "$#" -gt 0 ]]; do
+    case "$1" in
+      "--install-dir")
+        install_dir="$2"
+        shift 2
+        ;;
+      *)
+        if [[ "$1" =~ ^-- ]]; then
+          echo "unknown argument: $1" >&2
+          return 1
+        fi
+        if [ -n "$install_dir" ]; then
+          echo "too many arguments: $1 (already have $install_dir)" >&2
+          return 1
+        fi
+        install_dir="$1"
+        shift
+        ;;
+    esac
+  done
+
+  if [[ -z "${install_dir}" ]]; then
+    echo "install-dir argument is required" >&2
+    return 1
   fi
 
-  PD_REL="https://github.com/jgm/pandoc/releases"
-  OS=$(uname_os)
-  ARCH=$(uname_arch)
-
-  echo "OS: $OS  ARCH: $ARCH"
+  if [[ ! -d "${install_dir}" ]]; then
+    echo "${install_dir} does not exist" >&2
+    return 1
+  fi
 
-  BINARY_NAME="pandoc-${PANDOC_VERSION}-${OS}-${ARCH}.zip"
-  if [[ "$OS" = "darwin" ]]; then
-    if [[ "$ARCH" = "arm64" ]]; then
-      BINARY_NAME="pandoc-${PANDOC_VERSION}-arm64-macOS.zip"
-    elif [[ "$ARCH" = "amd64" ]]; then
-      BINARY_NAME="pandoc-${PANDOC_VERSION}-x86_64-macOS.zip"
+  # pandoc version
+  local pandoc_version=3.2.1
+  local pd_rel="https://github.com/jgm/pandoc/releases"
+  local os=$(uname_os)
+  local arch=$(uname_arch)
+
+  echo "OS: $os  ARCH: $arch"
+
+  local binary_name="pandoc-${pandoc_version}-${os}-${arch}.zip"
+  if [[ "$os" = "darwin" ]]; then
+    if [[ "$arch" = "arm64" ]]; then
+      binary_name="pandoc-${pandoc_version}-arm64-macOS.zip"
+    elif [[ "$arch" = "amd64" ]]; then
+      binary_name="pandoc-${pandoc_version}-x86_64-macOS.zip"
     fi
-  elif [[ "$OS" = "linux" ]]; then
-    if [[ "$ARCH" = "arm64" ]]; then
-      BINARY_NAME="pandoc-${PANDOC_VERSION}-linux-arm64.tar.gz"
-    elif [[ "$ARCH" = "amd64" ]]; then
-      BINARY_NAME="pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz"
+  elif [[ "$os" = "linux" ]]; then
+    if [[ "$arch" = "arm64" ]]; then
+      binary_name="pandoc-${pandoc_version}-linux-arm64.tar.gz"
+    elif [[ "$arch" = "amd64" ]]; then
+      binary_name="pandoc-${pandoc_version}-linux-amd64.tar.gz"
     fi
   fi
-  BINARY_URL=$PD_REL/download/${PANDOC_VERSION}/${BINARY_NAME}
-  echo "Downloading $BINARY_URL"
-
-  tmp=$(mktemp -d)
-  trap 'rm -rf ${tmp}' EXIT
-
-  curl -sL -o ${tmp}/${BINARY_NAME} $BINARY_URL
-  if [[ "$BINARY_NAME" =~ .zip$ ]]; then
-    unzip ${tmp}/${BINARY_NAME} -d ${tmp}
-    for a in `ls -d -1 ${tmp}/* | grep pandoc | grep -v zip`; do mv $a/* ${tmp}; rmdir $a; done
-  elif [[ "$BINARY_NAME" =~ .tar.gz$ ]]; then
-    tar xvzf ${tmp}/${BINARY_NAME} --strip-components 1 -C ${tmp}/
+  local binary_url=$pd_rel/download/${pandoc_version}/${binary_name}
+  echo "Downloading $binary_url"
+
+  curl -sL -o ${tmp}/${binary_name} $binary_url
+  if [[ "$binary_name" =~ .zip$ ]]; then
+    unzip ${install_dir}/${binary_name} -d ${install_dir}
+    for a in `ls -d -1 ${install_dir}/* | grep pandoc | grep -v zip`; do mv $a/* ${install_dir}; rmdir $a; done
+  elif [[ "$binary_name" =~ .tar.gz$ ]]; then
+    tar xvzf ${install_dir}/${binary_name} --strip-components 1 -C ${install_dir}/
   fi
-  PANDOC_BINARY="${tmp}/bin/pandoc"
 }
 
-install-pandoc
+tmp=$(mktemp -d)
+install-pandoc ${tmp}
+PANDOC_BINARY="${tmp}/bin/pandoc"
 
 ${PANDOC_BINARY} --from markdown --to gfm ${FAKE_REPOPATH}/docs/APIs.html > ${FAKE_REPOPATH}/docs/APIs.md
 
+rm -rf ${tmp}
 rm ${FAKE_REPOPATH}/docs/APIs.html
 

rust/Cargo.toml

@@ -4,6 +4,7 @@ workspace = { members = [
     "servesink",
     "serving",
     "numaflow-core",
+    "numaflow-grpc",
 ] }
 
 [[bin]]
@@ -22,5 +23,6 @@ backoff = { path = "backoff" }
 servesink = { path = "servesink" }
 serving = { path = "serving" }
 numaflow-core = { path = "numaflow-core" }
+numaflow-grpc = { path = "numaflow-grpc" }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }

rust/Makefile

@@ -1,3 +1,10 @@
+SHELL:=/bin/bash
+
+.PHONY: generate
+generate:
+	$(MAKE) --directory numaflow-models generate
+	$(MAKE) --directory numaflow-grpc generate
+
 .PHONY: build
 build:
 	cargo build --release
@@ -12,4 +19,4 @@ all-tests:
 
 .PHONY: clean
 clean:
-	cargo clean
+	cargo clean

rust/numaflow-core/Cargo.toml

@@ -23,6 +23,7 @@ tower = "0.4.13"
 uuid = { version = "1.10.0", features = ["v4"] }
 serde_json = "1.0.122"
 numaflow-models = { path = "../numaflow-models" }
+numaflow-grpc = { path = "../numaflow-grpc" }
 trait-variant = "0.1.2"
 rcgen = "0.13.1"
 rustls = { version = "0.23.12", features = ["aws_lc_rs"] }
@@ -42,4 +43,4 @@ tempfile = "3.11.0"
 numaflow = { git = "https://github.com/numaproj/numaflow-rs.git", rev = "30d8ce1972fd3f0c0b8059fee209516afeef0088" }
 
 [build-dependencies]
-tonic-build = "0.12.1"
+