Merge changes from main

Signed-off-by: Sreekanth <[email protected]>
numaproj · Jan 15, 2025 · 4e04250 · 4e04250
2 parents 011a649 + 376a602
commit 4e04250
Show file tree

Hide file tree

Showing 64 changed files with 3,268 additions and 803 deletions.
diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml
@@ -35,7 +35,7 @@ jobs:
       - name: Make checksums
         run: make checksums
       - name: store artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: binaries
           path: dist
@@ -65,7 +65,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: rust/numaflow-rs-linux-amd64
@@ -98,7 +98,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: rust/numaflow-rs-linux-arm64
@@ -127,19 +127,19 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Download Go binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: binaries
           path: dist/
 
       - name: Download Rust amd64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: dist/
 
       - name: Download Rust arm64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: dist/

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Make checksums
         run: make checksums
       - name: store artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: binaries
           path: dist
@@ -56,7 +56,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: rust/numaflow-rs-linux-amd64
@@ -82,7 +82,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: rust/numaflow-rs-linux-arm64
@@ -111,19 +111,19 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Download Go binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: binaries
           path: dist/
 
       - name: Download Rust amd64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: dist/
 
       - name: Download Rust arm64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: dist/
@@ -171,7 +171,7 @@ jobs:
       - run: bom generate --image quay.io/numaproj/numaflow:$VERSION -o /tmp/numaflow.spdx
       # pack the boms into one file to make it easy to download
       - run: cd /tmp && tar -zcf sbom.tar.gz *.spdx
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: sbom.tar.gz
           path: /tmp/sbom.tar.gz
@@ -191,11 +191,11 @@ jobs:
             echo "VERSION=${GITHUB_REF##*/}" >> $GITHUB_ENV
           fi
       - name: Download binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: binaries
           path: dist/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: sbom.tar.gz
           path: /tmp

diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json
@@ -21790,6 +21790,10 @@
           "description": "SASL mechanism to use",
           "type": "string"
         },
+        "oauth": {
+          "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.SASLOAuth",
+          "description": "OAuth contains the oauth config"
+        },
         "plain": {
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.SASLPlain",
           "description": "SASLPlain contains the sasl plain config"
@@ -21808,6 +21812,28 @@
       ],
       "type": "object"
     },
+    "io.numaproj.numaflow.v1alpha1.SASLOAuth": {
+      "properties": {
+        "clientID": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector",
+          "description": "ClientID refers to the secret that contains the client id"
+        },
+        "clientSecret": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector",
+          "description": "ClientSecret refers to the secret that contains the client secret"
+        },
+        "tokenEndpoint": {
+          "description": "TokenEndpoint refers to the token endpoint",
+          "type": "string"
+        }
+      },
+      "required": [
+        "clientID",
+        "clientSecret",
+        "tokenEndpoint"
+      ],
+      "type": "object"
+    },
     "io.numaproj.numaflow.v1alpha1.SASLPlain": {
       "properties": {
         "handshake": {

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -21780,6 +21780,10 @@
           "description": "SASL mechanism to use",
           "type": "string"
         },
+        "oauth": {
+          "description": "OAuth contains the oauth config",
+          "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.SASLOAuth"
+        },
         "plain": {
           "description": "SASLPlain contains the sasl plain config",
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.SASLPlain"
@@ -21794,6 +21798,28 @@
         }
       }
     },
+    "io.numaproj.numaflow.v1alpha1.SASLOAuth": {
+      "type": "object",
+      "required": [
+        "clientID",
+        "clientSecret",
+        "tokenEndpoint"
+      ],
+      "properties": {
+        "clientID": {
+          "description": "ClientID refers to the secret that contains the client id",
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
+        },
+        "clientSecret": {
+          "description": "ClientSecret refers to the secret that contains the client secret",
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
+        },
+        "tokenEndpoint": {
+          "description": "TokenEndpoint refers to the token endpoint",
+          "type": "string"
+        }
+      }
+    },
     "io.numaproj.numaflow.v1alpha1.SASLPlain": {
       "type": "object",
       "required": [

diff --git a/config/advanced-install/namespaced-numaflow-server.yaml b/config/advanced-install/namespaced-numaflow-server.yaml
@@ -156,6 +156,9 @@ data:
           required: false
       metrics:
         - metric_name: vertex_pending_messages
+          display_name: Vertex Pending Messages
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - pipeline
@@ -189,6 +192,9 @@ data:
           required: false
       metrics:
         - metric_name: monovtx_pending
+          display_name: Mono Vertex Pending Messages
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - mvtx_name
@@ -225,6 +231,10 @@ data:
           required: false
       metrics:
         - metric_name: monovtx_processing_time_bucket
+          display_name: Mono Vertex Processing Time Latency
+          # set "Units" or unset for default behaviour otherwise set "s" or "ms" for latency metrics
+          # Note: latency values are in μs
+          # unit: s
           required_filters:
             - namespace
             - mvtx_name
@@ -235,6 +245,10 @@ data:
                 - name: pod
                   required: false
         - metric_name: monovtx_sink_time_bucket
+          display_name: Mono Vertex Sink Write Time Latency
+          # set "Units" or unset for default behaviour otherwise set "s" or "ms" for latency metrics
+          # Note: latency values are in μs
+          # unit: ms
           required_filters:
             - namespace
             - mvtx_name
@@ -259,6 +273,9 @@ data:
           required: false
       metrics:
         - metric_name: forwarder_data_read_total
+          display_name: Vertex Read Processing Rate
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - pipeline
@@ -284,6 +301,9 @@ data:
           required: false
       metrics:
         - metric_name: monovtx_read_total
+          display_name: Mono Vertex Read Processing Rate
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - mvtx_name

diff --git a/config/advanced-install/numaflow-server.yaml b/config/advanced-install/numaflow-server.yaml
@@ -163,6 +163,9 @@ data:
           required: false
       metrics:
         - metric_name: vertex_pending_messages
+          display_name: Vertex Pending Messages
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - pipeline
@@ -196,6 +199,9 @@ data:
           required: false
       metrics:
         - metric_name: monovtx_pending
+          display_name: Mono Vertex Pending Messages
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - mvtx_name
@@ -232,6 +238,10 @@ data:
           required: false
       metrics:
         - metric_name: monovtx_processing_time_bucket
+          display_name: Mono Vertex Processing Time Latency
+          # set "Units" or unset for default behaviour otherwise set "s" or "ms" for latency metrics
+          # Note: latency values are in μs
+          # unit: s
           required_filters:
             - namespace
             - mvtx_name
@@ -242,6 +252,10 @@ data:
                 - name: pod
                   required: false
         - metric_name: monovtx_sink_time_bucket
+          display_name: Mono Vertex Sink Write Time Latency
+          # set "Units" or unset for default behaviour otherwise set "s" or "ms" for latency metrics
+          # Note: latency values are in μs
+          # unit: ms
           required_filters:
             - namespace
             - mvtx_name
@@ -266,6 +280,9 @@ data:
           required: false
       metrics:
         - metric_name: forwarder_data_read_total
+          display_name: Vertex Read Processing Rate
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - pipeline
@@ -291,6 +308,9 @@ data:
           required: false
       metrics:
         - metric_name: monovtx_read_total
+          display_name: Mono Vertex Read Processing Rate
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - mvtx_name