feat: rolling update for Pipeline Vertex (#2040)

Signed-off-by: Derek Wang <[email protected]>
numaproj · Sep 9, 2024 · c4b4d00 · c4b4d00
1 parent 2189ea9
commit c4b4d00
Show file tree

Hide file tree

Showing 29 changed files with 2,205 additions and 995 deletions.
diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json
@@ -17785,6 +17785,10 @@
         "udf": {
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UDF"
         },
+        "updateStrategy": {
+          "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UpdateStrategy",
+          "description": "The strategy to use to replace existing pods with new ones."
+        },
         "volumes": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.Volume"
@@ -20608,6 +20612,10 @@
         "udf": {
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UDF"
         },
+        "updateStrategy": {
+          "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UpdateStrategy",
+          "description": "The strategy to use to replace existing pods with new ones."
+        },
         "volumes": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.Volume"
@@ -20639,11 +20647,11 @@
           "x-kubernetes-patch-strategy": "merge"
         },
         "currentHash": {
-          "description": "If not empty, indicates the version of the Vertex used to generate Pods in the sequence [0,currentReplicas).",
+          "description": "If not empty, indicates the current version of the Vertex used to generate Pods.",
           "type": "string"
         },
-        "currentReplicas": {
-          "description": "The number of Pods created by the controller from the Vertex version indicated by currentHash.",
+        "desiredReplicas": {
+          "description": "The number of desired replicas.",
           "format": "int64",
           "type": "integer"
         },
@@ -20679,9 +20687,14 @@
           "type": "string"
         },
         "updateHash": {
-          "description": "If not empty, indicates the version of the Vertx used to generate Pods in the sequence [replicas-updatedReplicas,replicas)",
+          "description": "If not empty, indicates the updated version of the Vertex used to generate Pods.",
           "type": "string"
         },
+        "updatedReadyReplicas": {
+          "description": "The number of ready Pods created by the controller from the Vertex version indicated by updateHash.",
+          "format": "int64",
+          "type": "integer"
+        },
         "updatedReplicas": {
           "description": "The number of Pods created by the controller from the Vertex version indicated by updateHash.",
           "format": "int64",

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -17793,6 +17793,10 @@
         "udf": {
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UDF"
         },
+        "updateStrategy": {
+          "description": "The strategy to use to replace existing pods with new ones.",
+          "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UpdateStrategy"
+        },
         "volumes": {
           "type": "array",
           "items": {
@@ -20590,6 +20594,10 @@
         "udf": {
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UDF"
         },
+        "updateStrategy": {
+          "description": "The strategy to use to replace existing pods with new ones.",
+          "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.UpdateStrategy"
+        },
         "volumes": {
           "type": "array",
           "items": {
@@ -20617,11 +20625,11 @@
           "x-kubernetes-patch-strategy": "merge"
         },
         "currentHash": {
-          "description": "If not empty, indicates the version of the Vertex used to generate Pods in the sequence [0,currentReplicas).",
+          "description": "If not empty, indicates the current version of the Vertex used to generate Pods.",
           "type": "string"
         },
-        "currentReplicas": {
-          "description": "The number of Pods created by the controller from the Vertex version indicated by currentHash.",
+        "desiredReplicas": {
+          "description": "The number of desired replicas.",
           "type": "integer",
           "format": "int64"
         },
@@ -20657,9 +20665,14 @@
           "type": "string"
         },
         "updateHash": {
-          "description": "If not empty, indicates the version of the Vertx used to generate Pods in the sequence [replicas-updatedReplicas,replicas)",
+          "description": "If not empty, indicates the updated version of the Vertex used to generate Pods.",
           "type": "string"
         },
+        "updatedReadyReplicas": {
+          "description": "The number of ready Pods created by the controller from the Vertex version indicated by updateHash.",
+          "type": "integer",
+          "format": "int64"
+        },
         "updatedReplicas": {
           "description": "The number of Pods created by the controller from the Vertex version indicated by updateHash.",
           "type": "integer",

diff --git a/config/advanced-install/minimal-crds.yaml b/config/advanced-install/minimal-crds.yaml
@@ -206,7 +206,7 @@ spec:
     - jsonPath: .status.phase
       name: Phase
       type: string
-    - jsonPath: .spec.replicas
+    - jsonPath: .status.desiredReplicas
       name: Desired
       type: string
     - jsonPath: .status.replicas

diff --git a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml
@@ -9037,6 +9037,23 @@ spec:
                           - window
                           type: object
                       type: object
+                    updateStrategy:
+                      default:
+                        rollingUpdate:
+                          maxUnavailable: 25%
+                        type: RollingUpdate
+                      properties:
+                        rollingUpdate:
+                          properties:
+                            maxUnavailable:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                          type: object
+                        type:
+                          type: string
+                      type: object
                     volumes:
                       items:
                         properties:

diff --git a/config/base/crds/full/numaflow.numaproj.io_vertices.yaml b/config/base/crds/full/numaflow.numaproj.io_vertices.yaml
@@ -21,7 +21,7 @@ spec:
     - jsonPath: .status.phase
       name: Phase
       type: string
-    - jsonPath: .spec.replicas
+    - jsonPath: .status.desiredReplicas
       name: Desired
       type: string
     - jsonPath: .status.replicas
@@ -4701,6 +4701,23 @@ spec:
                     - window
                     type: object
                 type: object
+              updateStrategy:
+                default:
+                  rollingUpdate:
+                    maxUnavailable: 25%
+                  type: RollingUpdate
+                properties:
+                  rollingUpdate:
+                    properties:
+                      maxUnavailable:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                    type: object
+                  type:
+                    type: string
+                type: object
               volumes:
                 items:
                   properties:
@@ -5486,7 +5503,7 @@ spec:
                 type: array
               currentHash:
                 type: string
-              currentReplicas:
+              desiredReplicas:
                 format: int32
                 type: integer
               lastScaledAt:
@@ -5515,6 +5532,9 @@ spec:
                 type: string
               updateHash:
                 type: string
+              updatedReadyReplicas:
+                format: int32
+                type: integer
               updatedReplicas:
                 format: int32
                 type: integer

diff --git a/config/base/crds/minimal/numaflow.numaproj.io_vertices.yaml b/config/base/crds/minimal/numaflow.numaproj.io_vertices.yaml
@@ -17,7 +17,7 @@ spec:
     - jsonPath: .status.phase
       name: Phase
       type: string
-    - jsonPath: .spec.replicas
+    - jsonPath: .status.desiredReplicas
       name: Desired
       type: string
     - jsonPath: .status.replicas

diff --git a/config/install.yaml b/config/install.yaml
@@ -17310,6 +17310,23 @@ spec:
                           - window
                           type: object
                       type: object
+                    updateStrategy:
+                      default:
+                        rollingUpdate:
+                          maxUnavailable: 25%
+                        type: RollingUpdate
+                      properties:
+                        rollingUpdate:
+                          properties:
+                            maxUnavailable:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                          type: object
+                        type:
+                          type: string
+                      type: object
                     volumes:
                       items:
                         properties:
@@ -18168,7 +18185,7 @@ spec:
     - jsonPath: .status.phase
       name: Phase
       type: string
-    - jsonPath: .spec.replicas
+    - jsonPath: .status.desiredReplicas
       name: Desired
       type: string
     - jsonPath: .status.replicas
@@ -22848,6 +22865,23 @@ spec:
                     - window
                     type: object
                 type: object
+              updateStrategy:
+                default:
+                  rollingUpdate:
+                    maxUnavailable: 25%
+                  type: RollingUpdate
+                properties:
+                  rollingUpdate:
+                    properties:
+                      maxUnavailable:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                    type: object
+                  type:
+                    type: string
+                type: object
               volumes:
                 items:
                   properties:
@@ -23633,7 +23667,7 @@ spec:
                 type: array
               currentHash:
                 type: string
-              currentReplicas:
+              desiredReplicas:
                 format: int32
                 type: integer
               lastScaledAt:
@@ -23662,6 +23696,9 @@ spec:
                 type: string
               updateHash:
                 type: string
+              updatedReadyReplicas:
+                format: int32
+                type: integer
               updatedReplicas:
                 format: int32
                 type: integer

diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml
@@ -17310,6 +17310,23 @@ spec:
                           - window
                           type: object
                       type: object
+                    updateStrategy:
+                      default:
+                        rollingUpdate:
+                          maxUnavailable: 25%
+                        type: RollingUpdate
+                      properties:
+                        rollingUpdate:
+                          properties:
+                            maxUnavailable:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                          type: object
+                        type:
+                          type: string
+                      type: object
                     volumes:
                       items:
                         properties:
@@ -18168,7 +18185,7 @@ spec:
     - jsonPath: .status.phase
       name: Phase
       type: string
-    - jsonPath: .spec.replicas
+    - jsonPath: .status.desiredReplicas
       name: Desired
       type: string
     - jsonPath: .status.replicas
@@ -22848,6 +22865,23 @@ spec:
                     - window
                     type: object
                 type: object
+              updateStrategy:
+                default:
+                  rollingUpdate:
+                    maxUnavailable: 25%
+                  type: RollingUpdate
+                properties:
+                  rollingUpdate:
+                    properties:
+                      maxUnavailable:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                    type: object
+                  type:
+                    type: string
+                type: object
               volumes:
                 items:
                   properties:
@@ -23633,7 +23667,7 @@ spec:
                 type: array
               currentHash:
                 type: string
-              currentReplicas:
+              desiredReplicas:
                 format: int32
                 type: integer
               lastScaledAt:
@@ -23662,6 +23696,9 @@ spec:
                 type: string
               updateHash:
                 type: string
+              updatedReadyReplicas:
+                format: int32
+                type: integer
               updatedReplicas:
                 format: int32
                 type: integer