feat: enable restful daemon client option for UX server (numaproj#1826)

ayildirim21 · Jul 15, 2024 · 251e84f · 251e84f
1 parent 6ecf01e
commit 251e84f
Show file tree

Hide file tree

Showing 23 changed files with 1,950 additions and 64 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -81,7 +81,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.22'
         id: go
 
       - name: Check out code

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -101,7 +101,7 @@ jobs:
           fi
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.22'
       - uses: actions/checkout@v4
       - run: go install sigs.k8s.io/bom/cmd/[email protected]
       - run: go install github.com/spdx/spdx-sbom-generator/cmd/[email protected]

diff --git a/cmd/commands/server.go b/cmd/commands/server.go
@@ -30,15 +30,16 @@ import (
 
 func NewServerCommand() *cobra.Command {
 	var (
-		insecure           bool
-		port               int
-		namespaced         bool
-		managedNamespace   string
-		baseHref           string
-		disableAuth        bool
-		serverAddr         string
-		corsAllowedOrigins string
-		readOnly           bool
+		insecure             bool
+		port                 int
+		namespaced           bool
+		managedNamespace     string
+		baseHref             string
+		disableAuth          bool
+		serverAddr           string
+		corsAllowedOrigins   string
+		readOnly             bool
+		daemonClientProtocol string
 	)
 
 	command := &cobra.Command{
@@ -52,16 +53,17 @@ func NewServerCommand() *cobra.Command {
 				baseHref = baseHref + "/"
 			}
 			opts := svrcmd.ServerOptions{
-				Insecure:           insecure,
-				Port:               port,
-				Namespaced:         namespaced,
-				ManagedNamespace:   managedNamespace,
-				BaseHref:           baseHref,
-				DisableAuth:        disableAuth,
-				DexServerAddr:      common.NumaflowDexServerAddr,
-				ServerAddr:         serverAddr,
-				CorsAllowedOrigins: corsAllowedOrigins,
-				ReadOnly:           readOnly,
+				Insecure:             insecure,
+				Port:                 port,
+				Namespaced:           namespaced,
+				ManagedNamespace:     managedNamespace,
+				BaseHref:             baseHref,
+				DisableAuth:          disableAuth,
+				DexServerAddr:        common.NumaflowDexServerAddr,
+				ServerAddr:           serverAddr,
+				CorsAllowedOrigins:   corsAllowedOrigins,
+				ReadOnly:             readOnly,
+				DaemonClientProtocol: daemonClientProtocol,
 			}
 			server := svrcmd.NewServer(opts)
 			log := logging.NewLogger().Named("server")
@@ -77,5 +79,6 @@ func NewServerCommand() *cobra.Command {
 	command.Flags().BoolVar(&disableAuth, "disable-auth", sharedutil.LookupEnvBoolOr("NUMAFLOW_SERVER_DISABLE_AUTH", false), "Whether to disable authentication and authorization, defaults to false.")
 	command.Flags().StringVar(&serverAddr, "server-addr", sharedutil.LookupEnvStringOr("NUMAFLOW_SERVER_ADDRESS", "https://localhost:8443"), "The external address of the Numaflow server.")
 	command.Flags().StringVar(&corsAllowedOrigins, "cors-allowed-origins", sharedutil.LookupEnvStringOr("NUMAFLOW_SERVER_CORS_ALLOWED_ORIGINS", ""), "The values for allowed cors AllowOrigins header field, separated by comma.")
+	command.Flags().StringVar(&daemonClientProtocol, "daemon-client-protocol", sharedutil.LookupEnvStringOr("NUMAFLOW_SERVER_DAEMON_CLIENT_PROTOCOL", "grpc"), "The protocol used to connect to the Pipeline daemon service from Numaflow UX server, defaults to 'grpc'.")
 	return command
 }
diff --git a/config/advanced-install/namespaced-numaflow-server.yaml b/config/advanced-install/namespaced-numaflow-server.yaml
@@ -258,6 +258,12 @@ spec:
               key: server.cors.allowed.origins
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_DAEMON_CLIENT_PROTOCOL
+          valueFrom:
+            configMapKeyRef:
+              key: server.daemon.client.protocol
+              name: numaflow-cmd-params-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe:

diff --git a/config/advanced-install/numaflow-server.yaml b/config/advanced-install/numaflow-server.yaml
@@ -269,6 +269,12 @@ spec:
               key: server.cors.allowed.origins
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_DAEMON_CLIENT_PROTOCOL
+          valueFrom:
+            configMapKeyRef:
+              key: server.daemon.client.protocol
+              name: numaflow-cmd-params-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe:

diff --git a/config/base/numaflow-server/numaflow-server-deployment.yaml b/config/base/numaflow-server/numaflow-server-deployment.yaml
@@ -138,6 +138,12 @@ spec:
                 name: numaflow-cmd-params-config
                 key: server.cors.allowed.origins
                 optional: true
+          - name: NUMAFLOW_SERVER_DAEMON_CLIENT_PROTOCOL
+            valueFrom:
+              configMapKeyRef:
+                name: numaflow-cmd-params-config
+                key: server.daemon.client.protocol
+                optional: true
           resources:
             limits:
               cpu: 500m

diff --git a/config/base/shared-config/numaflow-cmd-params-config.yaml b/config/base/shared-config/numaflow-cmd-params-config.yaml
@@ -18,7 +18,7 @@ data:
   # controller.leader.election.lease.duration: 15s
   #
   ### The duration that the acting controlplane will retry refreshing leadership before giving up. 
-  #   Default is 10 seconds.
+  #   Default value is 10 seconds.
   #   The configuration has to be: lease.duration > lease.renew.deadline > lease.renew.period
   # controller.leader.election.lease.renew.deadline: 10s
   #
@@ -45,6 +45,11 @@ data:
   ### The external address of the Numaflow server. This is needed when using Dex for authentication.
   # server.address: https://localhost:8443
   #
-  ### The list of allowed origins for CORS on Numaflow server, separated by a comma, defaults to ''.
+  ### The list of allowed origins for CORS on Numaflow UX server, separated by a comma, defaults to ''.
   #   For example: server.cors.allowed.origins: "http://localhost:3000,http://localhost:3001"
-  #  server.cors.allowed.origins: ""
+  #  server.cors.allowed.origins: ""
+  #
+  ### The protocol used to connect to the Pipeline daemon service from Numaflow UX server.
+  #   Could be either 'grpc' or 'http', defaults to 'grpc'.
+  # 
+  # server.daemon.client.protocol: grpc
diff --git a/config/install.yaml b/config/install.yaml
@@ -18671,6 +18671,12 @@ spec:
               key: server.cors.allowed.origins
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_DAEMON_CLIENT_PROTOCOL
+          valueFrom:
+            configMapKeyRef:
+              key: server.daemon.client.protocol
+              name: numaflow-cmd-params-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe:

diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml
@@ -18563,6 +18563,12 @@ spec:
               key: server.cors.allowed.origins
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_DAEMON_CLIENT_PROTOCOL
+          valueFrom:
+            configMapKeyRef:
+              key: server.daemon.client.protocol
+              name: numaflow-cmd-params-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe: