From e9c3731bcb53f5b5c18695efee2d2a4c5b616adc Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Wed, 10 Apr 2024 11:11:37 -0700 Subject: [PATCH] feat: expose controller leader election duration and renew opts (#1657) Signed-off-by: Derek Wang --- .../namespaced-controller-wo-crds.yaml | 20 +++++- .../controller-manager-deployment.yaml | 20 +++++- .../numaflow-cmd-params-config.yaml | 17 ++++- config/install.yaml | 20 +++++- config/namespace-install.yaml | 20 +++++- docs/operations/installation.md | 27 +++++++- pkg/apis/numaflow/v1alpha1/const.go | 67 ++++++++++--------- pkg/reconciler/cmd/start.go | 26 +++++++ test/manifests/kustomization.yaml | 2 +- 9 files changed, 180 insertions(+), 39 deletions(-) diff --git a/config/advanced-install/namespaced-controller-wo-crds.yaml b/config/advanced-install/namespaced-controller-wo-crds.yaml index 41fb7acec6..09a5f0dfa2 100644 --- a/config/advanced-install/namespaced-controller-wo-crds.yaml +++ b/config/advanced-install/namespaced-controller-wo-crds.yaml @@ -326,7 +326,25 @@ spec: - name: NUMAFLOW_LEADER_ELECTION_DISABLED valueFrom: configMapKeyRef: - key: controller.disable.leader.election + key: controller.leader.election.disabled + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_DURATION + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.duration + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_DEADLINE + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.renew.deadline + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_PERIOD + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.renew.period name: numaflow-cmd-params-config optional: true image: quay.io/numaproj/numaflow:latest diff --git a/config/base/controller-manager/controller-manager-deployment.yaml b/config/base/controller-manager/controller-manager-deployment.yaml index 8ce8d215c4..d9f29d9f93 100644 --- a/config/base/controller-manager/controller-manager-deployment.yaml +++ b/config/base/controller-manager/controller-manager-deployment.yaml @@ -49,7 +49,25 @@ spec: valueFrom: configMapKeyRef: name: numaflow-cmd-params-config - key: controller.disable.leader.election + key: controller.leader.election.disabled + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_DURATION + valueFrom: + configMapKeyRef: + name: numaflow-cmd-params-config + key: controller.leader.election.lease.duration + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_DEADLINE + valueFrom: + configMapKeyRef: + name: numaflow-cmd-params-config + key: controller.leader.election.lease.renew.deadline + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_PERIOD + valueFrom: + configMapKeyRef: + name: numaflow-cmd-params-config + key: controller.leader.election.lease.renew.period optional: true volumeMounts: - mountPath: /etc/numaflow diff --git a/config/base/shared-config/numaflow-cmd-params-config.yaml b/config/base/shared-config/numaflow-cmd-params-config.yaml index 50cdd28bae..53681213cb 100644 --- a/config/base/shared-config/numaflow-cmd-params-config.yaml +++ b/config/base/shared-config/numaflow-cmd-params-config.yaml @@ -10,7 +10,22 @@ data: # managed.namespace: numaflow-system # ### Whether to disable leader election for the controller, defaults to false - # controller.disable.leader.election: "false" + # controller.leader.election.disabled: "false" + # + ### The duration that non-leader candidates will wait to force acquire leadership. + # This is measured against time of last observed ack. Default is 15 seconds. + # The configuration has to be: lease.duration > lease.renew.deadline > lease.renew.period + # controller.leader.election.lease.duration: 15s + # + ### The duration that the acting controlplane will retry refreshing leadership before giving up. + # Default is 10 seconds. + # The configuration has to be: lease.duration > lease.renew.deadline > lease.renew.period + # controller.leader.election.lease.renew.deadline: 10s + # + ### The duration the LeaderElector clients should wait between tries of actions, which means every + # this period of time, it tries to renew the lease. Default is 2 seconds. + # The configuration has to be: lease.duration > lease.renew.deadline > lease.renew.period + # controller.leader.election.lease.renew.period: 2s # ### Whether to disable TLS for UX server. # server.insecure: "false" diff --git a/config/install.yaml b/config/install.yaml index 9c277a2090..a4564ca341 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -17382,7 +17382,25 @@ spec: - name: NUMAFLOW_LEADER_ELECTION_DISABLED valueFrom: configMapKeyRef: - key: controller.disable.leader.election + key: controller.leader.election.disabled + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_DURATION + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.duration + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_DEADLINE + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.renew.deadline + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_PERIOD + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.renew.period name: numaflow-cmd-params-config optional: true image: quay.io/numaproj/numaflow:latest diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml index 710b123377..d2c0b0261d 100644 --- a/config/namespace-install.yaml +++ b/config/namespace-install.yaml @@ -17276,7 +17276,25 @@ spec: - name: NUMAFLOW_LEADER_ELECTION_DISABLED valueFrom: configMapKeyRef: - key: controller.disable.leader.election + key: controller.leader.election.disabled + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_DURATION + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.duration + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_DEADLINE + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.renew.deadline + name: numaflow-cmd-params-config + optional: true + - name: NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_PERIOD + valueFrom: + configMapKeyRef: + key: controller.leader.election.lease.renew.period name: numaflow-cmd-params-config optional: true image: quay.io/numaproj/numaflow:latest diff --git a/docs/operations/installation.md b/docs/operations/installation.md index 637a07fffc..91e1b93c1c 100644 --- a/docs/operations/installation.md +++ b/docs/operations/installation.md @@ -100,6 +100,31 @@ Similarly, another approach is to add `--managed-namespace` and the specific nam By default, the Numaflow controller is installed with `Active-Passive` HA strategy enabled, which means you can run the controller with multiple replicas (defaults to 1 in the manifests). +There are some parameters can be tuned for the leader election mechanism of HA. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: numaflow-cmd-params-config +data: + ### The duration that non-leader candidates will wait to force acquire leadership. + # This is measured against time of last observed ack. Default is 15 seconds. + # The configuration has to be: lease.duration > lease.renew.deadline > lease.renew.period + controller.leader.election.lease.duration: 15s + # + ### The duration that the acting controlplane will retry refreshing leadership before giving up. + # Default is 10 seconds. + # The configuration has to be: lease.duration > lease.renew.deadline > lease.renew.period + controller.leader.election.lease.renew.deadline: 10s + ### The duration the LeaderElector clients should wait between tries of actions, which means every + # this period of time, it tries to renew the lease. Default is 2 seconds. + # The configuration has to be: lease.duration > lease.renew.deadline > lease.renew.period + controller.leader.election.lease.renew.period: 2s +``` + +These parameters are useful when you want to tune the frequency of leader election renewal calls to K8s API server, which are usually configured at a high priority level of [API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). + To turn off HA, configure the ConfigMap `numaflow-cmd-params-config` as following. ```yaml @@ -109,7 +134,7 @@ metadata: name: numaflow-cmd-params-config data: # Whether to disable leader election for the controller, defaults to false - controller.disable.leader.election: "true" + controller.leader.election.disabled: "true" ``` If HA is turned off, the controller deployment should not run with multiple replicas. diff --git a/pkg/apis/numaflow/v1alpha1/const.go b/pkg/apis/numaflow/v1alpha1/const.go index 49bf6f3da0..a704aa0e27 100644 --- a/pkg/apis/numaflow/v1alpha1/const.go +++ b/pkg/apis/numaflow/v1alpha1/const.go @@ -87,38 +87,41 @@ const ( ControllerVertex = "vertex-controller" // ENV vars - EnvNamespace = "NUMAFLOW_NAMESPACE" - EnvPipelineName = "NUMAFLOW_PIPELINE_NAME" - EnvVertexName = "NUMAFLOW_VERTEX_NAME" - EnvPod = "NUMAFLOW_POD" - EnvReplica = "NUMAFLOW_REPLICA" - EnvVertexObject = "NUMAFLOW_VERTEX_OBJECT" - EnvPipelineObject = "NUMAFLOW_PIPELINE_OBJECT" - EnvSideInputObject = "NUMAFLOW_SIDE_INPUT_OBJECT" - EnvImage = "NUMAFLOW_IMAGE" - EnvImagePullPolicy = "NUMAFLOW_IMAGE_PULL_POLICY" - EnvISBSvcRedisSentinelURL = "NUMAFLOW_ISBSVC_REDIS_SENTINEL_URL" - EnvISBSvcSentinelMaster = "NUMAFLOW_ISBSVC_REDIS_SENTINEL_MASTER" - EnvISBSvcRedisURL = "NUMAFLOW_ISBSVC_REDIS_URL" - EnvISBSvcRedisUser = "NUMAFLOW_ISBSVC_REDIS_USER" - EnvISBSvcRedisPassword = "NUMAFLOW_ISBSVC_REDIS_PASSWORD" - EnvISBSvcRedisSentinelPassword = "NUMAFLOW_ISBSVC_REDIS_SENTINEL_PASSWORD" - EnvISBSvcRedisClusterMaxRedirects = "NUMAFLOW_ISBSVC_REDIS_CLUSTER_MAX_REDIRECTS" - EnvISBSvcJetStreamUser = "NUMAFLOW_ISBSVC_JETSTREAM_USER" - EnvISBSvcJetStreamPassword = "NUMAFLOW_ISBSVC_JETSTREAM_PASSWORD" - EnvISBSvcJetStreamURL = "NUMAFLOW_ISBSVC_JETSTREAM_URL" - EnvISBSvcJetStreamTLSEnabled = "NUMAFLOW_ISBSVC_JETSTREAM_TLS_ENABLED" - EnvISBSvcConfig = "NUMAFLOW_ISBSVC_CONFIG" - EnvLeaderElectionDisabled = "NUMAFLOW_LEADER_ELECTION_DISABLED" - EnvDebug = "NUMAFLOW_DEBUG" - EnvPPROF = "NUMAFLOW_PPROF" - EnvHealthCheckDisabled = "NUMAFLOW_HEALTH_CHECK_DISABLED" - EnvGRPCMaxMessageSize = "NUMAFLOW_GRPC_MAX_MESSAGE_SIZE" - EnvCPURequest = "NUMAFLOW_CPU_REQUEST" - EnvCPULimit = "NUMAFLOW_CPU_LIMIT" - EnvMemoryRequest = "NUMAFLOW_MEMORY_REQUEST" - EnvMemoryLimit = "NUMAFLOW_MEMORY_LIMIT" - EnvGoDebug = "GODEBUG" + EnvNamespace = "NUMAFLOW_NAMESPACE" + EnvPipelineName = "NUMAFLOW_PIPELINE_NAME" + EnvVertexName = "NUMAFLOW_VERTEX_NAME" + EnvPod = "NUMAFLOW_POD" + EnvReplica = "NUMAFLOW_REPLICA" + EnvVertexObject = "NUMAFLOW_VERTEX_OBJECT" + EnvPipelineObject = "NUMAFLOW_PIPELINE_OBJECT" + EnvSideInputObject = "NUMAFLOW_SIDE_INPUT_OBJECT" + EnvImage = "NUMAFLOW_IMAGE" + EnvImagePullPolicy = "NUMAFLOW_IMAGE_PULL_POLICY" + EnvISBSvcRedisSentinelURL = "NUMAFLOW_ISBSVC_REDIS_SENTINEL_URL" + EnvISBSvcSentinelMaster = "NUMAFLOW_ISBSVC_REDIS_SENTINEL_MASTER" + EnvISBSvcRedisURL = "NUMAFLOW_ISBSVC_REDIS_URL" + EnvISBSvcRedisUser = "NUMAFLOW_ISBSVC_REDIS_USER" + EnvISBSvcRedisPassword = "NUMAFLOW_ISBSVC_REDIS_PASSWORD" + EnvISBSvcRedisSentinelPassword = "NUMAFLOW_ISBSVC_REDIS_SENTINEL_PASSWORD" + EnvISBSvcRedisClusterMaxRedirects = "NUMAFLOW_ISBSVC_REDIS_CLUSTER_MAX_REDIRECTS" + EnvISBSvcJetStreamUser = "NUMAFLOW_ISBSVC_JETSTREAM_USER" + EnvISBSvcJetStreamPassword = "NUMAFLOW_ISBSVC_JETSTREAM_PASSWORD" + EnvISBSvcJetStreamURL = "NUMAFLOW_ISBSVC_JETSTREAM_URL" + EnvISBSvcJetStreamTLSEnabled = "NUMAFLOW_ISBSVC_JETSTREAM_TLS_ENABLED" + EnvISBSvcConfig = "NUMAFLOW_ISBSVC_CONFIG" + EnvLeaderElectionDisabled = "NUMAFLOW_LEADER_ELECTION_DISABLED" + EnvLeaderElectionLeaseDuration = "NUMAFLOW_LEADER_ELECTION_LEASE_DURATION" + EnvLeaderElectionLeaseRenewDeadline = "NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_DEADLINE" + EnvLeaderElectionLeaseRenewPeriod = "NUMAFLOW_LEADER_ELECTION_LEASE_RENEW_PERIOD" + EnvDebug = "NUMAFLOW_DEBUG" + EnvPPROF = "NUMAFLOW_PPROF" + EnvHealthCheckDisabled = "NUMAFLOW_HEALTH_CHECK_DISABLED" + EnvGRPCMaxMessageSize = "NUMAFLOW_GRPC_MAX_MESSAGE_SIZE" + EnvCPURequest = "NUMAFLOW_CPU_REQUEST" + EnvCPULimit = "NUMAFLOW_CPU_LIMIT" + EnvMemoryRequest = "NUMAFLOW_MEMORY_REQUEST" + EnvMemoryLimit = "NUMAFLOW_MEMORY_LIMIT" + EnvGoDebug = "GODEBUG" PathVarRun = "/var/run/numaflow" VertexMetricsPort = 2469 diff --git a/pkg/reconciler/cmd/start.go b/pkg/reconciler/cmd/start.go index 25eb49950a..327df0d2c9 100644 --- a/pkg/reconciler/cmd/start.go +++ b/pkg/reconciler/cmd/start.go @@ -19,6 +19,7 @@ package cmd import ( "context" "reflect" + "time" "go.uber.org/zap" appv1 "k8s.io/api/apps/v1" @@ -71,6 +72,31 @@ func Start(namespaced bool, managedNamespace string) { if sharedutil.LookupEnvStringOr(dfv1.EnvLeaderElectionDisabled, "false") == "true" { opts.LeaderElection = false + } else { + leaseDurationStr := sharedutil.LookupEnvStringOr(dfv1.EnvLeaderElectionLeaseDuration, "15s") // Defaults to 15s + leaseDuration, err := time.ParseDuration(leaseDurationStr) + if err != nil { + logger.Fatalf("Invalid ENV %s value: %s", dfv1.EnvLeaderElectionLeaseDuration, leaseDurationStr) + } + opts.LeaseDuration = &leaseDuration + leaseRenewDeadlineStr := sharedutil.LookupEnvStringOr(dfv1.EnvLeaderElectionLeaseRenewDeadline, "10s") // Defaults to 10s + leaseRenewDeadline, err := time.ParseDuration(leaseRenewDeadlineStr) + if err != nil { + logger.Fatalf("Invalid ENV %s value: %s", dfv1.EnvLeaderElectionLeaseRenewDeadline, leaseRenewDeadlineStr) + } + if leaseDuration <= leaseRenewDeadline { + logger.Fatalf("Invalid config: %s should always be greater than %s", dfv1.EnvLeaderElectionLeaseDuration, dfv1.EnvLeaderElectionLeaseRenewDeadline) + } + opts.RenewDeadline = &leaseRenewDeadline + leaseRenewPeriodStr := sharedutil.LookupEnvStringOr(dfv1.EnvLeaderElectionLeaseRenewPeriod, "2s") // Defaults to 2s + leaseRenewPeriod, err := time.ParseDuration(leaseRenewPeriodStr) + if err != nil { + logger.Fatalf("Invalid ENV %s value: %s", dfv1.EnvLeaderElectionLeaseRenewPeriod, leaseRenewPeriodStr) + } + if leaseRenewDeadline <= leaseRenewPeriod { + logger.Fatalf("Invalid config: %s should always be greater than %s", dfv1.EnvLeaderElectionLeaseRenewDeadline, dfv1.EnvLeaderElectionLeaseRenewPeriod) + } + opts.RetryPeriod = &leaseRenewPeriod } if namespaced { diff --git a/test/manifests/kustomization.yaml b/test/manifests/kustomization.yaml index 361d3c8b67..c002bb3156 100644 --- a/test/manifests/kustomization.yaml +++ b/test/manifests/kustomization.yaml @@ -58,7 +58,7 @@ patchesStrategicMerge: metadata: name: numaflow-cmd-params-config data: - controller.disable.leader.election: "true" + controller.leader.election.disabled: "true" namespace: numaflow-system