feat(CU-8695egu2z)!: update workerpool controller

This is basically a port of changes generated by spacelift-io/kube-workerpool-controller#128 to the Helm chart. To help the review, here is below the plain k8s manifest diff that I "Helmified". ```diff --- build/manifests/manifests.yaml 2025-01-09 14:51:37 +++ build/manifests/manifests.new.yaml 2025-01-09 15:54:16 @@ -2,12 +2,8 @@ kind: Namespace metadata: labels: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: system app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: namespace - app.kubernetes.io/part-of: spacelift-workerpool-controller + app.kubernetes.io/name: spacelift-workerpool-controller control-plane: controller-manager name: spacelift-worker-controller-system --- @@ -5215,12 +5211,8 @@ kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: controller-manager-sa app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: serviceaccount - app.kubernetes.io/part-of: spacelift-workerpool-controller + app.kubernetes.io/name: spacelift-workerpool-controller name: spacelift-worker-controllercontroller-manager namespace: spacelift-worker-controller-system --- @@ -5228,12 +5220,8 @@ kind: Role metadata: labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: leader-election-role - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: role - app.kubernetes.io/part-of: spacelift-workerpool-controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: spacelift-workerpool-controller name: spacelift-worker-controllerleader-election-role namespace: spacelift-worker-controller-system rules: @@ -5325,13 +5313,24 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: spacelift-workerpool-controller + name: spacelift-worker-controllermetrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: name: spacelift-worker-controllermetrics-reader rules: - nonResourceURLs: @@ -5343,37 +5342,108 @@ kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: proxy-role app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: spacelift-workerpool-controller - name: spacelift-worker-controllerproxy-role + app.kubernetes.io/name: spacelift-workerpool-controller + name: spacelift-worker-controllerworker-editor-role rules: - apiGroups: - - authentication.k8s.io + - workers.spacelift.io resources: - - tokenreviews + - workers verbs: - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - - authorization.k8s.io + - workers.spacelift.io resources: - - subjectaccessreviews + - workers/status verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: spacelift-workerpool-controller + name: spacelift-worker-controllerworker-viewer-role +rules: +- apiGroups: + - workers.spacelift.io + resources: + - workers + verbs: + - get + - list + - watch +- apiGroups: + - workers.spacelift.io + resources: + - workers/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: spacelift-workerpool-controller + name: spacelift-worker-controllerworkerpool-editor-role +rules: +- apiGroups: + - workers.spacelift.io + resources: + - workerpools + verbs: - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - workers.spacelift.io + resources: + - workerpools/status + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: spacelift-workerpool-controller + name: spacelift-worker-controllerworkerpool-viewer-role +rules: +- apiGroups: + - workers.spacelift.io + resources: + - workerpools + verbs: + - get + - list + - watch +- apiGroups: + - workers.spacelift.io + resources: + - workerpools/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: leader-election-rolebinding app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: rolebinding - app.kubernetes.io/part-of: spacelift-workerpool-controller + app.kubernetes.io/name: spacelift-workerpool-controller name: spacelift-worker-controllerleader-election-rolebinding namespace: spacelift-worker-controller-system roleRef: @@ -5389,12 +5459,8 @@ kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: manager-rolebinding app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: spacelift-workerpool-controller + app.kubernetes.io/name: spacelift-workerpool-controller name: spacelift-worker-controllermanager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io @@ -5408,18 +5474,11 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: spacelift-workerpool-controller - name: spacelift-worker-controllerproxy-rolebinding + name: spacelift-worker-controllermetrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: spacelift-worker-controllerproxy-role + name: spacelift-worker-controllermetrics-auth-role subjects: - kind: ServiceAccount name: spacelift-worker-controllercontroller-manager @@ -5429,12 +5488,8 @@ kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: controller-manager-metrics-service app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: service - app.kubernetes.io/part-of: spacelift-workerpool-controller + app.kubernetes.io/name: spacelift-workerpool-controller control-plane: controller-manager name: spacelift-worker-controllercontroller-manager-metrics-service namespace: spacelift-worker-controller-system @@ -5443,7 +5498,7 @@ - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: control-plane: controller-manager --- @@ -5451,12 +5506,8 @@ kind: Deployment metadata: labels: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: spacelift-workerpool-controller - app.kubernetes.io/instance: controller-manager app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: deployment - app.kubernetes.io/part-of: spacelift-workerpool-controller + app.kubernetes.io/name: spacelift-workerpool-controller control-plane: controller-manager name: spacelift-worker-controllercontroller-manager namespace: spacelift-worker-controller-system @@ -5488,32 +5539,7 @@ - linux containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --leader-elect command: - /spacelift-workerpool-controller @@ -5526,7 +5552,7 @@ periodSeconds: 20 name: manager ports: - - containerPort: 8080 + - containerPort: 8443 name: metrics - containerPort: 8081 name: health ```
spacelift-io · Jan 14, 2025 · 6035fad · 6035fad
1 parent 07c6ce9
commit 6035fad
Show file tree

Hide file tree

Showing 11 changed files with 600 additions and 204 deletions.
diff --git a/spacelift-workerpool-controller/crds/worker-crd.yaml b/spacelift-workerpool-controller/crds/worker-crd.yaml
@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.13.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: workers.workers.spacelift.io
 spec:
   group: workers.spacelift.io

diff --git a/spacelift-workerpool-controller/crds/workerpool-crd.yaml b/spacelift-workerpool-controller/crds/workerpool-crd.yaml
diff --git a/spacelift-workerpool-controller/templates/deployment.yaml b/spacelift-workerpool-controller/templates/deployment.yaml
@@ -3,9 +3,6 @@ kind: Deployment
 metadata:
   name: {{ include "spacelift-workerpool-controller.fullname" . }}-controller-manager
   labels:
-    app.kubernetes.io/component: manager
-    app.kubernetes.io/created-by: spacelift-workerpool-controller
-    app.kubernetes.io/part-of: spacelift-workerpool-controller
     control-plane: controller-manager
   {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
 spec:
@@ -40,25 +37,12 @@ spec:
                 values:
                 - linux
       containers:
-      {{- if .Values.metricsService.enabled }}
-      - args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }}
-        env:
-        - name: KUBERNETES_CLUSTER_DOMAIN
-          value: {{ quote .Values.kubernetesClusterDomain }}
-        image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag
-          | default .Chart.AppVersion }}
-        name: kube-rbac-proxy
-        ports:
-        - containerPort: 8443
-          name: https
-          protocol: TCP
-        resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent
-          10 }}
-        securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext
-          | nindent 10 }}
-      {{- end }}
       - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }}
-        - --metrics-bind-address={{ if .Values.metricsService.enabled }}127.0.0.1:8080{{ else }}0{{ end }}
+        {{- if .Values.metricsService.enabled }}
+        - --metrics-bind-address=:8443
+        - --metrics-secure={{ .Values.metricsService.secure | toYaml}}
+        - --enable-http2={{ .Values.metricsService.enableHTTP2 | toYaml}}
+        {{- end }}
         {{- range .Values.controllerManager.namespaces }}
         - --namespaces={{ . }}
         {{- end }}
@@ -73,7 +57,7 @@ spec:
           - containerPort: 8081
             name: health
           {{- if .Values.metricsService.enabled }}
-          - containerPort: 8080
+          - containerPort: 8443
             name: metrics
           {{- end }}
         livenessProbe:

diff --git a/spacelift-workerpool-controller/templates/leader-election-rbac.yaml b/spacelift-workerpool-controller/templates/leader-election-rbac.yaml
@@ -3,9 +3,6 @@ kind: Role
 metadata:
   name: {{ include "spacelift-workerpool-controller.fullname" . }}-leader-election-role
   labels:
-    app.kubernetes.io/component: rbac
-    app.kubernetes.io/created-by: spacelift-workerpool-controller
-    app.kubernetes.io/part-of: spacelift-workerpool-controller
   {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
 rules:
 - apiGroups:
@@ -45,9 +42,6 @@ kind: RoleBinding
 metadata:
   name: {{ include "spacelift-workerpool-controller.fullname" . }}-leader-election-rolebinding
   labels:
-    app.kubernetes.io/component: rbac
-    app.kubernetes.io/created-by: spacelift-workerpool-controller
-    app.kubernetes.io/part-of: spacelift-workerpool-controller
   {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -56,4 +50,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: '{{ include "spacelift-workerpool-controller.fullname" . }}-controller-manager'
-  namespace: '{{ .Release.Namespace }}'
+  namespace: '{{ .Release.Namespace }}'
diff --git a/spacelift-workerpool-controller/templates/manager-rbac.yaml b/spacelift-workerpool-controller/templates/manager-rbac.yaml
@@ -2,59 +2,25 @@
 - apiGroups:
     - ""
   resources:
-    - pods
+    - events
   verbs:
     - create
-    - delete
-    - get
-    - list
-    - watch
+    - patch
 - apiGroups:
     - ""
   resources:
+    - pods
     - secrets
   verbs:
     - create
     - delete
     - get
     - list
     - watch
-- apiGroups:
-    - ""
-  resources:
-    - events
-  verbs:
-    - create
-    - patch
 - apiGroups:
     - workers.spacelift.io
   resources:
     - workerpools
-  verbs:
-    - create
-    - delete
-    - get
-    - list
-    - patch
-    - update
-    - watch
-- apiGroups:
-    - workers.spacelift.io
-  resources:
-    - workerpools/finalizers
-  verbs:
-    - update
-- apiGroups:
-    - workers.spacelift.io
-  resources:
-    - workerpools/status
-  verbs:
-    - get
-    - patch
-    - update
-- apiGroups:
-    - workers.spacelift.io
-  resources:
     - workers
   verbs:
     - create
@@ -67,12 +33,14 @@
 - apiGroups:
     - workers.spacelift.io
   resources:
+    - workerpools/finalizers
     - workers/finalizers
   verbs:
     - update
 - apiGroups:
     - workers.spacelift.io
   resources:
+    - workerpools/status
     - workers/status
   verbs:
     - get
@@ -95,9 +63,6 @@ kind: ClusterRoleBinding
 metadata:
   name: {{ include "spacelift-workerpool-controller.fullname" . }}-manager-rolebinding
   labels:
-    app.kubernetes.io/component: rbac
-    app.kubernetes.io/created-by: spacelift-workerpool-controller
-    app.kubernetes.io/part-of: spacelift-workerpool-controller
   {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -126,9 +91,6 @@ metadata:
   name: {{ include "spacelift-workerpool-controller.fullname" $ }}-manager-rolebinding
   namespace: '{{ $namespace }}'
   labels:
-    app.kubernetes.io/component: rbac
-    app.kubernetes.io/created-by: spacelift-workerpool-controller
-    app.kubernetes.io/part-of: spacelift-workerpool-controller
   {{- include "spacelift-workerpool-controller.labels" $ | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -140,3 +102,20 @@ subjects:
     namespace: '{{ $.Release.Namespace }}'
 {{ end }}
 {{ end }}
+{{ if and .Values.metricsService.enabled .Values.metricsService.secure }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "spacelift-workerpool-controller.fullname" . }}-metrics-auth-rolebinding
+  labels:
+  {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: '{{ include "spacelift-workerpool-controller.fullname" . }}-metrics-auth-role'
+subjects:
+  - kind: ServiceAccount
+    name: '{{ include "spacelift-workerpool-controller.fullname" . }}-controller-manager'
+    namespace: '{{ .Release.Namespace }}'
+{{ end }}
diff --git a/spacelift-workerpool-controller/templates/metrics-rbac.yaml b/spacelift-workerpool-controller/templates/metrics-rbac.yaml
@@ -0,0 +1,33 @@
+{{ if and .Values.metricsService.enabled .Values.metricsService.secure }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "spacelift-workerpool-controller.fullname" . }}-metrics-auth-role
+  labels:
+  {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
+rules:
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "spacelift-workerpool-controller.fullname" . }}-metrics-reader
+  labels:
+  {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
+rules:
+  - nonResourceURLs:
+      - /metrics
+    verbs:
+      - get
+{{ end }}
diff --git a/spacelift-workerpool-controller/templates/metrics-reader-rbac.yaml b/spacelift-workerpool-controller/templates/metrics-reader-rbac.yaml
diff --git a/spacelift-workerpool-controller/templates/metrics-service.yaml b/spacelift-workerpool-controller/templates/metrics-service.yaml
@@ -4,9 +4,6 @@ kind: Service
 metadata:
   name: {{ include "spacelift-workerpool-controller.fullname" . }}-metrics-service
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
-    app.kubernetes.io/created-by: spacelift-workerpool-controller
-    app.kubernetes.io/part-of: spacelift-workerpool-controller
     control-plane: controller-manager
   {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
 spec:

diff --git a/spacelift-workerpool-controller/templates/proxy-rbac.yaml b/spacelift-workerpool-controller/templates/proxy-rbac.yaml
diff --git a/spacelift-workerpool-controller/templates/serviceaccount.yaml b/spacelift-workerpool-controller/templates/serviceaccount.yaml
@@ -3,12 +3,9 @@ kind: ServiceAccount
 metadata:
   name: {{ include "spacelift-workerpool-controller.fullname" . }}-controller-manager
   labels:
-    app.kubernetes.io/component: rbac
-    app.kubernetes.io/created-by: spacelift-workerpool-controller
-    app.kubernetes.io/part-of: spacelift-workerpool-controller
   {{- include "spacelift-workerpool-controller.labels" . | nindent 4 }}
   {{- with .Values.controllerManager.serviceAccount.labels }}
     {{- toYaml . | nindent 4 }}
   {{- end }}
   annotations:
-    {{- toYaml .Values.controllerManager.serviceAccount.annotations | nindent 4 }}
+    {{- toYaml .Values.controllerManager.serviceAccount.annotations | nindent 4 }}
diff --git a/spacelift-workerpool-controller/values.yaml b/spacelift-workerpool-controller/values.yaml
@@ -3,33 +3,7 @@ controllerManager:
   # and will be able to manage WorkerPools across all namespaces in your cluster.
   # If you do not want to grant cluster wide permissions to the controller, you can specify a list
   # of namespaces. That will create a Role per namespace and bind it to the service account used by the controller.
-  #
-  # PLEASE NOTE: currently the metrics service requires a ClusterRole in order to function, so
-  # if `metricsService.enabled` is set to true, a ClusterRole will be created even if you
-  # specify namespaces.
   namespaces: []
-  kubeRbacProxy:
-    args:
-    - --secure-listen-address=0.0.0.0:8443
-    - --upstream=http://127.0.0.1:8080/
-    - --logtostderr=true
-    - --v=0
-    containerSecurityContext:
-      readOnlyRootFilesystem: true
-      allowPrivilegeEscalation: false
-      capabilities:
-        drop:
-        - ALL
-    image:
-      repository: gcr.io/kubebuilder/kube-rbac-proxy
-      tag: v0.14.1
-    resources:
-      limits:
-        cpu: 500m
-        memory: 128Mi
-      requests:
-        cpu: 5m
-        memory: 64Mi
   manager:
     args:
     - --health-probe-bind-address=:8081
@@ -42,7 +16,7 @@ controllerManager:
         - ALL
     image:
       repository: public.ecr.aws/spacelift/kube-workerpool-controller
-      tag: v0.0.14
+      tag: v0.0.17
     resources:
       limits:
         memory: 128Mi
@@ -62,12 +36,20 @@ kubernetesClusterDomain: cluster.local
 # This is disabled by default, enable this if you want to enable controller observability.
 metricsService:
   enabled: false
+  # Enabling secure will also create ClusterRole to enable authn/authz to the metrics endpoint through RBAC.
+  # More details here https://book.kubebuilder.io/reference/metrics#by-using-authnauthz-enabled-by-default
+  # Secure is enabled by default to be consistent with Kubebuilder defaults.
+  #
+  # If you want to avoid cluster roles, you can keep this set to false and configure a NetworkPolicu instead.
+  # An example can be found in Kubebuilder docs here https://github.com/kubernetes-sigs/kubebuilder/blob/d063d5af162a772379a761fae5aaea8c91b877d4/docs/book/src/getting-started/testdata/project/config/network-policy/allow-metrics-traffic.yaml#L2
+  secure: true
+  enableHTTP2: false
   ports:
   - name: https
     port: 8443
     protocol: TCP
-    targetPort: https
+    targetPort: metrics
   type: ClusterIP
 
 spacelift-promex:
-  enabled: false
+  enabled: false