Merge pull request #1459 from facchettos/k8s-etcd

embedded etcd in k8s
loft-sh · Jan 29, 2024 · d28108f · d28108f
2 parents 7270e8e + d06294c
commit d28108f
Show file tree

Hide file tree

Showing 28 changed files with 508 additions and 235 deletions.
diff --git a/charts/eks/templates/_helpers.tpl b/charts/eks/templates/_helpers.tpl
@@ -176,3 +176,68 @@ Prints only the flags that modify the defaults:
 - '--map-host-service={{ $value.from }}={{ $value.to }}'
 {{- end }}
 {{- end -}}
+
+
+{{/*
+  deployment kind
+*/}}
+{{- define "vcluster.kind" -}}
+{{ if and .Values.embeddedEtcd.enabled .Values.pro }}StatefulSet{{ else }}Deployment{{ end }}
+{{- end -}}
+
+{{/*
+  service name for statefulset
+*/}}
+{{- define "vcluster.statefulset.serviceName" }}
+{{- if .Values.embeddedEtcd.enabled }}
+serviceName: {{ .Release.Name }}-headless
+{{- end }}
+{{- end -}}
+
+{{/*
+  volumeClaimTemplate
+*/}}
+{{- define "vcluster.statefulset.volumeClaimTemplate" }}
+{{- if .Values.embeddedEtcd.enabled }}
+{{- if .Values.autoDeletePersistentVolumeClaims }}
+{{- if ge (int .Capabilities.KubeVersion.Minor) 27 }}
+persistentVolumeClaimRetentionPolicy:
+  whenDeleted: Delete
+{{- end }}
+{{- end }}
+{{- if (hasKey .Values "volumeClaimTemplates") }}
+volumeClaimTemplates:
+{{ toYaml .Values.volumeClaimTemplates | indent 4 }}
+{{- else if .Values.syncer.storage.persistence }}
+volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      {{- if .Values.syncer.storage.className }}
+      storageClassName: {{ .Values.syncer.storage.className }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.syncer.storage.size }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+
+{{/*
+  deployment strategy
+*/}}
+{{- define "vcluster.deployment.strategy" }}
+{{- if not .Values.embeddedEtcd.enabled }}
+strategy:
+  rollingUpdate:
+    maxSurge: 1
+    {{- if (eq (int .Values.syncer.replicas) 1) }}
+    maxUnavailable: 0
+    {{- else }}
+    maxUnavailable: 1
+    {{- end }}
+  type: RollingUpdate
+{{- end }}
+{{- end -}}
diff --git a/charts/eks/templates/etcd-service.yaml b/charts/eks/templates/etcd-service.yaml
@@ -1,3 +1,4 @@
+{{- if not .Values.embeddedEtcd.enabled }}
 {{- if not .Values.headless }}
 {{- if not .Values.etcd.disabled }}
 apiVersion: v1
@@ -30,4 +31,5 @@ spec:
     app: vcluster-etcd
     release: {{ .Release.Name }}
   {{- end }}
-  {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/charts/eks/templates/etcd-statefulset-service.yaml b/charts/eks/templates/etcd-statefulset-service.yaml
@@ -1,3 +1,4 @@
+{{- if not .Values.embeddedEtcd.enabled }}
 {{- if not .Values.headless }}
 {{- if not .Values.etcd.disabled }}
 apiVersion: v1
@@ -31,4 +32,5 @@ spec:
     app: vcluster-etcd
     release: "{{ .Release.Name }}"
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}
diff --git a/charts/eks/templates/etcd-statefulset.yaml b/charts/eks/templates/etcd-statefulset.yaml
@@ -1,3 +1,4 @@
+{{- if not .Values.embeddedEtcd.enabled }}
 {{- if not .Values.headless }}
 {{- if not .Values.etcd.disabled }}
 apiVersion: apps/v1
@@ -170,3 +171,4 @@ spec:
           failureThreshold: 24
 {{- end }}
 {{- end }}
+{{- end }}
diff --git a/charts/eks/templates/statefulset-service.yaml b/charts/eks/templates/statefulset-service.yaml
@@ -0,0 +1,38 @@
+{{- if eq ( include "vcluster.kind" . ) "StatefulSet" }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-headless
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "vcluster.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+  {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }}
+  {{- if $annotations }}
+  annotations:
+{{ toYaml $annotations | indent 4 }}
+  {{- end }}
+spec:
+  publishNotReadyAddresses: true
+  ports:
+    - name: https
+      port: 443
+      targetPort: 8443
+      protocol: TCP
+    {{- if .Values.embeddedEtcd.enabled }}
+    - name: etcd
+      port: 2379
+      targetPort: 2379
+      protocol: TCP
+    - name: peer
+      port: 2380
+      targetPort: 2380
+      protocol: TCP
+    {{- end }}
+  clusterIP: None
+  selector:
+    app: vcluster
+    release: "{{ .Release.Name }}"
+{{- end }}
diff --git a/charts/eks/templates/syncer.yaml b/charts/eks/templates/syncer.yaml
@@ -1,6 +1,6 @@
 {{- if not .Values.headless }}
 apiVersion: apps/v1
-kind: Deployment
+kind: {{ include "vcluster.kind" . }}
 metadata:
   name: {{ .Release.Name }}
   namespace: {{ .Release.Namespace }}
@@ -18,16 +18,10 @@ metadata:
 {{ toYaml $annotations | indent 4 }}
   {{- end }}
 spec:
+  {{- include "vcluster.deployment.strategy" . | indent 2 }}
+  {{- include "vcluster.statefulset.serviceName" . | indent 2 }}
+  {{- include "vcluster.statefulset.volumeClaimTemplate" . | indent 2 }}
   replicas: {{ .Values.syncer.replicas }}
-  strategy:
-    rollingUpdate:
-      maxSurge: 1
-      {{- if (eq (int .Values.syncer.replicas) 1) }}
-      maxUnavailable: 0
-      {{- else }}
-      maxUnavailable: 1
-      {{- end }}
-    type: RollingUpdate
   selector:
     matchLabels:
       app: vcluster
@@ -227,6 +221,12 @@ spec:
           {{- if .Values.syncer.kubeConfigContextName }}
           - --kube-config-context-name={{ .Values.syncer.kubeConfigContextName }}
           {{- end }}
+          {{- if .Values.pro }}
+          {{- if .Values.embeddedEtcd.enabled }}
+          - --etcd-embedded
+          - --etcd-replicas={{ .Values.syncer.replicas }}
+          {{- end }}
+          {{- end }}
           {{- if .Values.enableHA }}
           - --leader-elect=true
           {{- else }}
@@ -311,6 +311,10 @@ spec:
         {{- include "vcluster.plugins.config" . | indent 10 }}
           - name: VCLUSTER_DISTRO
             value: eks
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
         {{- if eq (.Values.syncer.replicas | toString | atoi) 1 }}
           - name: VCLUSTER_NODE_NAME
             valueFrom:
@@ -341,7 +345,11 @@ spec:
                 - '--etcd-cafile=/pki/etcd/ca.crt'
                 - '--etcd-certfile=/pki/apiserver-etcd-client.crt'
                 - '--etcd-keyfile=/pki/apiserver-etcd-client.key'
+                {{- if .Values.embeddedEtcd.enabled }}
+                - '--etcd-servers=https://127.0.0.1:2379'
+                {{- else }}
                 - '--etcd-servers=https://{{ .Release.Name }}-etcd:2379'
+                {{- end }}
                 - '--proxy-client-cert-file=/pki/front-proxy-client.crt'
                 - '--proxy-client-key-file=/pki/front-proxy-client.key'
                 - '--requestheader-allowed-names=front-proxy-client'

diff --git a/charts/eks/tests/syncer_test.yaml b/charts/eks/tests/syncer_test.yaml
@@ -0,0 +1,38 @@
+suite: Syncer
+templates:
+  - syncer.yaml
+
+tests:
+  - it: should be a statefulset when embeddedEtcd is enabled
+    set:
+      pro: true
+      embeddedEtcd:
+        enabled: true
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: StatefulSet
+      - exists:
+          path: spec.serviceName
+      - exists:
+          path: spec.volumeClaimTemplates
+      - notExists:
+          path: spec.strategy
+
+  - it: should be a deployment when embeddedEtcd is disabled
+    set:
+      pro: true
+      embeddedEtcd:
+        enabled: false
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: Deployment
+      - notExists:
+          path: spec.serviceName
+      - notExists:
+          path: spec.volumeClaimTemplates
+      - exists:
+          path: spec.strategy
diff --git a/charts/eks/values.yaml b/charts/eks/values.yaml
@@ -8,18 +8,21 @@ globalAnnotations: {}
 # If vCluster.Pro is enabled
 pro: false
 
-# If true, will deploy vcluster in headless mode, which means no deployment
-# or statefulset is created.
+# Embedded etcd settings
+embeddedEtcd:
+ # If embedded etcd should be enabled, this is a PRO only feature
+ enabled: false
+
+# Extra Annotations for the stateful set
+annotations: {}
+podAnnotations: {}
+
 headless: false
 
 monitoring:
   serviceMonitor:
     enabled: false
 
-# If the control plane is deployed in high availability mode
-# Make sure to scale up the syncer.replicas, etcd.replicas, api.replicas & controller.replicas
-enableHA: false
-
 # Plugins that should get loaded. Usually you want to apply those via 'vcluster create ... -f https://.../plugin.yaml'
 plugin: {}
 # Manually configure a plugin called test
@@ -178,6 +181,16 @@ syncer:
   securityContext: {}
   podSecurityContext: {}
   serviceAnnotations: {}
+  # Storage settings for the vcluster
+  storage:
+   # If this is disabled, vcluster will use an emptyDir instead
+   # of a PersistentVolumeClaim
+   persistence: true
+   # Size of the persistent volume claim
+   size: 5Gi
+   # Optional StorageClass used for the pvc
+   # if empty default StorageClass defined in your host cluster will be used
+   #className:
 
 # Etcd settings
 etcd:
@@ -220,52 +233,12 @@ etcd:
 controller:
   image: public.ecr.aws/eks-distro/kubernetes/kube-controller-manager:v1.28.2-eks-1-28-6
   imagePullPolicy: ""
-  # The amount of replicas to run the deployment with
-  replicas: 1
-  # NodeSelector used
-  nodeSelector: {}
-  # Affinity to apply
-  affinity: {}
-  # Tolerations to apply
-  tolerations: []
-  # Extra Labels
-  labels: {}
-  # Extra Annotations
-  annotations: {}
-  podAnnotations: {}
-  podLabels: {}
-  resources:
-    requests:
-      cpu: 15m
-  priorityClassName: ""
-  securityContext: {}
 
 # Kubernetes API Server settings
 api:
   image: public.ecr.aws/eks-distro/kubernetes/kube-apiserver:v1.28.2-eks-1-28-6
   imagePullPolicy: ""
   extraArgs: []
-  # The amount of replicas to run the deployment with
-  replicas: 1
-  # NodeSelector used to schedule the syncer
-  nodeSelector: {}
-  # Affinity to apply to the syncer deployment
-  affinity: {}
-  # Tolerations to apply to the syncer deployment
-  tolerations: []
-  # Extra Labels for the syncer deployment
-  labels: {}
-  # Extra Annotations for the syncer deployment
-  annotations: {}
-  podAnnotations: {}
-  podLabels: {}
-  resources:
-    requests:
-      cpu: 40m
-      memory: 300Mi
-  priorityClassName: ""
-  securityContext: {}
-  serviceAnnotations: {}
 
 # Core DNS settings
 coredns:

diff --git a/charts/k0s/templates/_storage.tpl b/charts/k0s/templates/_storage.tpl
@@ -0,0 +1,20 @@
+{{/*
+  storage size
+*/}}
+{{- define "vcluster.storage.size" -}}
+{{if .Values.storage }}{{ .Values.storage.size }}{{ else }}{{ .Values.syncer.storage.size }}{{ end }}
+{{- end -}}
+
+{{/*
+  storage persistence
+*/}}
+{{- define "vcluster.storage.persistence" -}}
+{{if .Values.storage }}{{ .Values.storage.persistence }}{{ else }}{{ .Values.syncer.storage.persistence }}{{ end }}
+{{- end -}}
+
+{{/*
+  storage classname
+*/}}
+{{- define "vcluster.storage.className" -}}
+{{if .Values.storage }}{{ .Values.storage.className }}{{ else }}{{ .Values.syncer.storage.className }}{{ end }}
+{{- end -}}