Setup secret gen for operator

Makefile

@@ -1,8 +1,9 @@
 UNAME_S := $(shell uname -s)
+NC := $(shell tput sgr0) # No Color
 ifeq ($(UNAME_S),Linux)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
-  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.21.0/kind-linux-amd64
+  K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-linux-amd64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/linux/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
@@ -12,16 +13,19 @@ endif
 ifeq ($(UNAME_S),Darwin)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.darwin-10.9-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-darwin-amd64.tar.gz
-  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.21.0/kind-darwin-amd64
+  K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-darwin-arm64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/darwin/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_darwin_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64
   OPM_TAR ?= https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-4.8/opm-mac-4.8.57.tar.gz
   OPM_BIN ?= darwin-amd64-opm
 endif
 
-KIND_CLUSTER ?= chart-testing
-REPOSITORY ?= gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert
+K3D_CLUSTER ?= chart-testing
+REGISTRY ?= gcr.io
+REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
+DOCKER_NETWORK_NAME ?= ${K3D_CLUSTER}
+LOCAL_REGISTRY ?= "localhost:5000"
 
 export BUNDLE_IMAGE ?= cockroach-operator-bundle
 export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
@@ -56,7 +60,7 @@ build/chart: bin/helm ## build the helm chart to build/artifacts
 build/self-signer: bin/yq ## build the self-signer image
 	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Release
 
@@ -66,33 +70,57 @@ release: ## publish the build artifacts to S3
 build-and-push/self-signer: bin/yq ## push the self-signer image
 	@docker buildx build --platform=linux/amd64,linux/arm64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Dev
 dev/clean: ## remove built artifacts
 	@rm -r build/artifacts/
 
+## Setup/teardown registries for easier local dev
+dev/registries/up: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" = "" ]; then \
+		echo "$(CYAN)Starting local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		cd ../../bin/k3d; ./tests/k3d/registries.sh up $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/registries/down: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" != "" ]; then \
+		echo "$(CYAN)Stopping local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		cd ../../bin/k3d; ./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/push/local: dev/registries/up
+	@echo "$(CYAN)Pushing image to local registry...$(NC)"
+	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
+          	--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
+          	-t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+
 ##@ Test
+test/cluster: bin/k3d test/cluster_up ## start a local k3d cluster for testing
+
+test/cluster_up: bin/k3d
+	@bin/k3d cluster list | grep $(K3D_CLUSTER) || bin/k3d cluster create $(K3D_CLUSTER)
 
-test/cluster: bin/kind ## start a local kind cluster for testing
-	@bin/kind get clusters -q | grep $(KIND_CLUSTER) || bin/kind create cluster --name $(KIND_CLUSTER)
+test/cluster_down: bin/k3d
+	bin/k3d cluster delete $(K3D_CLUSTER)
 
 test/e2e/%: PKG=$*
-test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-kind ## run e2e tests for package (e.g. install or rotate)
+test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate)
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/$(PKG)/...
 
 test/lint: bin/helm ## lint the helm chart
 	@build/lint.sh && bin/helm lint cockroachdb
 
 IMAGE_LIST = cockroachdb/cockroach:v23.2.0 quay.io/jetstack/cert-manager-cainjector:v1.11.0 quay.io/jetstack/cert-manager-webhook:v1.11.0 quay.io/jetstack/cert-manager-controller:v1.11.0 quay.io/jetstack/cert-manager-ctl:v1.11.0
-test/publish-images-to-kind: bin/yq test/cluster ## publish signer and cockroach image to local kind registry
+test/publish-images-to-k3d: bin/yq test/cluster ## publish signer and cockroach image to local k3d registry
 	for i in $(IMAGE_LIST); do \
 		docker pull $$i; \
-		bin/kind load docker-image $$i --name $(KIND_CLUSTER); \
+		bin/k3d image import $$i -c $(K3D_CLUSTER); \
 	done
-	@bin/kind load docker-image \
-		${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
-		--name $(KIND_CLUSTER)
+	docker pull ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
+	bin/k3d image import \
+		${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
+		-c $(K3D_CLUSTER)
 
 test/template: bin/cockroach bin/helm ## Run template tests
 	@PATH="$(PWD)/bin:${PATH}" go test -v ./tests/template/...
@@ -101,7 +129,7 @@ test/units: bin/cockroach ## Run unit tests in ./pkg/...
 	@PATH="$(PWD)/bin:${PATH}" go test -v ./pkg/...
 
 ##@ Binaries
-bin: bin/cockroach bin/helm bin/kind bin/kubectl bin/yq ## install all binaries
+bin: bin/cockroach bin/helm bin/k3d bin/kubectl bin/yq ## install all binaries
 
 bin/cockroach: ## install cockroach
 	@mkdir -p bin
@@ -113,10 +141,10 @@ bin/helm: ## install helm
 	@curl -L $(HELM_BIN) | tar -xzf - -C bin/ --strip-components 1
 	@rm -f bin/README.md bin/LICENSE
 
-bin/kind: ## install kind
+bin/k3d: ## install k3d
 	@mkdir -p bin
-	@curl -Lo bin/kind $(KIND_BIN)	
-	@chmod +x bin/kind
+	@curl -Lo bin/k3d $(K3D_BIN)	
+	@chmod +x bin/k3d
 
 bin/kubectl: ## install kubectl
 	@mkdir -p bin

cmd/self-signer/generate.go

@@ -37,10 +37,12 @@ var (
 	caExpiry, nodeExpiry, clientExpiry       string
 	caSecret                                 string
 	clientOnly                               bool
+	operatorManaged                          bool
 )
 
 func init() {
 	generateCmd.Flags().BoolVar(&clientOnly, "client-only", false, "generate certificates for custom user")
+	generateCmd.Flags().BoolVar(&operatorManaged, "operator-managed", false, "generate certificates for operator managed cluster")
 	rootCmd.AddCommand(generateCmd)
 }
 
@@ -52,6 +54,7 @@ func generate(cmd *cobra.Command, args []string) {
 	}
 
 	genCert.CaSecret = caSecret
+	genCert.OperatorManaged = operatorManaged
 
 	namespace, exists := os.LookupEnv("NAMESPACE")
 	if !exists {

cockroachdb/templates/job-certSelfSigner.yaml

@@ -66,6 +66,9 @@ spec:
             - --client-expiry={{ .Values.tls.certs.selfSigner.clientCertExpiryWindow }}
             - --node-duration={{ .Values.tls.certs.selfSigner.nodeCertDuration }}
             - --node-expiry={{ .Values.tls.certs.selfSigner.nodeCertExpiryWindow }}
+            {{- if .Values.operator.enabled }}
+            - --operator-managed=true
+            {{- end}}
           env:
           - name: STATEFULSET_NAME
             value: {{ template "cockroachdb.fullname" . }}

cockroachdb/values.yaml

@@ -649,3 +649,7 @@ iap:
   # Create Google Cloud OAuth credentials and set client id and secret
   # clientId:
   # clientSecret:
+
+# Use the CRDB Operator to manage the CRDB clusters
+operator:
+ enabled: false

pkg/generator/generate_cert.go

@@ -68,6 +68,7 @@ type GenerateCert struct {
 	ClusterDomain             string
 	ReadinessWait             time.Duration
 	PodUpdateTimeout          time.Duration
+	OperatorManaged           bool
 }
 
 type certConfig struct {
@@ -240,6 +241,17 @@ func (rc *GenerateCert) generateCA(ctx context.Context, CASecretName string, nam
 			return errors.Wrap(err, "failed to update ca key secret ")
 		}
 
+		// If we are using the operator to manage secrets then we need to store the CA cert in a
+		// ConfigMap.
+		if rc.OperatorManaged {
+			cm := resource.CreateConfigMap(namespace, CASecretName, caCert,
+				resource.NewKubeResource(ctx, rc.client, namespace, kube.DefaultPersister))
+			if err = cm.Update(); err != nil {
+				return errors.Wrap(err, "failed to update CA cert in ConfigMap")
+			}
+			logrus.Infof("Generated and saved CA certificate in ConfigMap [%s]", CASecretName)
+		}
+
 		logrus.Infof("Generated and saved CA key and certificate in secret [%s]", CASecretName)
 		return nil
 	}
@@ -307,6 +319,16 @@ func (rc *GenerateCert) generateNodeCert(ctx context.Context, nodeSecretName str
 			fmt.Sprintf("*.%s.%s.svc.%s", rc.DiscoveryServiceName, namespace, rc.ClusterDomain),
 		}
 
+		if rc.OperatorManaged {
+			operatorJoinServiceHosts := []string{
+				fmt.Sprintf("%s-join", rc.DiscoveryServiceName),
+				fmt.Sprintf("%s-join.%s", rc.DiscoveryServiceName, namespace),
+				fmt.Sprintf("%s-join.%s.svc.%s", rc.DiscoveryServiceName, namespace, rc.ClusterDomain),
+			}
+
+			hosts = append(hosts, operatorJoinServiceHosts...)
+		}
+
 		// create the Node Pair certificates
 		if err = errors.Wrap(
 			security.CreateNodePair(

pkg/resource/config_map.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2021 The Cockroach Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resource
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type ConfigMap struct {
+	Resource
+
+	configMap *corev1.ConfigMap
+}
+
+// CreateConfigMap creates a ConfigMap in the specified namespace
+func CreateConfigMap(namespace string, secretName string, data []byte, r Resource) *ConfigMap {
+	// Define the ConfigMap object
+	configMap := &ConfigMap{
+		Resource: r,
+		configMap: &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: fmt.Sprintf("%s-crt", secretName),
+			},
+			Data: map[string]string{
+				"ca.crt": string(data),
+			},
+		},
+	}
+	return configMap
+}
+
+func (c *ConfigMap) Update() error {
+	data := c.configMap.Data
+	_, err := c.Persist(c.configMap, func() error {
+		c.configMap.Data = data
+		return nil
+	})
+
+	return err
+}
+
+func LoadConfigMap(name string, r Resource) (*ConfigMap, error) {
+	c := &ConfigMap{
+		Resource: r,
+		configMap: &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: name,
+			},
+		},
+	}
+
+	if err := r.Fetch(c.configMap); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (c *ConfigMap) GetConfigMap() *corev1.ConfigMap {
+	return c.configMap
+}
+
+func (c *ConfigMap) Name() string {
+	return c.configMap.Name
+}

pkg/resource/config_map_test.go

@@ -0,0 +1,31 @@
+package resource_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cockroachdb/helm-charts/pkg/kube"
+	"github.com/cockroachdb/helm-charts/pkg/resource"
+	"github.com/cockroachdb/helm-charts/pkg/testutils"
+)
+
+func TestUpdateConfigMap(t *testing.T) {
+	scheme := testutils.InitScheme(t)
+	fakeClient := testutils.NewFakeClient(scheme)
+	namespace := "default"
+	name := "test-configmap"
+
+	r := resource.NewKubeResource(context.TODO(), fakeClient, namespace, kube.DefaultPersister)
+	cm := resource.CreateConfigMap(namespace, name, []byte{}, r)
+
+	err := cm.Update()
+	require.NoError(t, err)
+
+	// fetch the configmap
+	cm, err = resource.LoadConfigMap(cm.Name(), r)
+	require.NoError(t, err)
+
+	require.Equal(t, "test-configmap-crt", cm.GetConfigMap().Name)
+}

tests/k3d/dev-cluster.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+CLUSTER_NAME=local
+
+NETWORK_NAME=k3d-local
+
+if [ $# -eq 0 ]
+  then
+      echo "No arguments supplied: "
+      echo "  up: Start cluster."
+      echo "     --nodes x: The cluster should have x nodes (default 1)"
+      echo "     --version x: The version of Kubernetes (default 1.24.14)"
+      echo "  down: Delete cluster."
+
+      exit 1
+fi
+
+COMMAND="${1-}"
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+nodes=${environment:-1}
+version=${version:-1.24.14}
+
+while [ $# -gt 0 ]; do
+
+   if [[ $1 == *"--"* ]]; then
+        param="${1/--/}"
+        declare $param="$2"
+        # echo $1 $2 // Optional to see the parameter:value result
+   fi
+
+  shift
+done
+
+case $COMMAND in
+  up)
+    k3d cluster create ${CLUSTER_NAME} \
+      --network ${NETWORK_NAME} \
+			--registry-config "$SCRIPT_DIR/registries.yaml" \
+			--image rancher/k3s:v${version}-k3s1 \
+			--agents ${nodes} \
+      --k3s-node-label "topology.kubernetes.io/region=us-east-1@agent:0" \
+      --k3s-node-label "topology.kubernetes.io/region=us-east-1@server:0" 
+  ;;
+  down)
+    k3d cluster delete ${CLUSTER_NAME}
+  ;;
+  *)
+    echo "Unknown command: $COMMAND"
+    exit 1;
+  ;;
+esac