diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8952f4f3..8dbf9900 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,8 +2,7 @@ name: Helm Chart Package CI on: pull_request: branches: - - 'master' - - 'cert-manager-feature-branch' + - '*' jobs: diff --git a/Makefile b/Makefile index d9ea3642..60aab980 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,7 @@ test/cluster/down: bin/k3d test/e2e/%: PKG=$* test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate) + @bin/kubectl get node -o yaml @PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/$(PKG)/... test/lint: bin/helm ## lint the helm chart diff --git a/go.mod b/go.mod index 23d3dbfd..355119e1 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/cockroachdb/cockroach-operator v0.0.0-20230531051823-2cb3e2e676f4 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible - github.com/gruntwork-io/terratest v0.41.19 + github.com/gruntwork-io/terratest v0.41.26 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/pkg/errors v0.9.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.51.2 diff --git a/go.sum b/go.sum index a850e1da..71f2eb55 100644 --- a/go.sum +++ b/go.sum @@ -341,8 +341,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRaxEM6G0ro= github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78= -github.com/gruntwork-io/terratest v0.41.19 h1:SCqYF28nHZuBlX+jZ+QI4bK+OLpPR8BgcxVUQxyQSEw= -github.com/gruntwork-io/terratest v0.41.19/go.mod h1:O6gajNBjO1wvc7Wl9WtbO+ORcdnhAV2GQiBE71ycwIk= +github.com/gruntwork-io/terratest v0.41.26 h1:ttDXBBDBAYV4KgP1itGQ5O61F6KwgMMUFHy64bzvuYU= +github.com/gruntwork-io/terratest v0.41.26/go.mod h1:O6gajNBjO1wvc7Wl9WtbO+ORcdnhAV2GQiBE71ycwIk= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= diff --git a/operator/templates/_operator_certs.tpl b/operator/templates/_operator_certs.tpl index aa03ec8e..f143ab4d 100644 --- a/operator/templates/_operator_certs.tpl +++ b/operator/templates/_operator_certs.tpl @@ -10,7 +10,7 @@ {{- define "operator.certs" -}} {{- $days := default .Values.certificate.validForDays 3650 | int -}} {{- $ca := genCA "cockroach-operator-certs" 3650 -}} -{{- $cert := genSignedCert "cert" nil (list "cockroach-webhook-service.default.svc" "cockroach-operator.default.svc") $days $ca -}} +{{- $cert := genSignedCert "cert" nil (list (printf "cockroach-webhook-service.%s.svc" .Release.Namespace) (printf "cockroach-operator.%s.svc" .Release.Namespace)) $days $ca -}} ca.crt: {{ $ca.Cert | b64enc }} tls.crt: {{ $cert.Cert | b64enc }} tls.key: {{ $cert.Key | b64enc }} diff --git a/operator/templates/cockroachdb-operator-certs.yaml b/operator/templates/cockroachdb-operator-certs.yaml index 44716a26..ba346aa5 100644 --- a/operator/templates/cockroachdb-operator-certs.yaml +++ b/operator/templates/cockroachdb-operator-certs.yaml @@ -3,9 +3,9 @@ apiVersion: v1 kind: Secret metadata: name: cockroach-operator-certs - namespace: default # Change the namespace if needed + namespace: {{ .Release.Namespace }} type: Opaque data: {{ index $operatorCerts 0 }} {{ index $operatorCerts 1 }} - {{ index $operatorCerts 2 }} \ No newline at end of file + {{ index $operatorCerts 2 }} diff --git a/operator/templates/operator.yaml b/operator/templates/operator.yaml index bbc1e655..9104fa81 100644 --- a/operator/templates/operator.yaml +++ b/operator/templates/operator.yaml @@ -380,7 +380,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: cockroach-operator-default - namespace: default + namespace: {{ .Release.Namespace }} labels: app: cockroach-operator @@ -397,7 +397,7 @@ roleRef: name: cockroach-operator-role subjects: - name: cockroach-operator-default - namespace: default + namespace: {{ .Release.Namespace }} kind: ServiceAccount --- @@ -406,7 +406,7 @@ apiVersion: v1 kind: Service metadata: name: cockroach-operator - namespace: default + namespace: {{ .Release.Namespace }} labels: app: cockroach-operator spec: @@ -424,7 +424,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: cockroach-operator - namespace: default + namespace: {{ .Release.Namespace }} labels: app: cockroach-operator spec: @@ -447,7 +447,7 @@ spec: priorityClassName: cockroach-operator containers: - name: cockroach-operator - image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:release-2024-07-10-0-384-g53408b608c + image: {{ .Values.image.registry }}/{{ .Values.image.repository }}@sha256:af1ad8c772bb38a5702db1cb4b04728f096bccbad09ccf785ebb04007de8f336 args: # Pin metrics port so it can be properly exposed by the "ports" # field below even in the event of a change to the default value. @@ -461,7 +461,7 @@ spec: - name: grpc containerPort: 9070 env: - - name: WATCH_NAMESPACE + - name: NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace @@ -512,17 +512,6 @@ spec: periodSeconds: 3 timeoutSeconds: 3 failureThreshold: 3 - # TODO(CC-27018) Use a node label that defines which nodes operator pods - # can run on, instead of blocking operator pods from running on some nodes. - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: crdb.io/app - operator: NotIn - values: - - "in-situ" volumes: - name: certs secret: diff --git a/tests/e2e/install/cockroachdb_helm_e2e_test.go b/tests/e2e/install/cockroachdb_helm_e2e_test.go index 5edf994a..bca180ea 100644 --- a/tests/e2e/install/cockroachdb_helm_e2e_test.go +++ b/tests/e2e/install/cockroachdb_helm_e2e_test.go @@ -1,6 +1,7 @@ package integration import ( + "encoding/json" "fmt" "io/fs" "log" @@ -25,16 +26,127 @@ import ( "github.com/cockroachdb/helm-charts/pkg/security" util "github.com/cockroachdb/helm-charts/pkg/utils" "github.com/cockroachdb/helm-charts/tests/testutil" + "github.com/gruntwork-io/terratest/modules/retry" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var ( - cfg = ctrl.GetConfigOrDie() - k8sClient, _ = client.New(cfg, client.Options{}) - releaseName = "crdb-test" - customCASecret = "custom-ca-secret" - helmChartPath, _ = filepath.Abs("../../../cockroachdb") + cfg = ctrl.GetConfigOrDie() + k8sClient, _ = client.New(cfg, client.Options{}) + releaseName = "crdb-test" + operatorReleaseName = "crdb-operator-test" + customCASecret = "custom-ca-secret" + helmChartPath, _ = filepath.Abs("../../../cockroachdb") + operatorChartPath, _ = filepath.Abs("../../../operator") + skipCleanup = os.Getenv("SKIP_CLEANUP") != "" ) +func mustMarshalJson(value interface{}) string { + out, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(out) +} + +func TestCockroachDBOperator(t *testing.T) { + namespaceName := fmt.Sprintf("cockroach-%s", strings.ToLower(t.Name())) + kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName) + + k8s.CreateNamespace(t, kubectlOptions, namespaceName) + if !skipCleanup { + defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName) + } + + const testDBName = "testdb" + + extraArgs := map[string][]string{ + "install": { + "--wait", + "--debug", + }, + } + + crdbCluster := testutil.CockroachCluster{ + Cfg: cfg, + K8sClient: k8sClient, + StatefulSetName: fmt.Sprintf("%s-cockroachdb", releaseName), + Namespace: namespaceName, + ClientSecret: fmt.Sprintf("%s-cockroachdb-client-secret", releaseName), + NodeSecret: fmt.Sprintf("%s-cockroachdb-node-secret", releaseName), + CaSecret: fmt.Sprintf("%s-cockroachdb-ca-secret", releaseName), + IsCaUserProvided: false, + DesiredNodes: 1, + } + + // Deploy operator + operatorOpts := &helm.Options{ + KubectlOptions: kubectlOptions, + ExtraArgs: extraArgs, + } + helm.Install(t, operatorOpts, operatorChartPath, operatorReleaseName) + if !skipCleanup { + defer cleanupResources( + t, + operatorReleaseName, + kubectlOptions, + operatorOpts, + []string{}, + ) + } + + // Wait for crd to be installed + k8s.WaitUntilServiceAvailable(t, kubectlOptions, "cockroach-operator", 30, 2*time.Second) + retry.DoWithRetryE(t, "wait-for-crd", 60, time.Second*5, func() (string, error) { + return k8s.RunKubectlAndGetOutputE(t, operatorOpts.KubectlOptions, "get", "crd", "crdbclusters.crdb.cockroachlabs.com") + }) + + // Deploy crdb + crdbOpts := &helm.Options{ + KubectlOptions: kubectlOptions, + SetValues: patchHelmValues(map[string]string{ + "operator.enabled": "true", + "operator.dataStore.volumeClaimTemplate.spec.resources.requests.storage": "1Gi", + }), + SetJsonValues: map[string]string{ + "operator.regions": mustMarshalJson([]map[string]interface{}{ + { + "code": "us-east-1", + "cloudProvider": "k3d", + "nodes": crdbCluster.DesiredNodes, + "namespace": namespaceName, + }, + }), + }, + ExtraArgs: extraArgs, + } + helm.Install(t, crdbOpts, helmChartPath, releaseName) + if !skipCleanup { + defer cleanupResources( + t, + releaseName, + kubectlOptions, + crdbOpts, + []string{}, + ) + } + + serviceName := fmt.Sprintf("%s-cockroachdb-public", releaseName) + k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 30, 2*time.Second) + + testutil.RequireCertificatesToBeValid(t, crdbCluster) + testutil.RequireCRDBClusterToBeReadyTimeout(t, kubectlOptions, crdbCluster, 600*time.Second) + + pods := k8s.ListPods(t, kubectlOptions, metav1.ListOptions{ + LabelSelector: "app=cockroachdb", + }) + require.True(t, len(pods) > 0) + podName := fmt.Sprintf("%s.%s-cockroachdb", pods[0].Name, releaseName) + + testutil.RequireCRDBClusterToFunction(t, crdbCluster, false, podName) + testutil.RequireCRDBDatabaseToFunction(t, crdbCluster, testDBName, podName) +} + func TestCockroachDbHelmInstall(t *testing.T) { namespaceName := "cockroach" + strings.ToLower(random.UniqueId()) kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName) @@ -212,8 +324,10 @@ func TestCockroachDbHelmMigration(t *testing.T) { cmdCa := shell.Command{ Command: "cockroach", - Args: []string{"cert", "create-ca", fmt.Sprintf("--certs-dir=%s", certsDir), - fmt.Sprintf("--ca-key=%s/ca.key", certsDir)}, + Args: []string{ + "cert", "create-ca", fmt.Sprintf("--certs-dir=%s", certsDir), + fmt.Sprintf("--ca-key=%s/ca.key", certsDir), + }, WorkingDir: ".", Env: nil, Logger: nil, @@ -243,8 +357,10 @@ func TestCockroachDbHelmMigration(t *testing.T) { cmdClient := shell.Command{ Command: "cockroach", - Args: []string{"cert", "create-client", security.RootUser, fmt.Sprintf("--certs-dir=%s", certsDir), - fmt.Sprintf("--ca-key=%s/ca.key", certsDir)}, + Args: []string{ + "cert", "create-client", security.RootUser, fmt.Sprintf("--certs-dir=%s", certsDir), + fmt.Sprintf("--ca-key=%s/ca.key", certsDir), + }, WorkingDir: ".", Env: nil, Logger: nil, @@ -313,7 +429,7 @@ func TestCockroachDbHelmMigration(t *testing.T) { "statefulset.updateStrategy.type": "OnDelete", }), ExtraArgs: map[string][]string{ - "upgrade": []string{ + "upgrade": { "--timeout=20m", }, }, diff --git a/tests/k3d/dev-cluster.sh b/tests/k3d/dev-cluster.sh index 75d9919b..d16313a1 100755 --- a/tests/k3d/dev-cluster.sh +++ b/tests/k3d/dev-cluster.sh @@ -4,35 +4,34 @@ zones=3 K3D_PATH="./bin/k3d" -if [ $# -eq 0 ] - then - echo "No arguments supplied: " - echo " up: Start cluster." - echo " --nodes x: The cluster should have x nodes (default 1)" - echo " --version x: The version of Kubernetes (default 1.24.14)" - echo " --name x: The name of the cluster (default local)" - echo " --network_name x: The name of the cluster's network (default k3d-\${name})" - echo " --region x: The name of the cluster's region for node labels topology.kubernetes.io/region (default us-east-1)" - echo " --zones x: The number of zones in the region for node labels topology.kubernetes.io/zone (default 3)" - - echo " down: Delete cluster." +if [ $# -eq 0 ]; then + echo "No arguments supplied: " + echo " up: Start cluster." + echo " --nodes x: The cluster should have x nodes (default 1)" + echo " --version x: The version of Kubernetes (default 1.24.14)" + echo " --name x: The name of the cluster (default local)" + echo " --network_name x: The name of the cluster's network (default k3d-\${name})" + echo " --region x: The name of the cluster's region for node labels topology.kubernetes.io/region (default us-east-1)" + echo " --zones x: The number of zones in the region for node labels topology.kubernetes.io/zone (default 3)" - exit 1 + echo " down: Delete cluster." + + exit 1 fi COMMAND="${1-}" -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) nodes=${environment:-1} -version=${version:-1.24.14} +version=${version:-1.31.2} while [ $# -gt 0 ]; do - if [[ $1 == *"--"* ]]; then - param="${1/--/}" - declare $param="$2" - # echo $1 $2 // Optional to see the parameter:value result - fi + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + # echo $1 $2 // Optional to see the parameter:value result + fi shift done @@ -48,7 +47,7 @@ set_node_labels() { local labels="" local az=(a b c d e f g h i j k l m n o p q r s t u v w x y z) - for ((i=0; i