test(mc): Add module for Prometheus, refactor, add tests (#1294)

# Description

* add module for Prometheus deployment
* add example and test for Prometheus module
* add tests for each testUtils.go functions
* refactor

## Related Issue

#1267 

## Checklist

- [x] I have read the [contributing
documentation](https://retina.sh/docs/contributing).
- [x] I signed and signed-off the commits (`git commit -S -s ...`). See
[this
documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification)
on signing commits.
- [x] I have correctly attributed the author(s) of the code.
- [x] I have tested the changes locally.
- [x] I have followed the project's style guidelines.
- [x] I have updated the documentation, if necessary.
- [x] I have added tests, if applicable.

## Screenshots (if applicable) or Testing Completed

Please add any relevant screenshots or GIFs to showcase the changes
made.

## Additional Notes

Add any additional notes or context about the pull request here.

---

Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more
information on how to contribute to this project.

test/multicloud/Makefile

@@ -31,9 +31,11 @@ destroy:
 	cd live/$(STACK_NAME) && \
 		tofu destroy --auto-approve
 
-clean: destroy
-	@cd live/$(STACK_NAME) && \
-		rm -rf .terraform && rm terraform.tfstate && rm terraform.tfstate.backup
+# !! DANGER remove state, backup, kubeconfig files and .terraform directories
+clean:
+	@find . -name '*.tfstate*' -delete
+	@find . -name '*-kind-config*' -delete
+	@find . -name '*.terraform' -type d -exec rm -rf {} +
 
 kind-kubeconfig:
 	@kubectl config set-context live/$(PREFIX)-kind/mc-kind-config

test/multicloud/examples/integration/prometheus-kind/main.tf

@@ -0,0 +1,9 @@
+module "kind" {
+  source = "../../../modules/kind"
+  prefix = var.prefix
+}
+
+module "prometheus" {
+  depends_on = [module.kind]
+  source     = "../../../modules/prometheus"
+}

test/multicloud/examples/integration/prometheus-kind/outputs.tf

@@ -0,0 +1,19 @@
+output "host" {
+  value     = module.kind.host
+  sensitive = true
+}
+
+output "cluster_ca_certificate" {
+  value     = module.kind.cluster_ca_certificate
+  sensitive = true
+}
+
+output "client_certificate" {
+  value     = module.kind.client_certificate
+  sensitive = true
+}
+
+output "client_key" {
+  value     = module.kind.client_key
+  sensitive = true
+}

test/multicloud/examples/integration/prometheus-kind/providers.tf

@@ -0,0 +1,26 @@
+terraform {
+  required_version = "1.8.3"
+  required_providers {
+    kind = {
+      source  = "tehcyx/kind"
+      version = "0.7.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.17.0"
+    }
+  }
+}
+
+# Initialize the kind provider
+provider "kind" {}
+
+# Initialize the Helm provider
+provider "helm" {
+  kubernetes {
+    host                   = module.kind.host
+    client_certificate     = module.kind.client_certificate
+    client_key             = module.kind.client_key
+    cluster_ca_certificate = module.kind.cluster_ca_certificate
+  }
+}

test/multicloud/examples/integration/prometheus-kind/variables.tf

@@ -0,0 +1,5 @@
+variable "prefix" {
+  description = "A prefix to add to all resources."
+  type        = string
+  default     = "mc"
+}

test/multicloud/modules/prometheus/main.tf

@@ -0,0 +1,14 @@
+resource "helm_release" "prometheus" {
+  name       = "prometheus"
+  repository = "https://prometheus-community.github.io/helm-charts"
+  chart      = "kube-prometheus-stack"
+  version    = var.prometheus_version
+
+  dynamic "set" {
+    for_each = var.values
+    content {
+      name  = set.value.name
+      value = set.value.value
+    }
+  }
+}

test/multicloud/modules/prometheus/provider.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.17.0"
+    }
+  }
+}

test/multicloud/modules/prometheus/variables.tf

@@ -0,0 +1,23 @@
+variable "prometheus_version" {
+  description = "The Prometheus version to install."
+  type        = string
+  default     = "68.4.3"
+}
+
+variable "values" {
+  description = "Configuration for set blocks, this corresponds to Helm values.yaml"
+  type = list(object({
+    name  = string
+    value = string
+  }))
+  default = [
+    {
+      name  = "global.prometheus.enabled"
+      value = "true"
+    },
+    {
+      name  = "global.grafana.enabled"
+      value = "true"
+    }
+  ]
+}

test/multicloud/modules/retina/variables.tf

@@ -1,5 +1,5 @@
 variable "retina_version" {
-  description = "The tag to apply to all resources."
+  description = "The Retina version to install."
   type        = string
   default     = "v0.0.23"
 }

test/multicloud/test/example_aks_test.go

@@ -10,7 +10,7 @@ func TestAKSExample(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: "../examples/aks",
+		TerraformDir: examplesPath + "aks",
 
 		Vars: map[string]interface{}{
 			"prefix":              "test-mc",

test/multicloud/test/example_gke_test.go

@@ -10,7 +10,7 @@ func TestGKEExample(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: "../examples/gke",
+		TerraformDir: examplesPath + "gke",
 
 		Vars: map[string]interface{}{
 			"prefix":       "test",

test/multicloud/test/example_kind_test.go

@@ -10,7 +10,7 @@ func TestKindExample(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: "../examples/kind",
+		TerraformDir: examplesPath + "kind",
 
 		Vars: map[string]interface{}{
 			"prefix": "test",

test/multicloud/test/integration_prometheus_kind_test.go

@@ -0,0 +1,60 @@
+package test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+func TestPrometheusKindIntegration(t *testing.T) {
+	t.Parallel()
+
+	opts := &terraform.Options{
+		TerraformDir: examplesPath + "integration/prometheus-kind",
+
+		Vars: map[string]interface{}{
+			"prefix": "test-integration",
+		},
+	}
+
+	// clean up at the end of the test
+	defer terraform.Destroy(t, opts)
+	terraform.InitAndApply(t, opts)
+
+	// get outputs
+	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := fetchSensitiveOutput(t, opts, "client_key")
+	host := fetchSensitiveOutput(t, opts, "host")
+
+	// build the REST config
+	restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
+
+	// create a Kubernetes clientset
+	clientSet, err := buildClientSet(restConfig)
+	if err != nil {
+		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
+
+	// test the cluster is accessible
+	testClusterAccess(t, clientSet)
+
+	podSelector := PodSelector{
+		Namespace:     "default",
+		LabelSelector: "app.kubernetes.io/instance=prometheus-kube-prometheus-prometheus",
+		ContainerName: "prometheus",
+	}
+
+	timeOut := time.Duration(60) * time.Second
+	// check the prometheus pods are running
+	result, err := arePodsRunning(clientSet, podSelector, timeOut)
+	if !result {
+		t.Fatalf("Prometheus pods did not start in time: %v\n", err)
+	}
+
+	// check the retina pods logs for errors
+	checkPodLogs(t, clientSet, podSelector)
+
+	// TODO: add more tests here
+}

test/multicloud/test/integration_retina_kind_test.go

@@ -2,6 +2,7 @@ package test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
 )
@@ -10,7 +11,7 @@ func TestRetinaKindIntegration(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: "../examples/integration/retina-kind",
+		TerraformDir: examplesPath + "integration/retina-kind",
 
 		Vars: map[string]interface{}{
 			"prefix":         "test-integration",
@@ -37,14 +38,24 @@ func TestRetinaKindIntegration(t *testing.T) {
 		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
 	}
 
-	// check the retina pods are running
-	checkRetinaPodsRunning(t, clientSet)
-
 	// test the cluster is accessible
 	testClusterAccess(t, clientSet)
 
+	retinaPodSelector := PodSelector{
+		Namespace:     "kube-system",
+		LabelSelector: "k8s-app=retina",
+		ContainerName: "retina",
+	}
+
+	timeOut := time.Duration(90) * time.Second
+	// check the retina pods are running
+	result, err := arePodsRunning(clientSet, retinaPodSelector, timeOut)
+	if !result {
+		t.Fatalf("Retina pods did not start in time: %v\n", err)
+	}
+
 	// check the retina pods logs for errors
-	checkRetinaLogs(t, clientSet)
+	checkPodLogs(t, clientSet, retinaPodSelector)
 
 	// TODO: add more tests here
 }

test/multicloud/test/types.go

@@ -0,0 +1,9 @@
+package test
+
+const examplesPath = "../examples/"
+
+type PodSelector struct {
+	Namespace     string
+	LabelSelector string
+	ContainerName string
+}