From 878769c2ff2826174730b28c4c2b886460d831b5 Mon Sep 17 00:00:00 2001
From: Simone Rodigari <srodigari@microsoft.com>
Date: Tue, 4 Feb 2025 09:40:35 +0000
Subject: [PATCH 1/3] test(mc): Add integration test for Retina-GKE, refactor
 test dir

---
 test/multicloud/Makefile                      |  2 +-
 test/multicloud/README.md                     | 16 ++++
 .../retina-gke/.terraform.lock.hcl            | 38 +++++++++
 .../examples/integration/retina-gke/main.tf   | 14 ++++
 .../integration/retina-gke/outputs.tf         | 14 ++++
 .../integration/retina-gke/providers.tf       | 30 +++++++
 .../integration/retina-gke/variables.tf       | 36 +++++++++
 test/multicloud/modules/gke/main.tf           |  1 +
 test/multicloud/test/example_kind_test.go     | 41 ----------
 .../prometheus_kind_test.go}                  | 23 +++---
 .../test/integration/retina_gke_test.go       | 81 +++++++++++++++++++
 .../retina_kind_test.go}                      | 25 +++---
 .../{example_aks_test.go => unit/aks_test.go} | 23 +++---
 .../{example_gke_test.go => unit/gke_test.go} | 17 ++--
 test/multicloud/test/unit/kind_test.go        | 42 ++++++++++
 test/multicloud/test/{ => utils}/types.go     |  7 +-
 test/multicloud/test/{ => utils}/utils.go     | 18 ++---
 .../multicloud/test/{ => utils}/utils_test.go | 20 ++---
 18 files changed, 343 insertions(+), 105 deletions(-)
 create mode 100644 test/multicloud/examples/integration/retina-gke/.terraform.lock.hcl
 create mode 100644 test/multicloud/examples/integration/retina-gke/main.tf
 create mode 100644 test/multicloud/examples/integration/retina-gke/outputs.tf
 create mode 100644 test/multicloud/examples/integration/retina-gke/providers.tf
 create mode 100644 test/multicloud/examples/integration/retina-gke/variables.tf
 delete mode 100644 test/multicloud/test/example_kind_test.go
 rename test/multicloud/test/{integration_prometheus_kind_test.go => integration/prometheus_kind_test.go} (57%)
 create mode 100644 test/multicloud/test/integration/retina_gke_test.go
 rename test/multicloud/test/{integration_retina_kind_test.go => integration/retina_kind_test.go} (53%)
 rename test/multicloud/test/{example_aks_test.go => unit/aks_test.go} (52%)
 rename test/multicloud/test/{example_gke_test.go => unit/gke_test.go} (60%)
 create mode 100644 test/multicloud/test/unit/kind_test.go
 rename test/multicloud/test/{ => utils}/types.go (52%)
 rename test/multicloud/test/{ => utils}/utils.go (89%)
 rename test/multicloud/test/{ => utils}/utils_test.go (93%)

diff --git a/test/multicloud/Makefile b/test/multicloud/Makefile
index 9f5db6ad5c..64a5e25a10 100644
--- a/test/multicloud/Makefile
+++ b/test/multicloud/Makefile
@@ -45,7 +45,7 @@ kind-kubeconfig:
 # Once we do this targets will be updated to
 # @cd test && go test -v -count=1 -timeout 30m ./...
 test:
-	@cd test && go test -run TestRetinaKindIntegration -count=1 -timeout 10m
+	@cd test/integration && go test -run TestRetinaKindIntegration -count=1 -timeout 20m
 
 fmt:
 	@tofu fmt -recursive
diff --git a/test/multicloud/README.md b/test/multicloud/README.md
index f82939a532..f606accde4 100644
--- a/test/multicloud/README.md
+++ b/test/multicloud/README.md
@@ -96,3 +96,19 @@ make test
 * [GKE resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster)
 * [AKS resource documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster)
 * [Kind resource documentation](https://registry.terraform.io/providers/tehcyx/kind/latest/docs/resources/cluster)
+
+
+## Troubleshooting
+
+In case the test fails due to timeout, validate the resource was created by the provider, and if it is, you can import into OpenTofu state.
+
+Here is an example on how to import resources for `modules/gke`
+
+```sh
+# move to the stack directory
+# i.e. examples/gke
+tofu import module.gke.google_container_cluster.gke europe-west2/test-gke-cluster
+tofu import module.gke.google_service_account.default projects/mc-retina/serviceAccounts/test-gke-service-account@mc-retina.iam.gserviceaccount.com
+```
+
+>Note: each resource documentation contains a section on how to import resources into the State. [Example for google_container_cluster resource](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#import)
diff --git a/test/multicloud/examples/integration/retina-gke/.terraform.lock.hcl b/test/multicloud/examples/integration/retina-gke/.terraform.lock.hcl
new file mode 100644
index 0000000000..b74b1a1ec9
--- /dev/null
+++ b/test/multicloud/examples/integration/retina-gke/.terraform.lock.hcl
@@ -0,0 +1,38 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/hashicorp/google" {
+  version     = "6.17.0"
+  constraints = "6.17.0"
+  hashes = [
+    "h1:aZkLSXbqbNThCCLAX1x0g8KTJANQAIosYq3xpy8JhFQ=",
+    "zh:0614cc52deb5914795253aecf19b4cbb5aa7e8a186839a33907ce5c35e23d537",
+    "zh:0b28ea31ec3b119aafc4c37a5992c29266c876db288dfc5bbfbde36631a533f1",
+    "zh:1b2c5df40ac55ec7c3db2b7c556ace7545fca4ccfacf605b7588a4e2be564ba8",
+    "zh:1db9a278cfddcaa7d9119acf231164438df07932bdabce95931a68bd3689cd39",
+    "zh:3729b7a9936f5ad545f7d06d11d05ce78b4bbe9941e77bf004a2b798e6a0866c",
+    "zh:3b7e35183a8d7980207ae7d082c490d7c2270f907e5de9e8393c6c4c32b07b3c",
+    "zh:581e14963dfef608285af0c08ccb5da4bdb8ae049366418d4863a4fd9fa55b74",
+    "zh:89d29f2d1269a30c6149bbaaae3cbe8fe0c6ed13874b77bf784ea6bbd73aee58",
+    "zh:c7c690c1f9fe0cbc69d4eb21727f2cf4b7ba016385184d27527d570832c393be",
+    "zh:dc1536fc325a0561bf0f606a38bed29e8dd2f4bec6d6e8a5301696d89c1c8079",
+  ]
+}
+
+provider "registry.opentofu.org/hashicorp/helm" {
+  version     = "2.17.0"
+  constraints = "2.17.0"
+  hashes = [
+    "h1:69PnHoYrrDrm7C8+8PiSvRGPI55taqL14SvQR/FGM+g=",
+    "zh:02690815e35131a42cb9851f63a3369c216af30ad093d05b39001d43da04b56b",
+    "zh:27a62f12b29926387f4d71aeeee9f7ffa0ccb81a1b6066ee895716ad050d1b7a",
+    "zh:2d0a5babfa73604b3fefc9dab9c87f91c77fce756c2e32b294e9f1290aed26c0",
+    "zh:3976400ceba6dda4636e1d297e3097e1831de5628afa534a166de98a70d1dcbe",
+    "zh:54440ef14f342b41d75c1aded7487bfcc3f76322b75894235b47b7e89ac4bfa4",
+    "zh:6512e2ab9f2fa31cbb90d9249647b5c5798f62eb1215ec44da2cdaa24e38ad25",
+    "zh:795f327ca0b8c5368af0ed03d5d4f6da7260692b4b3ca0bd004ed542e683464d",
+    "zh:ba659e1d94f224bc3f1fd34cbb9d2663e3a8e734108e5a58eb49eda84b140978",
+    "zh:c5c8575c4458835c2acbc3d1ed5570589b14baa2525d8fbd04295c097caf41eb",
+    "zh:e0877a5dac3de138e61eefa26b2f5a13305a17259779465899880f70e11314e0",
+  ]
+}
diff --git a/test/multicloud/examples/integration/retina-gke/main.tf b/test/multicloud/examples/integration/retina-gke/main.tf
new file mode 100644
index 0000000000..5229b21294
--- /dev/null
+++ b/test/multicloud/examples/integration/retina-gke/main.tf
@@ -0,0 +1,14 @@
+module "gke" {
+  source       = "../../../modules/gke"
+  location     = var.location
+  prefix       = var.prefix
+  project      = var.project
+  machine_type = var.machine_type
+}
+
+module "retina" {
+  depends_on = [module.gke]
+  source     = "../../../modules/retina"
+  retina_version = var.retina_version
+  values = var.values
+}
diff --git a/test/multicloud/examples/integration/retina-gke/outputs.tf b/test/multicloud/examples/integration/retina-gke/outputs.tf
new file mode 100644
index 0000000000..7586d8324e
--- /dev/null
+++ b/test/multicloud/examples/integration/retina-gke/outputs.tf
@@ -0,0 +1,14 @@
+output "host" {
+  value     = module.gke.host
+  sensitive = true
+}
+
+output "cluster_ca_certificate" {
+  value     = module.gke.cluster_ca_certificate
+  sensitive = true
+}
+
+output "access_token" {
+  value     = data.google_client_config.current.access_token
+  sensitive = true
+}
diff --git a/test/multicloud/examples/integration/retina-gke/providers.tf b/test/multicloud/examples/integration/retina-gke/providers.tf
new file mode 100644
index 0000000000..ac23c59c0e
--- /dev/null
+++ b/test/multicloud/examples/integration/retina-gke/providers.tf
@@ -0,0 +1,30 @@
+terraform {
+  required_version = "1.8.3"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.17.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.17.0"
+    }
+  }
+}
+
+# Initialize the Google provider
+provider "google" {
+  project = var.project
+  region  = var.location
+}
+
+data "google_client_config" "current" {}
+
+# Initialize the Helm provider
+provider "helm" {
+  kubernetes {
+    token                  = data.google_client_config.current.access_token
+    host                   = module.gke.host
+    cluster_ca_certificate = base64decode(module.gke.cluster_ca_certificate)
+  }
+}
\ No newline at end of file
diff --git a/test/multicloud/examples/integration/retina-gke/variables.tf b/test/multicloud/examples/integration/retina-gke/variables.tf
new file mode 100644
index 0000000000..b588e97454
--- /dev/null
+++ b/test/multicloud/examples/integration/retina-gke/variables.tf
@@ -0,0 +1,36 @@
+variable "project" {
+  description = "The Google Cloud project where resources will be deployed."
+  type        = string
+  default     = "mc-retina"
+}
+
+variable "location" {
+  description = "The Google Cloud location where GKE will be deployed to."
+  type        = string
+  default     = "eu-west2"
+}
+
+variable "prefix" {
+  description = "A prefix to add to all resources."
+  type        = string
+  default     = "mc"
+}
+
+variable "machine_type" {
+  description = "The machine type to use for the GKE nodes."
+  type        = string
+  default     = "e2-standard-4"
+}
+
+variable "retina_version" {
+  description = "The tag to apply to all resources."
+  type        = string
+}
+
+variable "values" {
+  description = "Configuration for set blocks, this corresponds to Helm values.yaml"
+  type = list(object({
+    name  = string
+    value = string
+  }))
+}
diff --git a/test/multicloud/modules/gke/main.tf b/test/multicloud/modules/gke/main.tf
index c4615b7858..8de256d40e 100644
--- a/test/multicloud/modules/gke/main.tf
+++ b/test/multicloud/modules/gke/main.tf
@@ -3,6 +3,7 @@ resource "google_service_account" "default" {
   display_name = "GKE Service Account for ${var.project}"
 }
 
+// https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview
 resource "google_container_cluster" "gke" {
   name     = "${var.prefix}-gke-cluster"
   location = var.location
diff --git a/test/multicloud/test/example_kind_test.go b/test/multicloud/test/example_kind_test.go
deleted file mode 100644
index 9382ae4263..0000000000
--- a/test/multicloud/test/example_kind_test.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package test
-
-import (
-	"testing"
-
-	"github.com/gruntwork-io/terratest/modules/terraform"
-)
-
-func TestKindExample(t *testing.T) {
-	t.Parallel()
-
-	opts := &terraform.Options{
-		TerraformDir: examplesPath + "kind",
-
-		Vars: map[string]interface{}{
-			"prefix": "test",
-		},
-	}
-
-	// clean up at the end of the test
-	defer terraform.Destroy(t, opts)
-	terraform.InitAndApply(t, opts)
-
-	// get outputs
-	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
-	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
-	clientKey := fetchSensitiveOutput(t, opts, "client_key")
-	host := fetchSensitiveOutput(t, opts, "host")
-
-	// build the REST config
-	restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
-
-	// create a Kubernetes clientset
-	clientSet, err := buildClientSet(restConfig)
-	if err != nil {
-		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
-	}
-
-	// test the cluster is accessible
-	testClusterAccess(t, clientSet)
-}
diff --git a/test/multicloud/test/integration_prometheus_kind_test.go b/test/multicloud/test/integration/prometheus_kind_test.go
similarity index 57%
rename from test/multicloud/test/integration_prometheus_kind_test.go
rename to test/multicloud/test/integration/prometheus_kind_test.go
index 7c7f1f2f50..12071bfb5c 100644
--- a/test/multicloud/test/integration_prometheus_kind_test.go
+++ b/test/multicloud/test/integration/prometheus_kind_test.go
@@ -5,13 +5,14 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
 )
 
 func TestPrometheusKindIntegration(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: examplesPath + "integration/prometheus-kind",
+		TerraformDir: utils.ExamplesPath + "integration/prometheus-kind",
 
 		Vars: map[string]interface{}{
 			"prefix": "test-integration",
@@ -23,24 +24,24 @@ func TestPrometheusKindIntegration(t *testing.T) {
 	terraform.InitAndApply(t, opts)
 
 	// get outputs
-	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
-	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
-	clientKey := fetchSensitiveOutput(t, opts, "client_key")
-	host := fetchSensitiveOutput(t, opts, "host")
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := utils.FetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := utils.FetchSensitiveOutput(t, opts, "client_key")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
 
 	// build the REST config
-	restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
+	restConfig := utils.CreateRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
 
 	// create a Kubernetes clientset
-	clientSet, err := buildClientSet(restConfig)
+	clientSet, err := utils.BuildClientSet(restConfig)
 	if err != nil {
 		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
 	}
 
 	// test the cluster is accessible
-	testClusterAccess(t, clientSet)
+	utils.TestClusterAccess(t, clientSet)
 
-	podSelector := PodSelector{
+	podSelector := utils.PodSelector{
 		Namespace:     "default",
 		LabelSelector: "app.kubernetes.io/instance=prometheus-kube-prometheus-prometheus",
 		ContainerName: "prometheus",
@@ -48,13 +49,13 @@ func TestPrometheusKindIntegration(t *testing.T) {
 
 	timeOut := time.Duration(60) * time.Second
 	// check the prometheus pods are running
-	result, err := arePodsRunning(clientSet, podSelector, timeOut)
+	result, err := utils.ArePodsRunning(clientSet, podSelector, timeOut)
 	if !result {
 		t.Fatalf("Prometheus pods did not start in time: %v\n", err)
 	}
 
 	// check the retina pods logs for errors
-	checkPodLogs(t, clientSet, podSelector)
+	utils.CheckPodLogs(t, clientSet, podSelector)
 
 	// TODO: add more tests here
 }
diff --git a/test/multicloud/test/integration/retina_gke_test.go b/test/multicloud/test/integration/retina_gke_test.go
new file mode 100644
index 0000000000..69510740b5
--- /dev/null
+++ b/test/multicloud/test/integration/retina_gke_test.go
@@ -0,0 +1,81 @@
+package test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
+)
+
+func TestRetinaGKEIntegration(t *testing.T) {
+	t.Parallel()
+
+	opts := &terraform.Options{
+		TerraformDir: utils.ExamplesPath + "integration/retina-gke",
+
+		Vars: map[string]interface{}{
+			"prefix":         "test",
+			"location":       "europe-west2", // London
+			"project":        "mc-retina",    // TODO: replace with actual project once we get gcloud access
+			"machine_type":   "e2-standard-4",
+			"retina_version": utils.RetinaVersion,
+			"values": []map[string]interface{}{
+				{
+					"name":  "logLevel",
+					"value": "info",
+				},
+				{
+					"name":  "operator.tag",
+					"value": utils.RetinaVersion,
+				},
+				{
+					"name":  "image.tag",
+					"value": utils.RetinaVersion,
+				},
+			},
+		},
+	}
+
+	// clean up at the end of the test
+	// defer terraform.Destroy(t, opts)
+	terraform.InitAndApply(t, opts)
+
+	// get outputs
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
+	token := utils.FetchSensitiveOutput(t, opts, "access_token")
+
+	// decode the base64 encoded cert
+	caCertString := utils.DecodeBase64(t, caCert)
+
+	// build the REST config
+	restConfig := utils.CreateRESTConfigWithBearer(caCertString, token, host)
+
+	// create a Kubernetes clientset
+	clientSet, err := utils.BuildClientSet(restConfig)
+	if err != nil {
+		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
+
+	// test the cluster is accessible
+	utils.TestClusterAccess(t, clientSet)
+
+	retinaPodSelector := utils.PodSelector{
+		Namespace:     "kube-system",
+		LabelSelector: "k8s-app=retina",
+		ContainerName: "retina",
+	}
+
+	timeOut := time.Duration(90) * time.Second
+	// check the retina pods are running
+	result, err := utils.ArePodsRunning(clientSet, retinaPodSelector, timeOut)
+	if !result {
+		t.Fatalf("Retina pods did not start in time: %v\n", err)
+	}
+
+	// check the retina pods logs for errors
+	utils.CheckPodLogs(t, clientSet, retinaPodSelector)
+
+	// TODO: add more tests here
+}
diff --git a/test/multicloud/test/integration_retina_kind_test.go b/test/multicloud/test/integration/retina_kind_test.go
similarity index 53%
rename from test/multicloud/test/integration_retina_kind_test.go
rename to test/multicloud/test/integration/retina_kind_test.go
index 1deae01805..28db20f721 100644
--- a/test/multicloud/test/integration_retina_kind_test.go
+++ b/test/multicloud/test/integration/retina_kind_test.go
@@ -5,17 +5,18 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
 )
 
 func TestRetinaKindIntegration(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: examplesPath + "integration/retina-kind",
+		TerraformDir: utils.ExamplesPath + "integration/retina-kind",
 
 		Vars: map[string]interface{}{
 			"prefix":         "test-integration",
-			"retina_version": "v0.0.24",
+			"retina_version": utils.RetinaVersion,
 		},
 	}
 
@@ -24,24 +25,24 @@ func TestRetinaKindIntegration(t *testing.T) {
 	terraform.InitAndApply(t, opts)
 
 	// get outputs
-	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
-	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
-	clientKey := fetchSensitiveOutput(t, opts, "client_key")
-	host := fetchSensitiveOutput(t, opts, "host")
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := utils.FetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := utils.FetchSensitiveOutput(t, opts, "client_key")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
 
 	// build the REST config
-	restConfig := createRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
+	restConfig := utils.CreateRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
 
 	// create a Kubernetes clientset
-	clientSet, err := buildClientSet(restConfig)
+	clientSet, err := utils.BuildClientSet(restConfig)
 	if err != nil {
 		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
 	}
 
 	// test the cluster is accessible
-	testClusterAccess(t, clientSet)
+	utils.TestClusterAccess(t, clientSet)
 
-	retinaPodSelector := PodSelector{
+	retinaPodSelector := utils.PodSelector{
 		Namespace:     "kube-system",
 		LabelSelector: "k8s-app=retina",
 		ContainerName: "retina",
@@ -49,13 +50,13 @@ func TestRetinaKindIntegration(t *testing.T) {
 
 	timeOut := time.Duration(90) * time.Second
 	// check the retina pods are running
-	result, err := arePodsRunning(clientSet, retinaPodSelector, timeOut)
+	result, err := utils.ArePodsRunning(clientSet, retinaPodSelector, timeOut)
 	if !result {
 		t.Fatalf("Retina pods did not start in time: %v\n", err)
 	}
 
 	// check the retina pods logs for errors
-	checkPodLogs(t, clientSet, retinaPodSelector)
+	utils.CheckPodLogs(t, clientSet, retinaPodSelector)
 
 	// TODO: add more tests here
 }
diff --git a/test/multicloud/test/example_aks_test.go b/test/multicloud/test/unit/aks_test.go
similarity index 52%
rename from test/multicloud/test/example_aks_test.go
rename to test/multicloud/test/unit/aks_test.go
index dbd4bd77a0..c379a094d0 100644
--- a/test/multicloud/test/example_aks_test.go
+++ b/test/multicloud/test/unit/aks_test.go
@@ -4,13 +4,14 @@ import (
 	"testing"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
 )
 
 func TestAKSExample(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: examplesPath + "aks",
+		TerraformDir: utils.ExamplesPath + "aks",
 
 		Vars: map[string]interface{}{
 			"prefix":              "test-mc",
@@ -29,25 +30,25 @@ func TestAKSExample(t *testing.T) {
 	terraform.InitAndApply(t, opts)
 
 	// get outputs
-	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
-	clientCert := fetchSensitiveOutput(t, opts, "client_certificate")
-	clientKey := fetchSensitiveOutput(t, opts, "client_key")
-	host := fetchSensitiveOutput(t, opts, "host")
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := utils.FetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := utils.FetchSensitiveOutput(t, opts, "client_key")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
 
 	// decode the base64 encoded strings
-	caCertDecoded := decodeBase64(t, caCert)
-	clientCertDecoded := decodeBase64(t, clientCert)
-	clientKeyDecoded := decodeBase64(t, clientKey)
+	caCertDecoded := utils.DecodeBase64(t, caCert)
+	clientCertDecoded := utils.DecodeBase64(t, clientCert)
+	clientKeyDecoded := utils.DecodeBase64(t, clientKey)
 
 	// build the REST config
-	restConfig := createRESTConfigWithClientCert(caCertDecoded, clientCertDecoded, clientKeyDecoded, host)
+	restConfig := utils.CreateRESTConfigWithClientCert(caCertDecoded, clientCertDecoded, clientKeyDecoded, host)
 
 	// create a Kubernetes clientset
-	clientSet, err := buildClientSet(restConfig)
+	clientSet, err := utils.BuildClientSet(restConfig)
 	if err != nil {
 		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
 	}
 
 	// test the cluster is accessible
-	testClusterAccess(t, clientSet)
+	utils.TestClusterAccess(t, clientSet)
 }
diff --git a/test/multicloud/test/example_gke_test.go b/test/multicloud/test/unit/gke_test.go
similarity index 60%
rename from test/multicloud/test/example_gke_test.go
rename to test/multicloud/test/unit/gke_test.go
index b2443ea1cb..2629d61a94 100644
--- a/test/multicloud/test/example_gke_test.go
+++ b/test/multicloud/test/unit/gke_test.go
@@ -4,13 +4,14 @@ import (
 	"testing"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
 )
 
 func TestGKEExample(t *testing.T) {
 	t.Parallel()
 
 	opts := &terraform.Options{
-		TerraformDir: examplesPath + "gke",
+		TerraformDir: utils.ExamplesPath + "gke",
 
 		Vars: map[string]interface{}{
 			"prefix":       "test",
@@ -25,22 +26,22 @@ func TestGKEExample(t *testing.T) {
 	terraform.InitAndApply(t, opts)
 
 	// get outputs
-	caCert := fetchSensitiveOutput(t, opts, "cluster_ca_certificate")
-	host := fetchSensitiveOutput(t, opts, "host")
-	token := fetchSensitiveOutput(t, opts, "access_token")
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
+	token := utils.FetchSensitiveOutput(t, opts, "access_token")
 
 	// decode the base64 encoded cert
-	caCertString := decodeBase64(t, caCert)
+	caCertString := utils.DecodeBase64(t, caCert)
 
 	// build the REST config
-	restConfig := createRESTConfigWithBearer(caCertString, token, host)
+	restConfig := utils.CreateRESTConfigWithBearer(caCertString, token, host)
 
 	// create a Kubernetes clientset
-	clientSet, err := buildClientSet(restConfig)
+	clientSet, err := utils.BuildClientSet(restConfig)
 	if err != nil {
 		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
 	}
 
 	// test the cluster is accessible
-	testClusterAccess(t, clientSet)
+	utils.TestClusterAccess(t, clientSet)
 }
diff --git a/test/multicloud/test/unit/kind_test.go b/test/multicloud/test/unit/kind_test.go
new file mode 100644
index 0000000000..f08f5d17f0
--- /dev/null
+++ b/test/multicloud/test/unit/kind_test.go
@@ -0,0 +1,42 @@
+package test
+
+import (
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/retina/test/multicloud/test/utils"
+)
+
+func TestKindExample(t *testing.T) {
+	t.Parallel()
+
+	opts := &terraform.Options{
+		TerraformDir: utils.ExamplesPath + "kind",
+
+		Vars: map[string]interface{}{
+			"prefix": "test",
+		},
+	}
+
+	// clean up at the end of the test
+	defer terraform.Destroy(t, opts)
+	terraform.InitAndApply(t, opts)
+
+	// get outputs
+	caCert := utils.FetchSensitiveOutput(t, opts, "cluster_ca_certificate")
+	clientCert := utils.FetchSensitiveOutput(t, opts, "client_certificate")
+	clientKey := utils.FetchSensitiveOutput(t, opts, "client_key")
+	host := utils.FetchSensitiveOutput(t, opts, "host")
+
+	// build the REST config
+	restConfig := utils.CreateRESTConfigWithClientCert(caCert, clientCert, clientKey, host)
+
+	// create a Kubernetes clientset
+	clientSet, err := utils.BuildClientSet(restConfig)
+	if err != nil {
+		t.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
+
+	// test the cluster is accessible
+	utils.TestClusterAccess(t, clientSet)
+}
diff --git a/test/multicloud/test/types.go b/test/multicloud/test/utils/types.go
similarity index 52%
rename from test/multicloud/test/types.go
rename to test/multicloud/test/utils/types.go
index 97ab948a47..dc07ee8608 100644
--- a/test/multicloud/test/types.go
+++ b/test/multicloud/test/utils/types.go
@@ -1,6 +1,9 @@
-package test
+package utils
 
-const examplesPath = "../examples/"
+const (
+	ExamplesPath  = "../../examples/"
+	RetinaVersion = "v0.0.24"
+)
 
 type PodSelector struct {
 	Namespace     string
diff --git a/test/multicloud/test/utils.go b/test/multicloud/test/utils/utils.go
similarity index 89%
rename from test/multicloud/test/utils.go
rename to test/multicloud/test/utils/utils.go
index 70cddcc359..ab03fec485 100644
--- a/test/multicloud/test/utils.go
+++ b/test/multicloud/test/utils/utils.go
@@ -1,4 +1,4 @@
-package test
+package utils
 
 import (
 	"bufio"
@@ -19,7 +19,7 @@ import (
 	"k8s.io/client-go/rest"
 )
 
-func buildClientSet(config *rest.Config) (*kubernetes.Clientset, error) {
+func BuildClientSet(config *rest.Config) (*kubernetes.Clientset, error) {
 	// Create a Kubernetes client
 	clientset, err := kubernetes.NewForConfig(config)
 	if err != nil {
@@ -29,7 +29,7 @@ func buildClientSet(config *rest.Config) (*kubernetes.Clientset, error) {
 }
 
 // Create a Bearer token REST config
-func createRESTConfigWithBearer(caCert, bearerToken, host string) *rest.Config {
+func CreateRESTConfigWithBearer(caCert, bearerToken, host string) *rest.Config {
 	config := &rest.Config{
 		Host:        host,
 		BearerToken: bearerToken,
@@ -41,7 +41,7 @@ func createRESTConfigWithBearer(caCert, bearerToken, host string) *rest.Config {
 }
 
 // Create REST config with client cert and key
-func createRESTConfigWithClientCert(caCert, clientCert, clientKey, host string) *rest.Config {
+func CreateRESTConfigWithClientCert(caCert, clientCert, clientKey, host string) *rest.Config {
 	config := &rest.Config{
 		Host: host,
 		TLSClientConfig: rest.TLSClientConfig{
@@ -53,7 +53,7 @@ func createRESTConfigWithClientCert(caCert, clientCert, clientKey, host string)
 	return config
 }
 
-func testClusterAccess(t *testing.T, clientset kubernetes.Interface) {
+func TestClusterAccess(t *testing.T, clientset kubernetes.Interface) {
 	// Test the cluster is accessible by listing nodes
 	_, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
@@ -85,7 +85,7 @@ func checkLogsForErrors(logs io.ReadCloser) error {
 	return nil
 }
 
-func checkPodLogs(t *testing.T, clientset kubernetes.Interface, podSelector PodSelector) {
+func CheckPodLogs(t *testing.T, clientset kubernetes.Interface, podSelector PodSelector) {
 	// Get the logs for the retina pods
 	pods, err := clientset.CoreV1().Pods(podSelector.Namespace).List(context.TODO(), metav1.ListOptions{
 		LabelSelector: podSelector.LabelSelector,
@@ -115,7 +115,7 @@ func checkPodLogs(t *testing.T, clientset kubernetes.Interface, podSelector PodS
 }
 
 // function to convert base64 encoded string to plain text
-func decodeBase64(t *testing.T, encoded string) string {
+func DecodeBase64(t *testing.T, encoded string) string {
 	// decode the base64 encoded string
 	decoded, err := base64.StdEncoding.DecodeString(encoded)
 	if err != nil {
@@ -126,7 +126,7 @@ func decodeBase64(t *testing.T, encoded string) string {
 }
 
 // fetch the sensitive output from OpenTofu
-func fetchSensitiveOutput(t *testing.T, options *terraform.Options, name string) string {
+func FetchSensitiveOutput(t *testing.T, options *terraform.Options, name string) string {
 	defer func() {
 		options.Logger = nil
 	}()
@@ -134,7 +134,7 @@ func fetchSensitiveOutput(t *testing.T, options *terraform.Options, name string)
 	return terraform.Output(t, options, name)
 }
 
-func arePodsRunning(clientset kubernetes.Interface, podSelector PodSelector, timeout time.Duration) (bool, error) {
+func ArePodsRunning(clientset kubernetes.Interface, podSelector PodSelector, timeout time.Duration) (bool, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
diff --git a/test/multicloud/test/utils_test.go b/test/multicloud/test/utils/utils_test.go
similarity index 93%
rename from test/multicloud/test/utils_test.go
rename to test/multicloud/test/utils/utils_test.go
index d4f7a6600e..1a5df25e8a 100644
--- a/test/multicloud/test/utils_test.go
+++ b/test/multicloud/test/utils/utils_test.go
@@ -1,4 +1,4 @@
-package test
+package utils
 
 import (
 	"bytes"
@@ -19,7 +19,7 @@ import (
 
 func TestBuildClientSet(t *testing.T) {
 	config := &rest.Config{}
-	clientset, err := buildClientSet(config)
+	clientset, err := BuildClientSet(config)
 	if err != nil {
 		t.Fatalf("Failed to build clientset: %v", err)
 	}
@@ -29,14 +29,14 @@ func TestBuildClientSet(t *testing.T) {
 }
 
 func TestCreateRESTConfigWithBearer(t *testing.T) {
-	config := createRESTConfigWithBearer("caCert", "bearerToken", "host")
+	config := CreateRESTConfigWithBearer("caCert", "bearerToken", "host")
 	if config.BearerToken != "bearerToken" {
 		t.Fatalf("Expected BearerToken to be 'bearerToken'")
 	}
 }
 
 func TestCreateRESTConfigWithClientCert(t *testing.T) {
-	config := createRESTConfigWithClientCert("caCert", "clientCert", "clientKey", "host")
+	config := CreateRESTConfigWithClientCert("caCert", "clientCert", "clientKey", "host")
 	if config.TLSClientConfig.CAData == nil || string(config.TLSClientConfig.CAData) != "caCert" {
 		t.Fatalf("Expected CAData to be 'caCert'")
 	}
@@ -53,7 +53,7 @@ func TestCreateRESTConfigWithClientCert(t *testing.T) {
 
 func TestTestClusterAccess(t *testing.T) {
 	clientset := fake.NewSimpleClientset()
-	testClusterAccess(t, clientset)
+	TestClusterAccess(t, clientset)
 
 	// Simulate a failure scenario
 	clientset = nil
@@ -62,7 +62,7 @@ func TestTestClusterAccess(t *testing.T) {
 			t.Fatalf("Expected panic due to nil clientset, but code did not panic")
 		}
 	}()
-	testClusterAccess(t, clientset)
+	TestClusterAccess(t, clientset)
 }
 
 func TestCheckLogsForErrors(t *testing.T) {
@@ -124,12 +124,12 @@ func TestCheckPodLogs(t *testing.T) {
 		LabelSelector: "app=test",
 		ContainerName: "test-container",
 	}
-	checkPodLogs(t, clientset, podSelector)
+	CheckPodLogs(t, clientset, podSelector)
 }
 
 func TestDecodeBase64(t *testing.T) {
 	encoded := base64.StdEncoding.EncodeToString([]byte("test"))
-	decoded := decodeBase64(t, encoded)
+	decoded := DecodeBase64(t, encoded)
 	if decoded != "test" {
 		t.Fatalf("Expected 'test', got '%s'", decoded)
 	}
@@ -169,7 +169,7 @@ func TestFetchSensitiveOutput(t *testing.T) {
 	}
 
 	// Fetch the sensitive output
-	output := fetchSensitiveOutput(t, opts, "test-output")
+	output := FetchSensitiveOutput(t, opts, "test-output")
 	if output != "sensitive-value" {
 		t.Fatalf("Expected 'sensitive-value', got '%s'", output)
 	}
@@ -269,7 +269,7 @@ func TestCheckPodsRunning(t *testing.T) {
 				Namespace:     "default",
 				LabelSelector: "app=test",
 			}
-			result, err := arePodsRunning(clientset, podSelector, time.Duration(1)*time.Second)
+			result, err := ArePodsRunning(clientset, podSelector, time.Duration(1)*time.Second)
 			if result != tc.expected {
 				t.Fatalf("Expected %v, got %v with error: %v", tc.expected, result, err)
 			}

From 409f12a3dde1cf2599c646966c433e449241c973 Mon Sep 17 00:00:00 2001
From: Simone Rodigari <srodigari@microsoft.com>
Date: Tue, 4 Feb 2025 09:43:59 +0000
Subject: [PATCH 2/3] test(mc): Fix md linting

---
 test/multicloud/README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/multicloud/README.md b/test/multicloud/README.md
index f606accde4..dc7bd0c230 100644
--- a/test/multicloud/README.md
+++ b/test/multicloud/README.md
@@ -97,7 +97,6 @@ make test
 * [AKS resource documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster)
 * [Kind resource documentation](https://registry.terraform.io/providers/tehcyx/kind/latest/docs/resources/cluster)
 
-
 ## Troubleshooting
 
 In case the test fails due to timeout, validate the resource was created by the provider, and if it is, you can import into OpenTofu state.

From dbe69f798e7613794e70d1c1c5aabf66f66f2caf Mon Sep 17 00:00:00 2001
From: Simone Rodigari <srodigari@microsoft.com>
Date: Tue, 4 Feb 2025 10:39:08 +0000
Subject: [PATCH 3/3] test(mc): Uncomment defer destroy

---
 test/multicloud/test/integration/retina_gke_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/multicloud/test/integration/retina_gke_test.go b/test/multicloud/test/integration/retina_gke_test.go
index 69510740b5..75b1af90a2 100644
--- a/test/multicloud/test/integration/retina_gke_test.go
+++ b/test/multicloud/test/integration/retina_gke_test.go
@@ -38,7 +38,7 @@ func TestRetinaGKEIntegration(t *testing.T) {
 	}
 
 	// clean up at the end of the test
-	// defer terraform.Destroy(t, opts)
+	defer terraform.Destroy(t, opts)
 	terraform.InitAndApply(t, opts)
 
 	// get outputs