diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml index 83b38e52be3..c0a30a09250 100644 --- a/.github/workflows/operator-e2e-integration-tests.yml +++ b/.github/workflows/operator-e2e-integration-tests.yml @@ -55,6 +55,8 @@ jobs: # Run the e2e tests cd infra/feast-operator/ make test-e2e + make test-previous-version + make test-upgrade - name: Debug KIND Cluster when there is a failure if: failure() diff --git a/infra/feast-operator/.golangci.yml b/infra/feast-operator/.golangci.yml index 020e5768657..6c104980d43 100644 --- a/infra/feast-operator/.golangci.yml +++ b/infra/feast-operator/.golangci.yml @@ -19,6 +19,12 @@ issues: - path: "test/*" linters: - lll + - path: "upgrade/*" + linters: + - lll + - path: "previous-version/*" + linters: + - lll linters: disable-all: true enable: diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 9c90f02d415..ccee1528908 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -113,12 +113,20 @@ vet: ## Run go vet against code. .PHONY: test test: build-installer vet lint envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v test/e2e | grep -v test/data-source-types) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v test/e2e | grep -v test/data-source-types | grep -v test/upgrade | grep -v test/previous_version) -coverprofile cover.out # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: - go test -timeout 30m ./test/e2e/ -v -ginkgo.v + go test -timeout 60m ./test/e2e/ -v -ginkgo.v + +.PHONY: test-upgrade # Run the upgrade tests against a Kind k8s instance that is spun up. +test-upgrade: + go test -timeout 60m ./test/upgrade/ -v -ginkgo.v + +.PHONY: test-previous-version # Run e2e tests against previous version in a Kind k8s instance that is spun up. +test-previous-version: + go test -timeout 60m ./test/previous-version/ -v -ginkgo.v # Requires python3 .PHONY: test-datasources diff --git a/infra/feast-operator/test/e2e/e2e_suite_test.go b/infra/feast-operator/test/e2e/e2e_suite_test.go index c45853a0073..fd22df15b12 100644 --- a/infra/feast-operator/test/e2e/e2e_suite_test.go +++ b/infra/feast-operator/test/e2e/e2e_suite_test.go @@ -18,10 +18,9 @@ package e2e import ( "fmt" - "testing" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "testing" ) // Run e2e tests using the Ginkgo runner. diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index b2773f579f0..e0a8bd647dc 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -17,217 +17,38 @@ limitations under the License. package e2e import ( - "fmt" - "os" - "os/exec" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/feast-dev/feast/infra/feast-operator/test/utils" -) - -const ( - feastControllerNamespace = "feast-operator-system" - timeout = 2 * time.Minute - controllerDeploymentName = "feast-operator-controller-manager" - feastPrefix = "feast-" + . "github.com/onsi/ginkgo/v2" ) var _ = Describe("controller", Ordered, func() { - BeforeAll(func() { - _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") - if !isRunOnOpenShiftCI { - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace) - _, _ = utils.Run(cmd) - - var err error - // projectimage stores the name of the image used in the example - var projectimage = "localhost/feast-operator:v0.0.1" - - By("building the manager(Operator) image") - cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - // this image will be built in above make target. - var feastImage = "feastdev/feature-server:dev" - var feastLocalImage = "localhost/feastdev/feature-server:dev" - - By("building the feast image") - cmd = exec.Command("make", "feast-ci-dev-docker-img") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("Tag the local feast image for the integration tests") - cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the feast image on Kind cluster") - err = utils.LoadImageToKindClusterWithName(feastLocalImage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + featureStoreName := "simple-feast-setup" + feastResourceName := utils.FeastPrefix + featureStoreName + feastK8sResourceNames := []string{ + feastResourceName + "-online", + feastResourceName + "-offline", + feastResourceName + "-ui", + } - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + runTestDeploySimpleCRFunc := utils.GetTestDeploySimpleCRFunc("/test/e2e", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", + featureStoreName, feastResourceName, feastK8sResourceNames) - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage), fmt.Sprintf("FS_IMG=%s", feastLocalImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + runTestWithRemoteRegistryFunction := utils.GetTestWithRemoteRegistryFunc("/test/e2e", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", + featureStoreName, feastResourceName, feastK8sResourceNames) - By("Validating that the controller-manager deployment is in available state") - err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - controllerDeploymentName, err, - )) - fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) - } + BeforeAll(func() { + utils.DeployOperatorFromCode("/test/e2e") }) AfterAll(func() { - // Add any post clean up code here. - _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") - if !isRunOnOpenShiftCI { - By("Uninstalling the feast CRD") - cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - } + utils.DeleteOperatorDeployment("/test/e2e") }) Context("Operator E2E Tests", func() { - It("Should be able to deploy and run a default feature store CR successfully", func() { - By("deploying the Simple Feast Custom Resource to Kubernetes") - namespace := "default" - - cmd := exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) - _, cmdOutputerr := utils.Run(cmd) - ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - - featureStoreName := "simple-feast-setup" - validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) - - By("deleting the feast deployment") - cmd = exec.Command("kubectl", "delete", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") - _, cmdOutputerr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - }) - - It("Should be able to deploy and run a feature store with remote registry CR successfully", func() { - By("deploying the Simple Feast Custom Resource to Kubernetes") - namespace := "default" - cmd := exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) - _, cmdOutputErr := utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - - featureStoreName := "simple-feast-setup" - validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) - - var remoteRegistryNs = "remote-registry" - By(fmt.Sprintf("Creating the remote registry namespace=%s", remoteRegistryNs)) - cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) - _, _ = utils.Run(cmd) - - By("deploying the Simple Feast remote registry Custom Resource on Kubernetes") - cmd = exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) - _, cmdOutputErr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - - remoteFeatureStoreName := "simple-feast-remote-setup" - - validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout) - - By("deleting the feast remote registry deployment") - cmd = exec.Command("kubectl", "delete", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) - _, cmdOutputErr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - - By("deleting the feast deployment") - cmd = exec.Command("kubectl", "delete", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) - _, cmdOutputErr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - }) + It("Should be able to deploy and run a default feature store CR successfully", runTestDeploySimpleCRFunc) + It("Should be able to deploy and run a feature store with remote registry CR successfully", runTestWithRemoteRegistryFunction) }) }) - -func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) { - hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", - featureStoreName, err)) - - feastResourceName := feastPrefix + featureStoreName - k8sResourceNames := []string{feastResourceName} - feastK8sResourceNames := []string{ - feastResourceName + "-online", - feastResourceName + "-offline", - feastResourceName + "-ui", - } - - if !hasRemoteRegistry { - feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") - } - - for _, deploymentName := range k8sResourceNames { - By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) - err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - deploymentName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", deploymentName) - } - - By("Check if the feast client - kubernetes config map exists.") - configMapName := feastResourceName + "-client" - err = checkIfConfigMapExists(namespace, configMapName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "config map %s is not available but expected to be available. \nError: %v\n", - configMapName, err, - )) - fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) - - for _, serviceAccountName := range k8sResourceNames { - By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) - err = checkIfServiceAccountExists(namespace, serviceAccountName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Service account %s does not exist in namespace %s. Error: %v", - serviceAccountName, namespace, err, - )) - fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) - } - - for _, serviceName := range feastK8sResourceNames { - By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) - err = checkIfKubernetesServiceExists(namespace, serviceName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "kubernetes service %s is not available but expected to be available. \nError: %v\n", - serviceName, err, - )) - fmt.Printf("kubernetes service %s is available\n", serviceName) - } - - By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) - err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", - featureStoreName, err, - )) - fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName) -} diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go deleted file mode 100644 index 743f04afc54..00000000000 --- a/infra/feast-operator/test/e2e/test_util.go +++ /dev/null @@ -1,209 +0,0 @@ -package e2e - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "os/exec" - "strings" - "time" - - appsv1 "k8s.io/api/apps/v1" - - "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" -) - -// dynamically checks if all conditions of custom resource featurestore are in "Ready" state. -func checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace string) error { - cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, "-o", "json") - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to get resource %s in namespace %s. Error: %v. Stderr: %s", - featureStoreName, namespace, err, stderr.String()) - } - - // Parse the JSON into FeatureStore - var resource v1alpha1.FeatureStore - if err := json.Unmarshal(out.Bytes(), &resource); err != nil { - return fmt.Errorf("failed to parse the resource JSON. Error: %v", err) - } - - // Validate all conditions - for _, condition := range resource.Status.Conditions { - if condition.Status != "True" { - return fmt.Errorf(" FeatureStore=%s condition '%s' is not in 'Ready' state. Status: %s", - featureStoreName, condition.Type, condition.Status) - } - } - - return nil -} - -// validates if a deployment exists and also in the availability state as True. -func checkIfDeploymentExistsAndAvailable(namespace string, deploymentName string, timeout time.Duration) error { - var output, errOutput bytes.Buffer - - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - - timeoutChan := time.After(timeout) - - for { - select { - case <-timeoutChan: - return fmt.Errorf("timed out waiting for deployment %s to become available", deploymentName) - case <-ticker.C: - // Run kubectl command - cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", namespace, "-o", "json") - cmd.Stdout = &output - cmd.Stderr = &errOutput - - if err := cmd.Run(); err != nil { - // Log error and retry - fmt.Printf("Deployment not yet found, we may try again to find the updated status: %s\n", errOutput.String()) - continue - } - - // Parse the JSON output into Deployment - var result appsv1.Deployment - if err := json.Unmarshal(output.Bytes(), &result); err != nil { - return fmt.Errorf("failed to parse deployment JSON: %v", err) - } - - // Check for Available condition - for _, condition := range result.Status.Conditions { - if condition.Type == "Available" && condition.Status == "True" { - return nil // Deployment is available - } - } - - // Reset buffers for the next loop iteration - output.Reset() - errOutput.Reset() - } - } -} - -// validates if a service account exists using the kubectl CLI. -func checkIfServiceAccountExists(namespace, saName string) error { - cmd := exec.Command("kubectl", "get", "sa", saName, "-n", namespace) - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to find service account %s in namespace %s. Error: %v. Stderr: %s", - saName, namespace, err, stderr.String()) - } - - // Check the output to confirm presence - if !strings.Contains(out.String(), saName) { - return fmt.Errorf("service account %s not found in namespace %s", saName, namespace) - } - - return nil -} - -// validates if a config map exists using the kubectl CLI. -func checkIfConfigMapExists(namespace, configMapName string) error { - cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace) - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to find config map %s in namespace %s. Error: %v. Stderr: %s", - configMapName, namespace, err, stderr.String()) - } - - // Check the output to confirm presence - if !strings.Contains(out.String(), configMapName) { - return fmt.Errorf("config map %s not found in namespace %s", configMapName, namespace) - } - - return nil -} - -// validates if a kubernetes service exists using the kubectl CLI. -func checkIfKubernetesServiceExists(namespace, serviceName string) error { - cmd := exec.Command("kubectl", "get", "service", serviceName, "-n", namespace) - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to find kubernetes service %s in namespace %s. Error: %v. Stderr: %s", - serviceName, namespace, err, stderr.String()) - } - - // Check the output to confirm presence - if !strings.Contains(out.String(), serviceName) { - return fmt.Errorf("kubernetes service %s not found in namespace %s", serviceName, namespace) - } - - return nil -} - -func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (bool, error) { - timeout := time.Second * 30 - interval := time.Second * 2 // Poll every 2 seconds - startTime := time.Now() - - for time.Since(startTime) < timeout { - cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, - "-o=jsonpath='{.status.applied.services.registry}'") - - output, err := cmd.Output() - if err != nil { - // Retry only on transient errors - if _, ok := err.(*exec.ExitError); ok { - time.Sleep(interval) - continue - } - return false, err // Return immediately on non-transient errors - } - - // Convert output to string and trim any extra spaces - result := strings.TrimSpace(string(output)) - - // Remove single quotes if present - if strings.HasPrefix(result, "'") && strings.HasSuffix(result, "'") { - result = strings.Trim(result, "'") - } - - if result == "" { - time.Sleep(interval) // Retry if result is empty - continue - } - - // Parse the JSON into a map - var registryConfig v1alpha1.Registry - if err := json.Unmarshal([]byte(result), ®istryConfig); err != nil { - return false, err // Return false on JSON parsing failure - } - - if registryConfig.Remote == nil { - return false, nil - } - - hasHostname := registryConfig.Remote.Hostname != nil - hasValidFeastRef := registryConfig.Remote.FeastRef != nil && - registryConfig.Remote.FeastRef.Name != "" - - return hasHostname || hasValidFeastRef, nil - } - - return false, errors.New("timeout waiting for featurestore registry status to be ready") -} diff --git a/infra/feast-operator/test/previous-version/previous_version_suite_test.go b/infra/feast-operator/test/previous-version/previous_version_suite_test.go new file mode 100644 index 00000000000..410133f1794 --- /dev/null +++ b/infra/feast-operator/test/previous-version/previous_version_suite_test.go @@ -0,0 +1,31 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" +) + +// Run upgrade tests using the Ginkgo runner. +func TestPreviousVersion(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting test previous version suite\n") + RunSpecs(t, "previous version operator") +} diff --git a/infra/feast-operator/test/previous-version/previous_version_test.go b/infra/feast-operator/test/previous-version/previous_version_test.go new file mode 100644 index 00000000000..9775d239bcc --- /dev/null +++ b/infra/feast-operator/test/previous-version/previous_version_test.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("previous version operator", Ordered, func() { + BeforeAll(func() { + utils.DeployPreviousVersionOperator() + }) + + AfterAll(func() { + utils.DeleteOperatorDeployment("/test/upgrade") + }) + + Context("Previous version operator Tests", func() { + feastK8sResourceNames := []string{ + utils.FeastResourceName + "-online", + utils.FeastResourceName + "-offline", + utils.FeastResourceName + "-ui", + } + + runTestDeploySimpleCRFunc := utils.GetTestDeploySimpleCRFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.FeatureStoreName, utils.FeastResourceName, feastK8sResourceNames) + runTestWithRemoteRegistryFunction := utils.GetTestWithRemoteRegistryFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.GetRemoteRegistryPreviousVerCR(), utils.FeatureStoreName, utils.FeastResourceName, feastK8sResourceNames) + + // Run Test on previous version operator + It("Should be able to deploy and run a default feature store CR successfully", runTestDeploySimpleCRFunc) + It("Should be able to deploy and run a feature store with remote registry CR successfully", runTestWithRemoteRegistryFunction) + }) +}) diff --git a/infra/feast-operator/test/upgrade/upgrade_suite_test.go b/infra/feast-operator/test/upgrade/upgrade_suite_test.go new file mode 100644 index 00000000000..bd0da7ab177 --- /dev/null +++ b/infra/feast-operator/test/upgrade/upgrade_suite_test.go @@ -0,0 +1,32 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Run upgrade tests using the Ginkgo runner. +func TestUpgrade(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting upgrade operator suite\n") + RunSpecs(t, "operator upgrade") +} diff --git a/infra/feast-operator/test/upgrade/upgrade_test.go b/infra/feast-operator/test/upgrade/upgrade_test.go new file mode 100644 index 00000000000..58b858d1850 --- /dev/null +++ b/infra/feast-operator/test/upgrade/upgrade_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("operator upgrade", Ordered, func() { + BeforeAll(func() { + utils.DeployPreviousVersionOperator() + utils.DeployOperatorFromCode("/test/e2e") + }) + + AfterAll(func() { + utils.DeleteOperatorDeployment("/test/e2e") + }) + + Context("Operator upgrade Tests", func() { + runTestDeploySimpleCRFunc := utils.GetTestDeploySimpleCRFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.FeatureStoreName, utils.FeastResourceName, []string{}) + runTestWithRemoteRegistryFunction := utils.GetTestWithRemoteRegistryFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.GetRemoteRegistryPreviousVerCR(), utils.FeatureStoreName, utils.FeastResourceName, []string{}) + + // Run Test on current version operator with previous version CR + It("Should be able to deploy and run a default feature store CR successfully", runTestDeploySimpleCRFunc) + It("Should be able to deploy and run a feature store with remote registry CR successfully", runTestWithRemoteRegistryFunction) + }) +}) diff --git a/infra/feast-operator/test/utils/test_util.go b/infra/feast-operator/test/utils/test_util.go new file mode 100644 index 00000000000..4a98a670e2d --- /dev/null +++ b/infra/feast-operator/test/utils/test_util.go @@ -0,0 +1,449 @@ +package utils + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + + "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" + "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" +) + +const ( + FeastControllerNamespace = "feast-operator-system" + Timeout = 3 * time.Minute + ControllerDeploymentName = "feast-operator-controller-manager" + FeastPrefix = "feast-" + FeatureStoreName = "simple-feast-setup" + FeastResourceName = FeastPrefix + FeatureStoreName +) + +// dynamically checks if all conditions of custom resource featurestore are in "Ready" state. +func checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace string) error { + // Wait 10 seconds to lets the feature store status update + time.Sleep(1 * time.Minute) + + cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, "-o", "json") + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to get resource %s in namespace %s. Error: %v. Stderr: %s", + featureStoreName, namespace, err, stderr.String()) + } + + // Parse the JSON into FeatureStore + var resource v1alpha1.FeatureStore + if err := json.Unmarshal(out.Bytes(), &resource); err != nil { + return fmt.Errorf("failed to parse the resource JSON. Error: %v", err) + } + + // Validate all conditions + for _, condition := range resource.Status.Conditions { + if condition.Status != "True" { + return fmt.Errorf(" FeatureStore=%s condition '%s' is not in 'Ready' state. Status: %s", + featureStoreName, condition.Type, condition.Status) + } + } + + return nil +} + +// CheckIfDeploymentExistsAndAvailable - validates if a deployment exists and also in the availability state as True. +func CheckIfDeploymentExistsAndAvailable(namespace string, deploymentName string, timeout time.Duration) error { + var output, errOutput bytes.Buffer + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + timeoutChan := time.After(timeout) + + for { + select { + case <-timeoutChan: + return fmt.Errorf("timed out waiting for deployment %s to become available", deploymentName) + case <-ticker.C: + // Run kubectl command + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", namespace, "-o", "json") + cmd.Stdout = &output + cmd.Stderr = &errOutput + + if err := cmd.Run(); err != nil { + // Log error and retry + fmt.Printf("Deployment not yet found, we may try again to find the updated status: %s\n", errOutput.String()) + continue + } + + // Parse the JSON output into Deployment + var result appsv1.Deployment + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + return fmt.Errorf("failed to parse deployment JSON: %v", err) + } + + // Check for Available condition + for _, condition := range result.Status.Conditions { + if condition.Type == "Available" && condition.Status == "True" { + return nil // Deployment is available + } + } + + // Reset buffers for the next loop iteration + output.Reset() + errOutput.Reset() + } + } +} + +// validates if a service account exists using the kubectl CLI. +func checkIfServiceAccountExists(namespace, saName string) error { + cmd := exec.Command("kubectl", "get", "sa", saName, "-n", namespace) + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to find service account %s in namespace %s. Error: %v. Stderr: %s", + saName, namespace, err, stderr.String()) + } + + // Check the output to confirm presence + if !strings.Contains(out.String(), saName) { + return fmt.Errorf("service account %s not found in namespace %s", saName, namespace) + } + + return nil +} + +// validates if a config map exists using the kubectl CLI. +func checkIfConfigMapExists(namespace, configMapName string) error { + cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace) + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to find config map %s in namespace %s. Error: %v. Stderr: %s", + configMapName, namespace, err, stderr.String()) + } + + // Check the output to confirm presence + if !strings.Contains(out.String(), configMapName) { + return fmt.Errorf("config map %s not found in namespace %s", configMapName, namespace) + } + + return nil +} + +// validates if a kubernetes service exists using the kubectl CLI. +func checkIfKubernetesServiceExists(namespace, serviceName string) error { + cmd := exec.Command("kubectl", "get", "service", serviceName, "-n", namespace) + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to find kubernetes service %s in namespace %s. Error: %v. Stderr: %s", + serviceName, namespace, err, stderr.String()) + } + + // Check the output to confirm presence + if !strings.Contains(out.String(), serviceName) { + return fmt.Errorf("kubernetes service %s not found in namespace %s", serviceName, namespace) + } + + return nil +} + +func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (bool, error) { + timeout := time.Second * 30 + interval := time.Second * 2 // Poll every 2 seconds + startTime := time.Now() + + for time.Since(startTime) < timeout { + cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, + "-o=jsonpath='{.status.applied.services.registry}'") + + output, err := cmd.Output() + if err != nil { + // Retry only on transient errors + if _, ok := err.(*exec.ExitError); ok { + time.Sleep(interval) + continue + } + return false, err // Return immediately on non-transient errors + } + + // Convert output to string and trim any extra spaces + result := strings.TrimSpace(string(output)) + + // Remove single quotes if present + if strings.HasPrefix(result, "'") && strings.HasSuffix(result, "'") { + result = strings.Trim(result, "'") + } + + if result == "" { + time.Sleep(interval) // Retry if result is empty + continue + } + + // Parse the JSON into a map + var registryConfig v1alpha1.Registry + if err := json.Unmarshal([]byte(result), ®istryConfig); err != nil { + return false, err // Return false on JSON parsing failure + } + + if registryConfig.Remote == nil { + return false, nil + } + + hasHostname := registryConfig.Remote.Hostname != nil + hasValidFeastRef := registryConfig.Remote.FeastRef != nil && + registryConfig.Remote.FeastRef.Name != "" + + return hasHostname || hasValidFeastRef, nil + } + + return false, errors.New("timeout waiting for featurestore registry status to be ready") +} + +func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, feastResourceName string, feastK8sResourceNames []string, timeout time.Duration) { + hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", + featureStoreName, err)) + + k8sResourceNames := []string{feastResourceName} + + if !hasRemoteRegistry { + feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") + } + + for _, deploymentName := range k8sResourceNames { + By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) + err = CheckIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + deploymentName, err, + )) + fmt.Printf("Feast Deployment %s is available\n", deploymentName) + } + + By("Check if the feast client - kubernetes config map exists.") + configMapName := feastResourceName + "-client" + err = checkIfConfigMapExists(namespace, configMapName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "config map %s is not available but expected to be available. \nError: %v\n", + configMapName, err, + )) + fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) + + for _, serviceAccountName := range k8sResourceNames { + By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) + err = checkIfServiceAccountExists(namespace, serviceAccountName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Service account %s does not exist in namespace %s. Error: %v", + serviceAccountName, namespace, err, + )) + fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) + } + + for _, serviceName := range feastK8sResourceNames { + By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) + err = checkIfKubernetesServiceExists(namespace, serviceName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "kubernetes service %s is not available but expected to be available. \nError: %v\n", + serviceName, err, + )) + fmt.Printf("kubernetes service %s is available\n", serviceName) + } + + By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) + err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", + featureStoreName, err, + )) + fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName) +} + +// GetTestDeploySimpleCRFunc - returns a simple CR deployment function +func GetTestDeploySimpleCRFunc(testDir string, crYaml string, featureStoreName string, feastResourceName string, feastK8sResourceNames []string) func() { + return func() { + By("deploying the Simple Feast Custom Resource to Kubernetes") + namespace := "default" + + cmd := exec.Command("kubectl", "apply", "-f", crYaml, "-n", namespace) + _, cmdOutputerr := Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + + validateTheFeatureStoreCustomResource(namespace, featureStoreName, feastResourceName, feastK8sResourceNames, Timeout) + + By("deleting the feast deployment") + cmd = exec.Command("kubectl", "delete", "-f", crYaml) + _, cmdOutputerr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + } +} + +// GetTestWithRemoteRegistryFunc - returns a CR deployment with a remote registry function +func GetTestWithRemoteRegistryFunc(testDir string, crYaml string, remoteRegistryCRYaml string, featureStoreName string, feastResourceName string, feastK8sResourceNames []string) func() { + return func() { + By("deploying the Simple Feast Custom Resource to Kubernetes") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", crYaml, "-n", namespace) + _, cmdOutputErr := Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + + validateTheFeatureStoreCustomResource(namespace, featureStoreName, feastResourceName, feastK8sResourceNames, Timeout) + + var remoteRegistryNs = "remote-registry" + By(fmt.Sprintf("Creating the remote registry namespace=%s", remoteRegistryNs)) + cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) + _, _ = Run(cmd, testDir) + + By("deploying the Simple Feast remote registry Custom Resource on Kubernetes") + cmd = exec.Command("kubectl", "apply", "-f", remoteRegistryCRYaml, "-n", remoteRegistryNs) + _, cmdOutputErr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + + remoteFeatureStoreName := "simple-feast-remote-setup" + remoteFeastResourceName := FeastPrefix + remoteFeatureStoreName + fixRemoteFeastK8sResourceNames(feastK8sResourceNames, remoteFeastResourceName) + validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, remoteFeastResourceName, feastK8sResourceNames, Timeout) + + By("deleting the feast remote registry deployment") + cmd = exec.Command("kubectl", "delete", "-f", remoteRegistryCRYaml, "-n", remoteRegistryNs) + _, cmdOutputErr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + + By("deleting the feast deployment") + cmd = exec.Command("kubectl", "delete", "-f", crYaml, "-n", namespace) + _, cmdOutputErr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + } +} + +func fixRemoteFeastK8sResourceNames(feastK8sResourceNames []string, remoteFeastResourceName string) { + for i, feastK8sResourceName := range feastK8sResourceNames { + if index := strings.LastIndex(feastK8sResourceName, "-"); index != -1 { + feastK8sResourceNames[i] = remoteFeastResourceName + feastK8sResourceName[index:] + } + } +} + +// DeployOperatorFromCode - Creates the images for the operator and deploys it +func DeployOperatorFromCode(testDir string) { + _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") + if !isRunOnOpenShiftCI { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", FeastControllerNamespace) + _, _ = Run(cmd, testDir) + + var err error + // projectimage stores the name of the image used in the example + var projectimage = "localhost/feast-operator:v0.0.1" + + By("building the manager(Operator) image") + cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the manager(Operator) image on Kind") + err = LoadImageToKindClusterWithName(projectimage, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + // this image will be built in above make target. + var feastImage = "feastdev/feature-server:dev" + var feastLocalImage = "localhost/feastdev/feature-server:dev" + + By("building the feast image") + cmd = exec.Command("make", "feast-ci-dev-docker-img") + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("Tag the local feast image for the integration tests") + cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the feast image on Kind cluster") + err = LoadImageToKindClusterWithName(feastLocalImage, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage), fmt.Sprintf("FS_IMG=%s", feastLocalImage)) + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("Validating that the controller-manager deployment is in available state") + err = CheckIfDeploymentExistsAndAvailable(FeastControllerNamespace, ControllerDeploymentName, Timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + ControllerDeploymentName, err, + )) + fmt.Printf("Feast Control Manager Deployment %s is available\n", ControllerDeploymentName) + } +} + +// DeleteOperatorDeployment - Deletes the operator deployment +func DeleteOperatorDeployment(testDir string) { + _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") + if !isRunOnOpenShiftCI { + By("Uninstalling the feast CRD") + cmd := exec.Command("kubectl", "delete", "deployment", ControllerDeploymentName, "-n", FeastControllerNamespace) + _, err := Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } +} + +// DeployPreviousVersionOperator - Deploys the previous version of the operator +func DeployPreviousVersionOperator() { + _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") + if !isRunOnOpenShiftCI { + var err error + + cmd := exec.Command("kubectl", "apply", "-f", fmt.Sprintf("https://raw.githubusercontent.com/feast-dev/feast/refs/tags/v%s/infra/feast-operator/dist/install.yaml", feastversion.FeastVersion)) + _, err = Run(cmd, "/test/upgrade") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = CheckIfDeploymentExistsAndAvailable(FeastControllerNamespace, ControllerDeploymentName, Timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + ControllerDeploymentName, err, + )) + fmt.Printf("Feast Control Manager Deployment %s is available\n", ControllerDeploymentName) + } +} + +// GetSimplePreviousVerCR - Get The previous version simple CR for tests +func GetSimplePreviousVerCR() string { + return fmt.Sprintf("https://raw.githubusercontent.com/feast-dev/feast/refs/tags/v%s/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", feastversion.FeastVersion) +} + +// GetRemoteRegistryPreviousVerCR - Get The previous version remote registry CR for tests +func GetRemoteRegistryPreviousVerCR() string { + return fmt.Sprintf("https://raw.githubusercontent.com/feast-dev/feast/refs/tags/v%s/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", feastversion.FeastVersion) +} diff --git a/infra/feast-operator/test/utils/utils.go b/infra/feast-operator/test/utils/utils.go index 9b57f9af61c..4cd8277f574 100644 --- a/infra/feast-operator/test/utils/utils.go +++ b/infra/feast-operator/test/utils/utils.go @@ -18,11 +18,10 @@ package utils import ( "fmt" + . "github.com/onsi/ginkgo/v2" //nolint:golint,revive "os" "os/exec" "strings" - - . "github.com/onsi/ginkgo/v2" //nolint:golint,revive ) const ( @@ -39,16 +38,16 @@ func warnError(err error) { } // InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { +func InstallPrometheusOperator(testDir string) error { url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) + _, err := Run(cmd, testDir) return err } // Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { - dir, _ := GetProjectDir() +func Run(cmd *exec.Cmd, testDir string) ([]byte, error) { + dir, _ := GetProjectDir(testDir) cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { @@ -67,28 +66,28 @@ func Run(cmd *exec.Cmd) ([]byte, error) { } // UninstallPrometheusOperator uninstalls the prometheus -func UninstallPrometheusOperator() { +func UninstallPrometheusOperator(testDir string) { url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { + if _, err := Run(cmd, testDir); err != nil { warnError(err) } } // UninstallCertManager uninstalls the cert manager -func UninstallCertManager() { +func UninstallCertManager(testDir string) { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { + if _, err := Run(cmd, testDir); err != nil { warnError(err) } } // InstallCertManager installs the cert manager bundle. -func InstallCertManager() error { +func InstallCertManager(testDir string) error { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) cmd := exec.Command("kubectl", "apply", "-f", url) - if _, err := Run(cmd); err != nil { + if _, err := Run(cmd, testDir); err != nil { return err } // Wait for cert-manager-webhook to be ready, which can take time if cert-manager @@ -99,12 +98,12 @@ func InstallCertManager() error { "--timeout", "5m", ) - _, err := Run(cmd) + _, err := Run(cmd, testDir) return err } // LoadImageToKindCluster loads a local docker image to the kind cluster -func LoadImageToKindClusterWithName(name string) error { +func LoadImageToKindClusterWithName(name string, testDir string) error { cluster := "kind" if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { cluster = v @@ -112,7 +111,7 @@ func LoadImageToKindClusterWithName(name string) error { fmt.Println("cluster used in the test is -", cluster) kindOptions := []string{"load", "docker-image", name, "--name", cluster} cmd := exec.Command("kind", kindOptions...) - _, err := Run(cmd) + _, err := Run(cmd, testDir) return err } @@ -131,11 +130,11 @@ func GetNonEmptyLines(output string) []string { } // GetProjectDir will return the directory where the project is -func GetProjectDir() (string, error) { +func GetProjectDir(projectDir string) (string, error) { wd, err := os.Getwd() if err != nil { return wd, err } - wd = strings.Replace(wd, "/test/e2e", "", -1) + wd = strings.Replace(wd, projectDir, "", -1) return wd, nil }