From 40c0c00ec5cf124073646166ff545870d88ca245 Mon Sep 17 00:00:00 2001 From: Neelanjan Manna Date: Wed, 18 May 2022 10:45:23 +0530 Subject: [PATCH 1/6] updated pipeline for gcp label based experiments, added BDDs, updated Makefile Signed-off-by: Neelanjan Manna --- .github/workflows/gcp-experiment-pipeline.yml | 43 ++++- .../nightly-gcp-experiment-pipeline.yml | 12 +- Makefile | 16 ++ .../gcp/gcp-vm-disk-loss-by-label_test.go | 154 ++++++++++++++++++ .../gcp/gcp-vm-instance-stop-by-label_test.go | 151 +++++++++++++++++ 5 files changed, 362 insertions(+), 14 deletions(-) create mode 100644 platform/gcp/gcp-vm-disk-loss-by-label_test.go create mode 100644 platform/gcp/gcp-vm-instance-stop-by-label_test.go diff --git a/.github/workflows/gcp-experiment-pipeline.yml b/.github/workflows/gcp-experiment-pipeline.yml index dfcf0d255..ec32c7e95 100644 --- a/.github/workflows/gcp-experiment-pipeline.yml +++ b/.github/workflows/gcp-experiment-pipeline.yml @@ -15,7 +15,7 @@ on: default: "Always" jobs: - GCP_VM_Instance_Stop: + GCP_VM_Instance_Stop_by_Name_And_Label: runs-on: ubuntu-latest env: KUBECONFIG: /etc/rancher/k3s/k3s.yaml @@ -70,7 +70,8 @@ jobs: run: | gcloud compute instances create litmus-e2e-first-vm-${{ github.run_number }} litmus-e2e-second-vm-${{ github.run_number }} \ --machine-type=f1-micro \ - --zone=us-central1-a + --zone=us-east1-b \ + --labels=vm=litmus-e2e-vm-${{ github.run_number }} - name: Litmus Infra Setup if: always() @@ -84,17 +85,29 @@ jobs: env: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} VM_INSTANCE_NAMES: "litmus-e2e-first-vm-${{ github.run_number }},litmus-e2e-second-vm-${{ github.run_number }}" - INSTANCE_ZONES: "us-central1-a,us-central1-a" + INSTANCE_ZONES: "us-east1-b,us-east1-b" EXPERIMENT_IMAGE: "${{ github.event.inputs.experimentImage }}" EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}" CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}" run: make gcp-vm-instance-stop + - name: Run GCP VM Instance Stop By Label experiment in serial & parallel mode + if: always() + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + INSTANCE_LABEL: "vm : litmus-e2e-vm-${{ github.run_number }}" + INSTANCE_ZONES: "us-east1-b" + INSTANCE_AFFECTED_PERC: "100" + EXPERIMENT_IMAGE: "${{ github.event.inputs.experimentImage }}" + EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}" + CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}" + run: make gcp-vm-instance-stop-by-label + - name: Delete target GCP VM Instances if: always() run: | gcloud compute instances delete litmus-e2e-first-vm-${{ github.run_number }} litmus-e2e-second-vm-${{ github.run_number }} \ - --zone=us-central1-a \ + --zone=us-east1-b \ --quiet - name: "[Debug]: check chaos resources" @@ -119,7 +132,7 @@ jobs: if: always() run: /usr/local/bin/k3s-uninstall.sh - GCP_VM_Disk_Loss: + GCP_VM_Disk_Loss_By_Name_And_Label: runs-on: ubuntu-latest env: KUBECONFIG: /etc/rancher/k3s/k3s.yaml @@ -175,9 +188,11 @@ jobs: run: | gcloud compute instances create litmus-e2e-vm-${{ github.run_number }} \ --machine-type=f1-micro \ - --zone=us-central1-a \ + --zone=us-east1-b \ --create-disk name=litmus-e2e-first-disk-${{ github.run_number }},size=1GB,device-name=litmus-e2e-first-disk-${{ github.run_number }} \ --create-disk name=litmus-e2e-second-disk-${{ github.run_number }},size=1GB,device-name=litmus-e2e-second-disk-${{ github.run_number }} + gcloud compute disks add-labels litmus-e2e-first-disk-${{ github.run_number }} --labels=disk=litmus-e2e-disk-${{ github.run_number }} --zone=us-east1-b + gcloud compute disks add-labels litmus-e2e-second-disk-${{ github.run_number }} --labels=disk=litmus-e2e-disk-${{ github.run_number }} --zone=us-east1-b - name: Litmus Infra Setup if: always() @@ -191,18 +206,30 @@ jobs: env: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} DISK_VOLUME_NAMES: "litmus-e2e-first-disk-${{ github.run_number }},litmus-e2e-second-disk-${{ github.run_number }}" - DISK_ZONES: "us-central1-a,us-central1-a" + DISK_ZONES: "us-east1-b,us-east1-b" DEVICE_NAMES: "litmus-e2e-first-disk-${{ github.run_number }},litmus-e2e-second-disk-${{ github.run_number }}" EXPERIMENT_IMAGE: "${{ github.event.inputs.experimentImage }}" EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}" CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}" run: make gcp-vm-disk-loss + - name: Run GCP VM Disk Loss By Label experiment in serial & parallel mode + if: always() + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + DISK_VOLUME_LABEL: "disk : litmus-e2e-disk-${{ github.run_number }}" + DISK_ZONES: "us-east1-b" + DISK_AFFECTED_PERC: "100" + EXPERIMENT_IMAGE: "${{ github.event.inputs.experimentImage }}" + EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}" + CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}" + run: make gcp-vm-disk-loss-by-label + - name: Delete the VM Instance and target Disk Volumes if: always() run: | gcloud compute instances delete litmus-e2e-vm-${{ github.run_number }} \ - --zone=us-central1-a \ + --zone=us-east1-b \ --delete-disks=all \ --quiet diff --git a/.github/workflows/nightly-gcp-experiment-pipeline.yml b/.github/workflows/nightly-gcp-experiment-pipeline.yml index e2ae5d684..7c43412bb 100644 --- a/.github/workflows/nightly-gcp-experiment-pipeline.yml +++ b/.github/workflows/nightly-gcp-experiment-pipeline.yml @@ -62,7 +62,7 @@ jobs: run: | gcloud compute instances create litmus-e2e-first-vm-${{ github.run_number }} litmus-e2e-second-vm-${{ github.run_number }} \ --machine-type=f1-micro \ - --zone=us-central1-a + --zone=us-east1-b - name: Litmus Infra Setup if: always() @@ -73,14 +73,14 @@ jobs: env: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} VM_INSTANCE_NAMES: "litmus-e2e-first-vm-${{ github.run_number }},litmus-e2e-second-vm-${{ github.run_number }}" - INSTANCE_ZONES: "us-central1-a,us-central1-a" + INSTANCE_ZONES: "us-east1-b,us-east1-b" run: make gcp-vm-instance-stop - name: Delete target GCP VM Instances if: always() run: | gcloud compute instances delete litmus-e2e-first-vm-${{ github.run_number }} litmus-e2e-second-vm-${{ github.run_number }} \ - --zone=us-central1-a \ + --zone=us-east1-b \ --quiet - name: "[Debug]: check chaos resources" @@ -161,7 +161,7 @@ jobs: run: | gcloud compute instances create litmus-e2e-vm-${{ github.run_number }} \ --machine-type=f1-micro \ - --zone=us-central1-a \ + --zone=us-east1-b \ --create-disk name=litmus-e2e-first-disk-${{ github.run_number }},size=1GB,device-name=litmus-e2e-first-disk-${{ github.run_number }} \ --create-disk name=litmus-e2e-second-disk-${{ github.run_number }},size=1GB,device-name=litmus-e2e-second-disk-${{ github.run_number }} @@ -174,7 +174,7 @@ jobs: env: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} DISK_VOLUME_NAMES: "litmus-e2e-first-disk-${{ github.run_number }},litmus-e2e-second-disk-${{ github.run_number }}" - DISK_ZONES: "us-central1-a,us-central1-a" + DISK_ZONES: "us-east1-b,us-east1-b" DEVICE_NAMES: "litmus-e2e-first-disk-${{ github.run_number }},litmus-e2e-second-disk-${{ github.run_number }}" run: make gcp-vm-disk-loss @@ -182,7 +182,7 @@ jobs: if: always() run: | gcloud compute instances delete litmus-e2e-vm-${{ github.run_number }} \ - --zone=us-central1-a \ + --zone=us-east1-b \ --delete-disks=all \ --quiet diff --git a/Makefile b/Makefile index a9b3fb3bf..4a08729ea 100644 --- a/Makefile +++ b/Makefile @@ -257,6 +257,22 @@ gcp-vm-disk-loss: @echo "------------------------------------------" @go test platform/gcp/gcp-vm-disk-loss_test.go -v -count=1 -timeout=20m +.PHONY: gcp-vm-instance-stop-by-label +gcp-vm-instance-stop-by-label: + + @echo "------------------------------------------" + @echo "Running gcp-vm-instance-stop-by-label experiment" + @echo "------------------------------------------" + @go test platform/gcp/gcp-vm-instance-stop-by-label_test.go -v -count=1 -timeout=20m + +.PHONY: gcp-vm-disk-loss-by-label +gcp-vm-disk-loss-by-label: + + @echo "------------------------------------------" + @echo "Running gcp-vm-disk-loss-by-label experiment" + @echo "------------------------------------------" + @go test platform/gcp/gcp-vm-disk-loss-by-label_test.go -v -count=1 -timeout=20m + .PHONY: vm-poweroff vm-poweroff: diff --git a/platform/gcp/gcp-vm-disk-loss-by-label_test.go b/platform/gcp/gcp-vm-disk-loss-by-label_test.go new file mode 100644 index 000000000..841c4aad5 --- /dev/null +++ b/platform/gcp/gcp-vm-disk-loss-by-label_test.go @@ -0,0 +1,154 @@ +package tests + +import ( + "testing" + + "github.com/litmuschaos/chaos-operator/pkg/apis/litmuschaos/v1alpha1" + "github.com/litmuschaos/litmus-e2e/pkg" + "github.com/litmuschaos/litmus-e2e/pkg/environment" + "github.com/litmuschaos/litmus-e2e/pkg/log" + "github.com/litmuschaos/litmus-e2e/pkg/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/klog" +) + +func TestGoGCPVMDiskLossByLabel(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "BDD test") +} + +//BDD for testing experiment +var _ = Describe("BDD of gcp-vm-disk-loss-by-label experiment", func() { + + // BDD TEST CASE 1 - gcp-vm-disk-loss-by-label in parallel mode + Context("Check for gcp-vm-disk-loss-by-label experiment", func() { + + It("Should check for the gcp vm disk loss by label in parallel", func() { + + testsDetails := types.TestDetails{} + clients := environment.ClientSets{} + chaosExperiment := v1alpha1.ChaosExperiment{} + chaosEngine := v1alpha1.ChaosEngine{} + + //Getting kubeConfig and Generate ClientSets + By("[PreChaos]: Getting kubeconfig and generate clientset") + err := clients.GenerateClientSetFromKubeConfig() + Expect(err).To(BeNil(), "Unable to Get the kubeconfig, due to {%v}", err) + + //Fetching all the default ENV + By("[PreChaos]: Fetching all default ENVs") + klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) + environment.GetENV(&testsDetails, "gcp-vm-disk-loss-by-label", "gcp-disk-engine-par") + log.Infof("[Info]: The target disk volumes are: %v", testsDetails.DiskVolumeNames) + + testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/rbac.yaml" + testsDetails.ExperimentPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/experiment.yaml" + testsDetails.EnginePath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/engine.yaml" + testsDetails.ChaosNamespace = "default" + testsDetails.AppNS = "default" + + // Checking the chaos operator running status + By("[Status]: Checking chaos operator status") + err = pkg.OperatorStatusCheck(&testsDetails, clients) + Expect(err).To(BeNil(), "Operator status check failed, due to {%v}", err) + + // Prepare Chaos Execution + By("[Prepare]: Prepare Chaos Execution") + err = pkg.PrepareChaos(&testsDetails, &chaosExperiment, &chaosEngine, clients, false) + Expect(err).To(BeNil(), "fail to prepare chaos, due to {%v}", err) + + //Checking runner pod running state + By("[Status]: Runner pod running status check") + err = pkg.RunnerPodStatus(&testsDetails, testsDetails.AppNS, clients) + Expect(err).To(BeNil(), "Runner pod status check failed, due to {%v}", err) + + //Chaos pod running status check + err = pkg.ChaosPodStatus(&testsDetails, clients) + Expect(err).To(BeNil(), "Chaos pod status check failed, due to {%v}", err) + + //Waiting for chaos pod to get completed + //And Print the logs of the chaos pod + By("[Status]: Wait for chaos pod completion and then print logs") + err = pkg.ChaosPodLogs(&testsDetails, clients) + Expect(err).To(BeNil(), "Fail to get the experiment chaos pod logs, due to {%v}", err) + + //Checking the chaosresult verdict + By("[Verdict]: Checking the chaosresult verdict") + err = pkg.ChaosResultVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChasoResult Verdict check failed, due to {%v}", err) + + //Checking chaosengine verdict + By("Checking the Verdict of Chaos Engine") + err = pkg.ChaosEngineVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChaosEngine Verdict check failed, due to {%v}", err) + + }) + }) + + // BDD TEST CASE 2 - gcp-vm-disk-loss-by-label in serial mode + Context("Check for gcp-vm-disk-loss-by-label experiment", func() { + + It("Should check for the gcp vm disk loss by label in serial", func() { + + testsDetails := types.TestDetails{} + clients := environment.ClientSets{} + chaosExperiment := v1alpha1.ChaosExperiment{} + chaosEngine := v1alpha1.ChaosEngine{} + + //Getting kubeConfig and Generate ClientSets + By("[PreChaos]: Getting kubeconfig and generate clientset") + err := clients.GenerateClientSetFromKubeConfig() + Expect(err).To(BeNil(), "Unable to Get the kubeconfig, due to {%v}", err) + + //Fetching all the default ENV + By("[PreChaos]: Fetching all default ENVs") + klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) + environment.GetENV(&testsDetails, "gcp-vm-disk-loss-by-label", "gcp-disk-engine-ser") + log.Infof("[Info]: The target disk volumes are: %v", testsDetails.DiskVolumeNames) + + testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/rbac.yaml" + testsDetails.ExperimentPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/experiment.yaml" + testsDetails.EnginePath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/engine.yaml" + testsDetails.Sequence = "serial" + testsDetails.ChaosNamespace = "default" + testsDetails.AppNS = "default" + + // Checking the chaos operator running status + By("[Status]: Checking chaos operator status") + err = pkg.OperatorStatusCheck(&testsDetails, clients) + Expect(err).To(BeNil(), "Operator status check failed, due to {%v}", err) + + // Prepare Chaos Execution + By("[Prepare]: Prepare Chaos Execution") + err = pkg.PrepareChaos(&testsDetails, &chaosExperiment, &chaosEngine, clients, false) + Expect(err).To(BeNil(), "fail to prepare chaos, due to {%v}", err) + + //Checking runner pod running state + By("[Status]: Runner pod running status check") + err = pkg.RunnerPodStatus(&testsDetails, testsDetails.AppNS, clients) + Expect(err).To(BeNil(), "Runner pod status check failed, due to {%v}", err) + + //Chaos pod running status check + err = pkg.ChaosPodStatus(&testsDetails, clients) + Expect(err).To(BeNil(), "Chaos pod status check failed, due to {%v}", err) + + //Waiting for chaos pod to get completed + //And Print the logs of the chaos pod + By("[Status]: Wait for chaos pod completion and then print logs") + err = pkg.ChaosPodLogs(&testsDetails, clients) + Expect(err).To(BeNil(), "Fail to get the experiment chaos pod logs, due to {%v}", err) + + //Checking the chaosresult verdict + By("[Verdict]: Checking the chaosresult verdict") + err = pkg.ChaosResultVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChasoResult Verdict check failed, due to {%v}", err) + + //Checking chaosengine verdict + By("Checking the Verdict of Chaos Engine") + err = pkg.ChaosEngineVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChaosEngine Verdict check failed, due to {%v}", err) + + }) + }) +}) diff --git a/platform/gcp/gcp-vm-instance-stop-by-label_test.go b/platform/gcp/gcp-vm-instance-stop-by-label_test.go new file mode 100644 index 000000000..af7e8af10 --- /dev/null +++ b/platform/gcp/gcp-vm-instance-stop-by-label_test.go @@ -0,0 +1,151 @@ +package tests + +import ( + "testing" + + "github.com/litmuschaos/chaos-operator/pkg/apis/litmuschaos/v1alpha1" + "github.com/litmuschaos/litmus-e2e/pkg" + "github.com/litmuschaos/litmus-e2e/pkg/environment" + "github.com/litmuschaos/litmus-e2e/pkg/log" + "github.com/litmuschaos/litmus-e2e/pkg/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/klog" +) + +func TestGoGCPVMInstanceStopByLabel(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "BDD test") +} + +var _ = Describe("BDD of gcp-vm-instance-stop-by-label experiment", func() { + + // BDD TEST CASE 1 - gcp-vm-instance-stop-by-label in parallel mode + Context("Check for gcp-vm-instance-stop-by-label experiment", func() { + It("Should check for the gcp vm instance stop by label in parallel", func() { + + testsDetails := types.TestDetails{} + clients := environment.ClientSets{} + chaosExperiment := v1alpha1.ChaosExperiment{} + chaosEngine := v1alpha1.ChaosEngine{} + + //Getting kubeConfig and Generate ClientSets + By("[PreChaos]: Getting kubeconfig and generate clientset") + err := clients.GenerateClientSetFromKubeConfig() + Expect(err).To(BeNil(), "Unable to Get the kubeconfig, due to {%v}", err) + + //Fetching all the default ENV + By("[PreChaos]: Fetching all default ENVs") + klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) + environment.GetENV(&testsDetails, "gcp-vm-instance-stop-by-label", "gcp-vm-engine-par") + log.Infof("[Info]: The target VM instances are: %v", testsDetails.VMInstanceNames) + + testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/rbac.yaml" + testsDetails.ExperimentPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/experiment.yaml" + testsDetails.EnginePath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/engine.yaml" + testsDetails.ChaosNamespace = "default" + testsDetails.AppNS = "default" + + // Checking the chaos operator running status + By("[Status]: Checking chaos operator status") + err = pkg.OperatorStatusCheck(&testsDetails, clients) + Expect(err).To(BeNil(), "Operator status check failed, due to {%v}", err) + + // Prepare Chaos Execution + By("[Prepare]: Prepare Chaos Execution") + err = pkg.PrepareChaos(&testsDetails, &chaosExperiment, &chaosEngine, clients, false) + Expect(err).To(BeNil(), "fail to prepare chaos, due to {%v}", err) + + //Checking runner pod running state + By("[Status]: Runner pod running status check") + err = pkg.RunnerPodStatus(&testsDetails, testsDetails.AppNS, clients) + Expect(err).To(BeNil(), "Runner pod status check failed, due to {%v}", err) + + //Chaos pod running status check + err = pkg.ChaosPodStatus(&testsDetails, clients) + Expect(err).To(BeNil(), "Chaos pod status check failed, due to {%v}", err) + + //Waiting for chaos pod to get completed + //And Print the logs of the chaos pod + By("[Status]: Wait for chaos pod completion and then print logs") + err = pkg.ChaosPodLogs(&testsDetails, clients) + Expect(err).To(BeNil(), "Fail to get the experiment chaos pod logs, due to {%v}", err) + + //Checking the chaosresult verdict + By("[Verdict]: Checking the chaosresult verdict") + err = pkg.ChaosResultVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChasoResult Verdict check failed, due to {%v}", err) + + //Checking chaosengine verdict + By("Checking the Verdict of Chaos Engine") + err = pkg.ChaosEngineVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChaosEngine Verdict check failed, due to {%v}", err) + }) + }) + + // BDD TEST CASE 2 - gcp-vm-instance-stop-by-label in serial mode + Context("Check for gcp-vm-instance-stop-by-label experiment", func() { + + It("Should check for the gcp vm instance stop by label in serial", func() { + + testsDetails := types.TestDetails{} + clients := environment.ClientSets{} + chaosExperiment := v1alpha1.ChaosExperiment{} + chaosEngine := v1alpha1.ChaosEngine{} + + //Getting kubeConfig and Generate ClientSets + By("[PreChaos]: Getting kubeconfig and generate clientset") + err := clients.GenerateClientSetFromKubeConfig() + Expect(err).To(BeNil(), "Unable to Get the kubeconfig, due to {%v}", err) + + //Fetching all the default ENV + By("[PreChaos]: Fetching all default ENVs") + klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) + environment.GetENV(&testsDetails, "gcp-vm-instance-stop-by-label", "gcp-vm-engine-ser") + log.Infof("[Info]: The target VM instances are: %v", testsDetails.VMInstanceNames) + + testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/rbac.yaml" + testsDetails.ExperimentPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/experiment.yaml" + testsDetails.EnginePath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/engine.yaml" + testsDetails.Sequence = "serial" + testsDetails.ChaosNamespace = "default" + testsDetails.AppNS = "default" + + // Checking the chaos operator running status + By("[Status]: Checking chaos operator status") + err = pkg.OperatorStatusCheck(&testsDetails, clients) + Expect(err).To(BeNil(), "Operator status check failed, due to {%v}", err) + + // Prepare Chaos Execution + By("[Prepare]: Prepare Chaos Execution") + err = pkg.PrepareChaos(&testsDetails, &chaosExperiment, &chaosEngine, clients, false) + Expect(err).To(BeNil(), "fail to prepare chaos, due to {%v}", err) + + //Checking runner pod running state + By("[Status]: Runner pod running status check") + err = pkg.RunnerPodStatus(&testsDetails, testsDetails.AppNS, clients) + Expect(err).To(BeNil(), "Runner pod status check failed, due to {%v}", err) + + //Chaos pod running status check + err = pkg.ChaosPodStatus(&testsDetails, clients) + Expect(err).To(BeNil(), "Chaos pod status check failed, due to {%v}", err) + + //Waiting for chaos pod to get completed + //And Print the logs of the chaos pod + By("[Status]: Wait for chaos pod completion and then print logs") + err = pkg.ChaosPodLogs(&testsDetails, clients) + Expect(err).To(BeNil(), "Fail to get the experiment chaos pod logs, due to {%v}", err) + + //Checking the chaosresult verdict + By("[Verdict]: Checking the chaosresult verdict") + err = pkg.ChaosResultVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChasoResult Verdict check failed, due to {%v}", err) + + //Checking chaosengine verdict + By("Checking the Verdict of Chaos Engine") + err = pkg.ChaosEngineVerdict(&testsDetails, clients) + Expect(err).To(BeNil(), "ChaosEngine Verdict check failed, due to {%v}", err) + + }) + }) +}) From f64bd2ef9664343156dcd974047f4c780cd2d71b Mon Sep 17 00:00:00 2001 From: Neelanjan Manna Date: Wed, 18 May 2022 11:08:25 +0530 Subject: [PATCH 2/6] updated operator setup envs; environment and types files Signed-off-by: Neelanjan Manna --- pkg/environment/environment.go | 4 ++ pkg/install.go | 10 +++ pkg/types/types.go | 126 +++++++++++++++++---------------- 3 files changed, 79 insertions(+), 61 deletions(-) diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 41f0413d0..deccd9c97 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -39,6 +39,10 @@ func GetENV(testDetails *types.TestDetails, expName, engineName string) { testDetails.DiskVolumeNames = Getenv("DISK_VOLUME_NAMES", "") testDetails.DiskZones = Getenv("DISK_ZONES", "") testDetails.DeviceNames = Getenv("DEVICE_NAMES", "") + testDetails.InstanceLabel = Getenv("INSTANCE_LABEL", "") + testDetails.InstanceAffectedPercentage = Getenv("INSTANCE_AFFECTED_PERC", "") + testDetails.DiskVolumeLabel = Getenv("DISK_VOLUME_LABEL", "") + testDetails.DiskAffectedPercentage = Getenv("DISK_AFFECTED_PERC", "") testDetails.VMIds = Getenv("APP_VM_MOIDS", "") testDetails.Region = Getenv("REGION", "us-west-1") testDetails.UpdateWebsite = Getenv("UPDATE_WEBSITE", "false") diff --git a/pkg/install.go b/pkg/install.go index 0bfa08efe..cdb8b1580 100644 --- a/pkg/install.go +++ b/pkg/install.go @@ -336,6 +336,16 @@ func setEngineVar(chaosEngine *v1alpha1.ChaosEngine, testsDetails *types.TestDet SetEnv("DISK_VOLUME_NAMES", testsDetails.DiskVolumeNames). SetEnv("DISK_ZONES", testsDetails.DiskZones). SetEnv("DEVICE_NAMES", testsDetails.DeviceNames) + case "gcp-vm-instance-stop-by-label": + envDetails.SetEnv("GCP_PROJECT_ID", testsDetails.GCPProjectID). + SetEnv("INSTANCE_LABEL", testsDetails.InstanceLabel). + SetEnv("INSTANCE_ZONES", testsDetails.InstanceZones). + SetEnv("INSTANCE_AFFECTED_PERC", testsDetails.InstanceAffectedPercentage) + case "gcp-vm-disk-loss-by-label": + envDetails.SetEnv("GCP_PROJECT_ID", testsDetails.GCPProjectID). + SetEnv("DISK_VOLUME_LABEL", testsDetails.DiskVolumeLabel). + SetEnv("DISK_ZONES", testsDetails.DiskZones). + SetEnv("DISK_AFFECTED_PERC", testsDetails.DiskAffectedPercentage) case "vm-poweroff": envDetails.SetEnv("APP_VM_MOIDS", testsDetails.VMIds) case "process-kill": diff --git a/pkg/types/types.go b/pkg/types/types.go index 71bfb2185..c8cf9810f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -2,65 +2,69 @@ package types // TestDetails is for collecting all the test-related details type TestDetails struct { - ExperimentName string - EngineName string - OperatorName string - ChaosNamespace string - RbacPath string - ExperimentPath string - EnginePath string - AnsibleRbacPath string - AnsibleExperimentPath string - AnsibleEnginePath string - AnsibleExperimentImage string - AppNS string - AppLabel string - JobCleanUpPolicy string - AnnotationCheck string - ApplicationNodeName string - ExperimentImage string - InstallLitmus string - OperatorImage string - ImagePullPolicy string - ExperimentImagePullPolicy string - RunnerImage string - ChaosDuration int - AdminRbacPath string - ChaosServiceAccount string - NewExperimentName string - NodeSelectorName string - Delay int - Duration int - TargetPod string - LibImage string - FillPercentage int - CPUKillCommand string - MemoryKillCommand string - NetworkLatency string - UpdateWebsite string - InstanceID string - InstanceTag string - EBSVolumeID string - EBSVolumeTag string - VMInstanceNames string - GCPProjectID string - InstanceZones string - DiskVolumeNames string - DiskZones string - DeviceNames string - VMIds string - Region string - Lib string - PodsAffectedPercentage string - Sequence string - Version string - TargetNodes string - NodeLabel string - AzureResourceGroup string - AzureInstanceName string - AzureDiskName string - AzureScaleSet string - ProcessIds string - Args string - Command string + ExperimentName string + EngineName string + OperatorName string + ChaosNamespace string + RbacPath string + ExperimentPath string + EnginePath string + AnsibleRbacPath string + AnsibleExperimentPath string + AnsibleEnginePath string + AnsibleExperimentImage string + AppNS string + AppLabel string + JobCleanUpPolicy string + AnnotationCheck string + ApplicationNodeName string + ExperimentImage string + InstallLitmus string + OperatorImage string + ImagePullPolicy string + ExperimentImagePullPolicy string + RunnerImage string + ChaosDuration int + AdminRbacPath string + ChaosServiceAccount string + NewExperimentName string + NodeSelectorName string + Delay int + Duration int + TargetPod string + LibImage string + FillPercentage int + CPUKillCommand string + MemoryKillCommand string + NetworkLatency string + UpdateWebsite string + InstanceID string + InstanceTag string + EBSVolumeID string + EBSVolumeTag string + VMInstanceNames string + GCPProjectID string + InstanceZones string + DiskVolumeNames string + DiskZones string + DeviceNames string + InstanceLabel string + InstanceAffectedPercentage string + DiskVolumeLabel string + DiskAffectedPercentage string + VMIds string + Region string + Lib string + PodsAffectedPercentage string + Sequence string + Version string + TargetNodes string + NodeLabel string + AzureResourceGroup string + AzureInstanceName string + AzureDiskName string + AzureScaleSet string + ProcessIds string + Args string + Command string } From 166d9fed5d2b14e8518591287dbcd135e87b7cc6 Mon Sep 17 00:00:00 2001 From: Neelanjan Manna Date: Wed, 18 May 2022 11:20:33 +0530 Subject: [PATCH 3/6] updated needs Signed-off-by: Neelanjan Manna --- .github/workflows/gcp-experiment-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcp-experiment-pipeline.yml b/.github/workflows/gcp-experiment-pipeline.yml index ec32c7e95..23c7d7192 100644 --- a/.github/workflows/gcp-experiment-pipeline.yml +++ b/.github/workflows/gcp-experiment-pipeline.yml @@ -136,7 +136,7 @@ jobs: runs-on: ubuntu-latest env: KUBECONFIG: /etc/rancher/k3s/k3s.yaml - needs: GCP_VM_Instance_Stop + needs: GCP_VM_Instance_Stop_by_Name_And_Label steps: #Install and configure a k3s cluster From 7e8a95d8f00acdb1b3a27953ec57cf66dfe844b2 Mon Sep 17 00:00:00 2001 From: Neelanjan Manna Date: Wed, 18 May 2022 12:36:28 +0530 Subject: [PATCH 4/6] updated engine names Signed-off-by: Neelanjan Manna --- platform/gcp/gcp-vm-disk-loss-by-label_test.go | 4 ++-- platform/gcp/gcp-vm-instance-stop-by-label_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platform/gcp/gcp-vm-disk-loss-by-label_test.go b/platform/gcp/gcp-vm-disk-loss-by-label_test.go index 841c4aad5..f0eeb0561 100644 --- a/platform/gcp/gcp-vm-disk-loss-by-label_test.go +++ b/platform/gcp/gcp-vm-disk-loss-by-label_test.go @@ -39,7 +39,7 @@ var _ = Describe("BDD of gcp-vm-disk-loss-by-label experiment", func() { //Fetching all the default ENV By("[PreChaos]: Fetching all default ENVs") klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) - environment.GetENV(&testsDetails, "gcp-vm-disk-loss-by-label", "gcp-disk-engine-par") + environment.GetENV(&testsDetails, "gcp-vm-disk-loss-by-label", "gcp-disk-label-engine-par") log.Infof("[Info]: The target disk volumes are: %v", testsDetails.DiskVolumeNames) testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/rbac.yaml" @@ -104,7 +104,7 @@ var _ = Describe("BDD of gcp-vm-disk-loss-by-label experiment", func() { //Fetching all the default ENV By("[PreChaos]: Fetching all default ENVs") klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) - environment.GetENV(&testsDetails, "gcp-vm-disk-loss-by-label", "gcp-disk-engine-ser") + environment.GetENV(&testsDetails, "gcp-vm-disk-loss-by-label", "gcp-disk-label-engine-ser") log.Infof("[Info]: The target disk volumes are: %v", testsDetails.DiskVolumeNames) testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-disk-loss-by-label/rbac.yaml" diff --git a/platform/gcp/gcp-vm-instance-stop-by-label_test.go b/platform/gcp/gcp-vm-instance-stop-by-label_test.go index af7e8af10..797ca57b4 100644 --- a/platform/gcp/gcp-vm-instance-stop-by-label_test.go +++ b/platform/gcp/gcp-vm-instance-stop-by-label_test.go @@ -37,7 +37,7 @@ var _ = Describe("BDD of gcp-vm-instance-stop-by-label experiment", func() { //Fetching all the default ENV By("[PreChaos]: Fetching all default ENVs") klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) - environment.GetENV(&testsDetails, "gcp-vm-instance-stop-by-label", "gcp-vm-engine-par") + environment.GetENV(&testsDetails, "gcp-vm-instance-stop-by-label", "gcp-vm-label-engine-par") log.Infof("[Info]: The target VM instances are: %v", testsDetails.VMInstanceNames) testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/rbac.yaml" @@ -101,7 +101,7 @@ var _ = Describe("BDD of gcp-vm-instance-stop-by-label experiment", func() { //Fetching all the default ENV By("[PreChaos]: Fetching all default ENVs") klog.Infof("[PreReq]: Getting the ENVs for the %v test", testsDetails.ExperimentName) - environment.GetENV(&testsDetails, "gcp-vm-instance-stop-by-label", "gcp-vm-engine-ser") + environment.GetENV(&testsDetails, "gcp-vm-instance-stop-by-label", "gcp-vm-label-engine-ser") log.Infof("[Info]: The target VM instances are: %v", testsDetails.VMInstanceNames) testsDetails.RbacPath = "https://hub.litmuschaos.io/api/chaos/master?file=charts/gcp/gcp-vm-instance-stop-by-label/rbac.yaml" From 91e20203b0dae747e7ac393ac804c8d3b5b315c4 Mon Sep 17 00:00:00 2001 From: Neelanjan Manna Date: Wed, 18 May 2022 16:30:25 +0530 Subject: [PATCH 5/6] updated the nightly pipeline Signed-off-by: Neelanjan Manna --- .../nightly-gcp-experiment-pipeline.yml | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-gcp-experiment-pipeline.yml b/.github/workflows/nightly-gcp-experiment-pipeline.yml index 7c43412bb..1077ba52f 100644 --- a/.github/workflows/nightly-gcp-experiment-pipeline.yml +++ b/.github/workflows/nightly-gcp-experiment-pipeline.yml @@ -5,16 +5,14 @@ on: - cron: "30 22 * * *" # Daily 02:30 AM in midnight jobs: - GCP_VM_Instance_Stop: + GCP_VM_Instance_Stop_by_Name_And_Label: runs-on: ubuntu-latest - env: + env: KUBECONFIG: /etc/rancher/k3s/k3s.yaml steps: #Install and configure a k3s cluster - name: Installing Prerequisites (K3S Cluster) - env: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml run: | curl -sfL https://get.k3s.io | sh -s - --docker --write-kubeconfig-mode 664 kubectl wait node --all --for condition=ready --timeout=90s @@ -62,11 +60,15 @@ jobs: run: | gcloud compute instances create litmus-e2e-first-vm-${{ github.run_number }} litmus-e2e-second-vm-${{ github.run_number }} \ --machine-type=f1-micro \ - --zone=us-east1-b + --zone=us-east1-b \ + --labels=vm=litmus-e2e-vm-${{ github.run_number }} - name: Litmus Infra Setup if: always() run: make build-litmus + env: + OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}" + RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}" - name: Run GCP VM Instance Stop experiment in serial & parallel mode if: always() @@ -76,6 +78,15 @@ jobs: INSTANCE_ZONES: "us-east1-b,us-east1-b" run: make gcp-vm-instance-stop + - name: Run GCP VM Instance Stop By Label experiment in serial & parallel mode + if: always() + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + INSTANCE_LABEL: "vm : litmus-e2e-vm-${{ github.run_number }}" + INSTANCE_ZONES: "us-east1-b" + INSTANCE_AFFECTED_PERC: "100" + run: make gcp-vm-instance-stop-by-label + - name: Delete target GCP VM Instances if: always() run: | @@ -98,18 +109,18 @@ jobs: cat logs.txt - name: Litmus Cleanup - if: ${{ always() }} + if: ${{ always() }} run: make litmus-cleanup - name: Deleting K3S cluster if: always() run: /usr/local/bin/k3s-uninstall.sh - GCP_VM_Disk_Loss: + GCP_VM_Disk_Loss_By_Name_And_Label: runs-on: ubuntu-latest env: KUBECONFIG: /etc/rancher/k3s/k3s.yaml - needs: GCP_VM_Instance_Stop + needs: GCP_VM_Instance_Stop_by_Name_And_Label steps: #Install and configure a k3s cluster @@ -164,10 +175,15 @@ jobs: --zone=us-east1-b \ --create-disk name=litmus-e2e-first-disk-${{ github.run_number }},size=1GB,device-name=litmus-e2e-first-disk-${{ github.run_number }} \ --create-disk name=litmus-e2e-second-disk-${{ github.run_number }},size=1GB,device-name=litmus-e2e-second-disk-${{ github.run_number }} + gcloud compute disks add-labels litmus-e2e-first-disk-${{ github.run_number }} --labels=disk=litmus-e2e-disk-${{ github.run_number }} --zone=us-east1-b + gcloud compute disks add-labels litmus-e2e-second-disk-${{ github.run_number }} --labels=disk=litmus-e2e-disk-${{ github.run_number }} --zone=us-east1-b - name: Litmus Infra Setup if: always() run: make build-litmus + env: + OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}" + RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}" - name: Run GCP VM Disk Loss experiment in serial & parallel mode if: always() @@ -178,6 +194,15 @@ jobs: DEVICE_NAMES: "litmus-e2e-first-disk-${{ github.run_number }},litmus-e2e-second-disk-${{ github.run_number }}" run: make gcp-vm-disk-loss + - name: Run GCP VM Disk Loss By Label experiment in serial & parallel mode + if: always() + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + DISK_VOLUME_LABEL: "disk : litmus-e2e-disk-${{ github.run_number }}" + DISK_ZONES: "us-east1-b" + DISK_AFFECTED_PERC: "100" + run: make gcp-vm-disk-loss-by-label + - name: Delete the VM Instance and target Disk Volumes if: always() run: | @@ -201,7 +226,7 @@ jobs: cat logs.txt - name: Litmus Cleanup - if: ${{ always() }} + if: ${{ always() }} run: make litmus-cleanup - name: Deleting K3S cluster From 9bf297734a7ac49e1ab7dbdb4e15652365ad6074 Mon Sep 17 00:00:00 2001 From: Neelanjan Manna Date: Tue, 24 May 2022 15:28:14 +0530 Subject: [PATCH 6/6] removed make envs Signed-off-by: Neelanjan Manna --- .github/workflows/nightly-gcp-experiment-pipeline.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/nightly-gcp-experiment-pipeline.yml b/.github/workflows/nightly-gcp-experiment-pipeline.yml index 1077ba52f..a304ab5d0 100644 --- a/.github/workflows/nightly-gcp-experiment-pipeline.yml +++ b/.github/workflows/nightly-gcp-experiment-pipeline.yml @@ -66,9 +66,6 @@ jobs: - name: Litmus Infra Setup if: always() run: make build-litmus - env: - OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}" - RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}" - name: Run GCP VM Instance Stop experiment in serial & parallel mode if: always() @@ -181,9 +178,6 @@ jobs: - name: Litmus Infra Setup if: always() run: make build-litmus - env: - OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}" - RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}" - name: Run GCP VM Disk Loss experiment in serial & parallel mode if: always()