From 931c23f1992d980711de20a94571a1e7c6869eb9 Mon Sep 17 00:00:00 2001 From: Swapnil Bobade Date: Mon, 9 Dec 2024 12:30:42 +0530 Subject: [PATCH] Automation for ocp-cluster-etcdc-operator Signed-off-by: Swapnil Bobade --- examples/all.yaml | 6 ++ playbooks/main.yml | 5 +- playbooks/ocp-cluster-etcd-operator.yml | 6 ++ .../roles/ocp-cluster-etcd-operator/README.md | 53 +++++++++++ .../defaults/main.yaml | 6 ++ .../files/api-intensive.yaml | 34 +++++++ .../files/cluster-density.yaml | 9 ++ .../files/crd-scale.yaml | 27 ++++++ .../kubelet-density-cni-networkpolicy.yaml | 9 ++ .../files/kubelet-density-cni.yaml | 9 ++ .../files/kubelet-density-heavy.yaml | 9 ++ .../files/kubelet-density.yaml | 46 ++++++++++ .../tasks/install-kube-burner.yaml | 25 +++++ .../ocp-cluster-etcd-operator/tasks/main.yaml | 91 +++++++++++++++++++ 14 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 playbooks/ocp-cluster-etcd-operator.yml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/README.md create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/defaults/main.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/files/api-intensive.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/files/cluster-density.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/files/crd-scale.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni-networkpolicy.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-heavy.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/tasks/install-kube-burner.yaml create mode 100644 playbooks/roles/ocp-cluster-etcd-operator/tasks/main.yaml diff --git a/examples/all.yaml b/examples/all.yaml index d8902bad..da486362 100644 --- a/examples/all.yaml +++ b/examples/all.yaml @@ -411,3 +411,9 @@ ocp_logging_tests_private_e2e_git_branch: "" #Git repo branch for e2e tests golang_tarball_logging_e2e: "https://dl.google.com/go/go1.22.6.linux-ppc64le.tar.gz" #HTTP URL for golang tarball logging_e2e_testcases: "" #Set to either all or critical +# ocp-cluster-etcd-operator vars +cluster_etcd_enabled: false +etcd_namespace: "openshift-etcd" +kube_burner_repo: "https://github.com/kube-burner/kube-burner.git" +kube_burner_dir: "/root/kube-burner" +kube_burner_path: "/root/kube-burner/examples/workloads" diff --git a/playbooks/main.yml b/playbooks/main.yml index 2fe7456c..d5926da3 100644 --- a/playbooks/main.yml +++ b/playbooks/main.yml @@ -120,4 +120,7 @@ when: ocp_etcd_key_rotation_enable is defined and ocp_etcd_key_rotation_enable - import_playbook: ocp-logging-tests-private-e2e.yml - when: ocp_logging_tests_private_e2e_enabled is defined and ocp_logging_tests_private_e2e_enabled \ No newline at end of file + when: ocp_logging_tests_private_e2e_enabled is defined and ocp_logging_tests_private_e2e_enabled + +- import_playbook: ocp-cluster-etcd-operator.yml + when: cluster_etcd_enabled is defined and cluster_etcd_enabled diff --git a/playbooks/ocp-cluster-etcd-operator.yml b/playbooks/ocp-cluster-etcd-operator.yml new file mode 100644 index 00000000..3a690f56 --- /dev/null +++ b/playbooks/ocp-cluster-etcd-operator.yml @@ -0,0 +1,6 @@ +--- + +- name: OCP cluster etcd operator verification + hosts: bastion + roles: + - ocp-cluster-etcd-operator \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/README.md b/playbooks/roles/ocp-cluster-etcd-operator/README.md new file mode 100644 index 00000000..6400af0c --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/README.md @@ -0,0 +1,53 @@ +ocp-cluster-etcd operator +========================= + +The control plane hardware speed values are "Standard", "Slower", or the default, which is "". +The default setting allows the system to decide which speed to use. This value enables upgrades from versions where this feature does not exist, as the system can select values from previous versions + +By selecting one of the other values, you are overriding the default. If you see many leader elections due to timeouts or missed heartbeats and your system is set to "" or "Standard", set the hardware speed to "Slower" to make the system more tolerant to the increased latency + +This playbook will handle below: +- It will check the DaemonSet for HEARTBEAT_INTERVAL and LEADER_ELECTION_TIMEOUT parameters +- Change the control plane hardware speed tolerance to "Slower" +- Run the acme-air workload and e2e conformance test suite +- Also it will run the kube-burner workload test + +Requirements +------------ + + - Running OCP 4.x cluster # better on PowerVS + +Role Variables +-------------- +| Variable | Required | Default | Comments | +|--------------------------------|----------|-------------|------------------------------------------------| +| cluster_etcd_enabled | no | false | Set it to true to run this playbook | +| etcd_namespace | no | false | cluster etcd namespace. | +| kube_burner_repo | no | false | https://github.com/kube-burner/kube-burner.git | +| kube_burner_dir | no | false | /root/kube-burner | +| kube_burner_path | no | false | /root/kube-burner/examples/workloads | + + +Dependencies +------------ + + - Role ocp-e2e # to run e2e conformance test suite + +Example Playbook +---------------- + +``` + - name: Verify cluster etcd operator + include_role: + name: ocp-cluster-etcd-operator +``` + +License +------- + +See LICENCE.txt + +Author Information +------------------ + +swapnil.bobade@ibm.com diff --git a/playbooks/roles/ocp-cluster-etcd-operator/defaults/main.yaml b/playbooks/roles/ocp-cluster-etcd-operator/defaults/main.yaml new file mode 100644 index 00000000..ad27275e --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/defaults/main.yaml @@ -0,0 +1,6 @@ +## ocp_cluster_etcd-operator vars +cluster_etcd_enabled: false +etcd_namespace: openshift-etcd +kube_burner_repo: "https://github.com/kube-burner/kube-burner.git" +kube_burner_dir: "/root/kube-burner" +kube_burner_path: "/root/kube-burner/examples/workloads" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/files/api-intensive.yaml b/playbooks/roles/ocp-cluster-etcd-operator/files/api-intensive.yaml new file mode 100644 index 00000000..485c5eca --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/files/api-intensive.yaml @@ -0,0 +1,34 @@ +--- +- name: Read the content of the file + slurp: + src: "{{ kube_burner_path }}/api-intensive/api-intensive.yml" + register: file_content + +- name: Decode the file content and extract first 4 lines + set_fact: + first_four_lines: "{{ (file_content['content'] | b64decode).split('\n')[3:7] }}" + +- name: Add '#' to the first 4 lines + lineinfile: + path: "{{ kube_burner_path }}/api-intensive/api-intensive.yml" + regexp: "^{{ item | regex_escape }}" + line: "# {{ item }}" + backrefs: yes + state: present + loop: "{{ first_four_lines }}" + +- name: Update jobIterations to 100 + lineinfile: + path: "{{ kube_burner_path }}/api-intensive/api-intensive.yml" + regexp: '^(\s*)jobIterations:\s*700\s*$' + line: '\1jobIterations: 100' + backrefs: yes + +- name: Run the api-intensity workload + shell: kube-burner init -c api-intensive.yml + args: + chdir: "{{ kube_burner_dir }}/examples/workloads/api-intensive" + register: api_intensive_output + failed_when: + - api_intensive_output.rc != 0 + - "'👋 Exiting kube-burner' not in api_intensive_output.stdout" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/files/cluster-density.yaml b/playbooks/roles/ocp-cluster-etcd-operator/files/cluster-density.yaml new file mode 100644 index 00000000..1f73176c --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/files/cluster-density.yaml @@ -0,0 +1,9 @@ +--- +- name: Run the cluster density workload + shell: kube-burner init -c cluster-density.yml + args: + chdir: "{{ kube_burner_dir }}/examples/workloads/cluster-density" + register: cluster_density_output + failed_when: + - cluster_density_output.rc != 0 + - "'👋 Exiting kube-burner' not in cluster_density_output.stdout" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/files/crd-scale.yaml b/playbooks/roles/ocp-cluster-etcd-operator/files/crd-scale.yaml new file mode 100644 index 00000000..f29062b0 --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/files/crd-scale.yaml @@ -0,0 +1,27 @@ +--- +- name: Read the content of the file + slurp: + src: "{{ kube_burner_path }}/crd-scale/crd-scale.yml" + register: file_content + +- name: Decode the file content and extract first 4 lines + set_fact: + first_four_lines: "{{ (file_content['content'] | b64decode).split('\n')[3:7] }}" + +- name: Add '#' to the first 4 lines + lineinfile: + path: "{{ kube_burner_path }}/crd-scale/crd-scale.yml" + regexp: "^{{ item | regex_escape }}" + line: "# {{ item }}" + backrefs: yes + state: present + loop: "{{ first_four_lines }}" + +- name: Run the crd scale workload + shell: kube-burner init -c crd-scale.yml + args: + chdir: "{{ kube_burner_dir }}/examples/workloads/crd-scale" + register: crd_scale_output + failed_when: + - crd_scale_output.rc != 0 + - "'👋 Exiting kube-burner' not in crd_scale_output.stdout" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni-networkpolicy.yaml b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni-networkpolicy.yaml new file mode 100644 index 00000000..17dba15b --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni-networkpolicy.yaml @@ -0,0 +1,9 @@ +--- +- name: Run the kubelet-density-cni-networkpolicy workload + shell: kube-burner init -c kubelet-density-cni-networkpolicy.yml + args: + chdir: "{{ kube_burner_dir }}/examples/workloads/kubelet-density-cni-networkpolicy" + register: kube_den_cni_net_output + failed_when: + - kube_den_cni_net_output.rc != 0 + - "'👋 Exiting kube-burner' not in kube_den_cni_net_output.stdout" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni.yaml b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni.yaml new file mode 100644 index 00000000..136a5ea8 --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-cni.yaml @@ -0,0 +1,9 @@ +--- +- name: Run the kubelet-density-cni workload + shell: kube-burner init -c kubelet-density-cni.yml + args: + chdir: "{{ kube_burner_dir }}/examples/workloads/kubelet-density-cni" + register: kubelet_density_cni_output + failed_when: + - kubelet_density_cni_output.rc != 0 + - "'👋 Exiting kube-burner' not in kubelet_density_cni_output.stdout" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-heavy.yaml b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-heavy.yaml new file mode 100644 index 00000000..b1eff2b9 --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density-heavy.yaml @@ -0,0 +1,9 @@ +--- +- name: Run the kubelet-density-heavy workload + shell: kube-burner init -c kubelet-density-heavy.yml + args: + chdir: "{{ kube_burner_dir }}/examples/workloads/kubelet-density-heavy" + register: kube_den_heavy_output + failed_when: + - kube_den_heavy_output.rc != 0 + - "'👋 Exiting kube-burner' not in kube_den_heavy_output.stdout" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density.yaml b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density.yaml new file mode 100644 index 00000000..20395374 --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/files/kubelet-density.yaml @@ -0,0 +1,46 @@ +--- +# Add security to pod +- name: Insert allowPrivilegeEscalation + lineinfile: + path: "{{ kube_burner_path }}/kubelet-density/templates/pod.yml" + insertafter: 'privileged:' + line: " allowPrivilegeEscalation: false" + +- name: Insert allowPrivilegeEscalation + lineinfile: + path: "{{ kube_burner_path }}/kubelet-density/templates/pod.yml" + insertafter: 'allowPrivilegeEscalation: false' + line: " capabilities:" + +- name: Insert capabilities.drop + lineinfile: + path: "{{ kube_burner_path }}/kubelet-density/templates/pod.yml" + insertafter: 'capabilities:' + line: " drop: [\"ALL\"]" + +- name: Insert runAsNonRoot + lineinfile: + path: "{{ kube_burner_path }}/kubelet-density/templates/pod.yml" + insertafter: 'drop:' + line: " runAsNonRoot: true" + +- name: Insert seccompProfile + lineinfile: + path: "{{ kube_burner_path }}/kubelet-density/templates/pod.yml" + insertafter: 'runAsNonRoot: true' + line: " seccompProfile:" + +- name: Insert seccompProfile.type + lineinfile: + path: "{{ kube_burner_path }}/kubelet-density/templates/pod.yml" + insertafter: 'seccompProfile:' + line: " type: RuntimeDefault" + +- name: Run the cluster density workload + shell: kube-burner init -c kubelet-density.yml + args: + chdir: "{{ kube_burner_dir }}/examples/workloads/kubelet-density" + register: kubelet_density_output + failed_when: + - kubelet_density_output.rc != 0 + - "'👋 Exiting kube-burner' not in kubelet_density_output.stdout" \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/tasks/install-kube-burner.yaml b/playbooks/roles/ocp-cluster-etcd-operator/tasks/install-kube-burner.yaml new file mode 100644 index 00000000..fe28addb --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/tasks/install-kube-burner.yaml @@ -0,0 +1,25 @@ +--- + +- name: Download kube-burner git repo + get_url: + url: "https://github.com/kube-burner/kube-burner/releases/download/v1.10.4/kube-burner-V1.10.4-linux-ppc64le.tar.gz" + dest: "/tmp/kube-burner-V1.10.4-linux-ppc64le.tar.gz" + mode: '0644' + +- name: Extract kube-burner tarball + unarchive: + src: "/tmp/kube-burner-V1.10.4-linux-ppc64le.tar.gz" + dest: "/tmp/" + remote_src: yes # Use to indicate the file is already on the remote machine + creates: "/tmp/kube-burner-V1.10.4-linux-ppc64le" # Prevent extraction if the folder already exists + +- name: Verify extracted files + command: ls -la /tmp/ | grep kube + register: result + failed_when: result.rc == 0 + +- name: Move the kube-burner executable to /bin + command: "mv /tmp/kube-burner /bin/" + +- name: kube-burner version + shell: /bin/kube-burner version \ No newline at end of file diff --git a/playbooks/roles/ocp-cluster-etcd-operator/tasks/main.yaml b/playbooks/roles/ocp-cluster-etcd-operator/tasks/main.yaml new file mode 100644 index 00000000..bd4ae81b --- /dev/null +++ b/playbooks/roles/ocp-cluster-etcd-operator/tasks/main.yaml @@ -0,0 +1,91 @@ +# tasks file for playbooks/roles/ocp-cluster-etcd-operator + +# Cluster health check +- name: Check if cluster operators and nodes are healthy + include_role: + name: check-cluster-health + +# Verify tiers, if cluster is on PowerVS +- name: Verify tiers + shell: oc get sc | grep tier + register: tier_output + failed_when: tier_output.stdout_lines|length == 0 + +# Check hardware speed and change the profile to slower if not already +- name: Verify if Control Plane Hardware Speed profile is "Slower" + shell: oc describe etcd/cluster | grep -i slower + register: speed_check + failed_when: false + +- name: Update Control Plane Hardware Speed profile to "Slower" + shell: | + oc patch etcd cluster --type=merge --patch '{"spec": { "controlPlaneHardwareSpeed": "Slower" }}' + when: speed_check.rc != 0 + +- name: Wait for changes to take effect + wait_for: + timeout: 60 + when: speed_check.rc != 0 + +- name: Perform actions if speed was already "Slower" + block: + - name: Retrieve the etcd pod name + shell: oc get pods -n openshift-etcd --selector=app=etcd --output=jsonpath='{.items[0].metadata.name}' + register: etcd_pod_name + + - name: Extract HEARTBEAT_INTERVAL and ELECTION_TIMEOUT values + shell: | + oc describe pod {{ etcd_pod_name.stdout }} -n openshift-etcd | grep -e HEARTBEAT_INTERVAL -e ELECTION_TIMEOUT | awk '{print $2}' + register: pod_values + failed_when: + - "'500' not in pod_values.stdout" + - "'2500' not in pod_values.stdout" + when: speed_check.rc == 0 + +## Run e2e conformance test suite +- name: Run e2e conformance test suite + include_role: + name: ocp-e2e + +# Run the acme-air workload +- name: Clone acmeair-mainservice-java repo + git: + repo: https://github.com/ocp-power-demos/acmeair-mainservice-java.git + dest: /root/acmeair-mainservice-java/ + +- name: Deployment of acmeair-mainservice-java pods + shell: | + oc project default + sh /root/acmeair-mainservice-java/scripts/deployToOpenshift.sh + kubectl wait --all --namespace=acme-air --for=condition=Ready pods --timeout=300s + +# kube-burner workload test +- name: Download and install kube-burner executable + include_tasks: "{{ role_path }}/tasks/install-kube-burner.yaml" + +# Clone kube-burner git repo to run the test +- name: Clone kube-burner git repo + git: + repo: "{{ kube_burner_repo }}" + dest: "{{ kube_burner_dir }}" + +- name: Run cluster-density workload + include_tasks: "{{ role_path }}/files/cluster-density.yaml" + +- name: Run api-intensive workload + include_tasks: "{{ role_path }}/files/api-intensive.yaml" + +- name: Run crd-scale workload + include_tasks: "{{ role_path }}/files/crd-scale.yaml" + +- name: Run kubelet-density workload + include_tasks: "{{ role_path }}/files/kubelet-density.yaml" + +- name: Run kubelet-density-cni workload + include_tasks: "{{ role_path }}/files/kubelet-density-cni.yaml" + +- name: Run kubelet-density-cni-networkpolicy workload + include_tasks: "{{ role_path }}/files/kubelet-density-cni-networkpolicy.yaml" + +- name: Run kubelet-density-heavy workload + include_tasks: "{{ role_path }}/files/kubelet-density-heavy.yaml"