hog deployment + tests

src/krkn_lib/k8s/krkn_kubernetes.py

@@ -39,6 +39,7 @@
     Volume,
     VolumeMount,
 )
+from krkn_lib.models.krkn import HogConfig, HogType
 from krkn_lib.models.telemetry import ClusterEvent, NodeInfo, Taint
 from krkn_lib.utils import filter_dictionary, get_random_string
 from krkn_lib.utils.safe_logger import SafeLogger
@@ -2014,41 +2015,21 @@ def get_api_resources_by_group(self, group, version):
 
         return None
 
-    def get_node_cpu_count(
-        self, node_name: str, api_version: str = "v1"
-    ) -> int:
+    def get_node_cpu_count(self, node_name: str) -> int:
         """
         Returns the number of cpus of a specified node
 
         :param node_name: the name of the node
-        :param api_version: the kubernetes api version
         :return: the number of cpus or 0 if any exception is raised
         """
         api_client = self.api_client
 
         if api_client:
             try:
-                path_params: Dict[str, str] = {}
-                query_params: List[str] = []
-                header_params: Dict[str, str] = {}
-                auth_settings = ["BearerToken"]
-                header_params["Accept"] = api_client.select_header_accept(
-                    ["application/json"]
-                )
-
-                path = f"/api/{api_version}/nodes/{node_name}"
-                (data) = api_client.call_api(
-                    path,
-                    "GET",
-                    path_params,
-                    query_params,
-                    header_params,
-                    response_type="str",
-                    auth_settings=auth_settings,
-                )
-
-                json_obj = ast.literal_eval(data[0])
-                return int(json_obj["status"]["capacity"]["cpu"])
+                v1 = self.cli
+                node = v1.read_node(node_name)
+                cpu_capacity = node.status.capacity.get("cpu")
+                return int(cpu_capacity)
             except Exception:
                 return 0
 
@@ -3290,3 +3271,58 @@ def deploy_syn_flood(
         )
 
         self.create_pod(namespace=namespace, body=pod_body)
+
+    def deploy_hog(
+        self,
+        pod_name: str,
+        namespace: str,
+        image: str,
+        hog_config: HogConfig,
+    ):
+        """
+        Deploys a Pod to run the Syn Flood scenario
+
+        :param pod_name: The name of the pod that will be deployed
+        :param namespace: The namespace where the pod will be deployed
+        :param image: the syn flood scenario container image
+        :param hog_config: Hog Configuration
+        """
+        has_selector = len(hog_config.node_selector.keys()) > 0
+        if has_selector:
+            node_selector_key = list(hog_config.node_selector.keys())[0]
+            node_selector_value = hog_config.node_selector[
+                list(hog_config.node_selector.keys())[0]
+            ]
+        else:
+            node_selector_key = ""
+            node_selector_value = ""
+
+        file_loader = PackageLoader("krkn_lib.k8s", "templates")
+        env = Environment(loader=file_loader, autoescape=True)
+        io_volume = {"volumes": [hog_config.io_target_pod_volume]}
+        yaml_data = yaml.dump(io_volume, default_flow_style=False, indent=2)
+        pod_template = env.get_template("hog_pod.j2")
+        pod_body = yaml.safe_load(
+            pod_template.render(
+                name=pod_name,
+                namespace=namespace,
+                hog_type=hog_config.type.value,
+                hog_type_io=HogType.IO.value,
+                has_selector=has_selector,
+                node_selector_key=node_selector_key,
+                node_selector_value=node_selector_value,
+                image=image,
+                duration=hog_config.duration,
+                cpu_load_percentage=hog_config.cpu_load_percentage,
+                cpu_method=hog_config.cpu_method,
+                io_block_size=hog_config.io_block_size,
+                io_write_bytes=hog_config.io_write_bytes,
+                io_target_pod_folder=hog_config.io_target_pod_folder,
+                io_volume_mount=yaml_data,
+                memory_vm_bytes=hog_config.memory_vm_bytes,
+                workers=hog_config.workers,
+                target_pod_folder=hog_config.io_target_pod_folder,
+            )
+        )
+
+        self.create_pod(namespace=namespace, body=pod_body)

src/krkn_lib/k8s/templates/hog_pod.j2

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ name }}
+  namespace: {{ namespace }}
+spec:
+    {% if has_selector == true %}
+    nodeSelector:
+        {{ node_selector_key }}: {{ node_selector_value }}
+    {% endif %}
+    {% if hog_type == hog_type_io %}
+    {{ io_volume_mount | safe | indent(4)}}
+    {% endif %}
+    restartPolicy: Never
+    imagePullPolicy: IfNotPresent
+    containers:
+      - name: krkn-hog
+        image: {{ image }}
+        securityContext:
+            privileged: true
+        {% if hog_type == hog_type_io %}
+        volumeMounts:
+            - name: node-volume
+              mountPath: {{ target_pod_folder }}
+        {% endif %}
+        env:
+            - name: HOG_TYPE
+              value: '{{ hog_type }}'
+            - name: LOAD_PERCENTAGE
+              value: '{{ cpu_load_percentage }}'
+            - name: CPU_METHOD
+              value: '{{ cpu_method }}'
+            - name: HDD_WRITE_SIZE
+              value: '{{ io_block_size }}'
+            - name: HDD_BYTES
+              value: '{{ io_write_bytes }}'
+            - name: STRESS_PATH
+              value: '{{ target_pod_folder }}'
+            - name: VM_BYTES
+              value: '{{ memory_vm_bytes }}'
+            - name: WORKERS
+              value: '{{ workers }}'
+            - name: DURATION
+              value: '{{ duration }}'

src/krkn_lib/models/krkn/models.py

@@ -2,6 +2,7 @@
 
 import json
 from dataclasses import dataclass
+from enum import Enum
 
 from krkn_lib.models.telemetry import ChaosRunTelemetry
 
@@ -95,3 +96,49 @@ def __init__(self):
 
     def to_json(self) -> str:
         return json.dumps(self, default=lambda o: o.__dict__, indent=4)
+
+
+class HogType(Enum):
+    CPU = "cpu"
+    MEMORY = "memory"
+    IO = "io"
+
+
+class HogConfig:
+    type: HogType
+    image: str
+    # cpu hog
+    cpu_load_percentage: int
+    cpu_method: str
+
+    # io hog
+    io_block_size: str
+    io_write_bytes: str
+    io_target_pod_folder: str
+    io_target_pod_volume: dict[str, any]
+
+    # memory hog
+    memory_vm_bytes: str
+
+    workers: int
+    duration: int
+    namespace: str
+    node_selector: dict[str, str]
+
+    def __init__(self):
+        self.type = HogType.CPU
+        self.image = "quay.io/krkn-chaos/krkn-hog"
+        self.cpu_load_percentage = 80
+        self.cpu_method = "all"
+        self.io_block_size = "1m"
+        self.io_write_bytes = "10m"
+        self.io_target_pod_folder = "/hog-data"
+        self.io_target_pod_volume = {
+            "hostPath": {"path": "/tmp"},
+            "name": "node-volume",
+        }
+        self.memory_vm_bytes = "10%"
+        self.workers = 1
+        self.duration = 30
+        self.namespace = "default"
+        self.node_selector = {}

src/krkn_lib/tests/test_krkn_kubernetes_get.py

@@ -240,10 +240,7 @@ def test_get_node_cpu_count(self):
         node_cpus = self.lib_k8s.get_node_cpu_count(nodes[0])
         self.assertGreater(node_cpus, 0)
 
-        node_cpus = self.lib_k8s.get_node_cpu_count("does_not_exist", "v1")
-        self.assertEqual(node_cpus, 0)
-
-        node_cpus = self.lib_k8s.get_node_cpu_count("does_not_exist", "v3")
+        node_cpus = self.lib_k8s.get_node_cpu_count("does_not_exist")
         self.assertEqual(node_cpus, 0)
 
 

src/krkn_lib/tests/test_krkn_kubernetes_misc.py

@@ -1,3 +1,4 @@
+import ast
 import datetime
 import logging
 import random
@@ -6,6 +7,7 @@
 
 import yaml
 
+from krkn_lib.models.krkn import HogConfig, HogType
 from krkn_lib.tests import BaseTest
 from tzlocal import get_localzone
 from kubernetes.client import ApiException
@@ -216,6 +218,117 @@ def test_deploy_syn_flood(self):
         )
         self.lib_k8s.delete_namespace(namespace)
 
+    def get_node_resources_info(self, node_name: str):
+        path_params: dict[str, str] = {}
+        query_params: list[str] = []
+        header_params: dict[str, str] = {}
+        auth_settings = ["BearerToken"]
+        header_params["Accept"] = self.lib_k8s.api_client.select_header_accept(
+            ["application/json"]
+        )
+        path = f"/api/v1/nodes/{node_name}/proxy/stats/summary"
+        (data) = self.lib_k8s.api_client.call_api(
+            path,
+            "GET",
+            path_params,
+            query_params,
+            header_params,
+            response_type="str",
+            auth_settings=auth_settings,
+        )
+
+        json_obj = ast.literal_eval(data[0])
+        return (
+            json_obj["node"]["cpu"]["usageNanoCores"],
+            json_obj["node"]["memory"]["availableBytes"],
+            json_obj["node"]["fs"]["availableBytes"],
+        )
+
+    def test_deploy_hog(self):
+        """ """
+        increase_baseline = 70
+        nodes = self.lib_k8s.list_nodes()
+        node_cpus = self.lib_k8s.get_node_cpu_count(nodes[0])
+        node_resources_start = self.get_node_resources_info(nodes[0])
+        pod_name = f"test-hog-pod-{self.get_random_string(5)}"
+        namespace = f"test-hog-pod-{self.get_random_string(5)}"
+        self.deploy_namespace(namespace, labels=[])
+        # tests CPU Hog detecting a memory increase of
+        # 80% minimum
+
+        config = HogConfig()
+        config.duration = 30
+        config.io_target_pod_volume = {
+            "hostPath": {"path": "/"},
+            "name": "node-volume",
+        }
+        config.type = HogType.CPU
+        config.cpu_load_percentage = 90
+        config.workers = node_cpus
+        config.node_selector["kubernetes.io/hostname"] = nodes[0]
+        self.lib_k8s.deploy_hog(
+            pod_name, namespace, "quay.io/krkn-chaos/krkn-hog", config
+        )
+
+        while not self.lib_k8s.is_pod_running(pod_name, namespace):
+            continue
+
+        time.sleep(19)
+        node_resources_after = self.get_node_resources_info(nodes[0])
+        cpu_delta = node_resources_after[0] / node_resources_start[0] * 100
+        print(f"DETECTED CPU PERCENTAGE INCREASE: {cpu_delta/node_cpus}%")
+        self.assertGreaterEqual(cpu_delta, increase_baseline * node_cpus)
+
+        # tests memory Hog detecting a memory increase of
+        # 80% minimum
+
+        config.type = HogType.MEMORY
+        config.memory_vm_bytes = "90%"
+        config.workers = 4
+        pod_name = f"test-hog-pod-{self.get_random_string(5)}"
+        self.lib_k8s.deploy_hog(
+            pod_name, namespace, "quay.io/krkn-chaos/krkn-hog", config
+        )
+        while not self.lib_k8s.is_pod_running(pod_name, namespace):
+            continue
+        # grabbing the peak during the 20s chaos run
+        time.sleep(19)
+        node_resources_after = self.get_node_resources_info(nodes[0])
+        memory_delta = node_resources_after[1] / node_resources_start[1] * 100
+        print(f"DETECTED MEMORY PERCENTAGE INCREASE: {memory_delta}%")
+        self.assertGreaterEqual(memory_delta, increase_baseline)
+
+        # tests IO Hog detecting a disk increase of
+        # 400MB minimum and checks that the space is
+        # deallocated after the test
+
+        config.type = HogType.IO
+        config.io_write_bytes = "512m"
+        config.workers = 4
+        pod_name = f"test-hog-pod-{self.get_random_string(5)}"
+        self.lib_k8s.deploy_hog(
+            pod_name, namespace, "quay.io/krkn-chaos/krkn-hog", config
+        )
+        while not self.lib_k8s.is_pod_running(pod_name, namespace):
+            continue
+        time.sleep(29)
+        node_resources_after = self.get_node_resources_info(nodes[0])
+        disk_delta = node_resources_start[2] - node_resources_after[2]
+        print(f"DISK SPACE ALLOCATED (MB): {disk_delta/1024/1024}")
+
+        self.assertGreaterEqual(disk_delta / 1024 / 1024, 400)
+
+        # test that after the test the disk space is deallocated
+
+        time.sleep(10)
+        node_resources_end = self.get_node_resources_info(nodes[0])
+        disk_delta = node_resources_start[2] - node_resources_end[2]
+        print(
+            f"DISK SPACE DEALLOCATED AFTER CHAOS (MB): {disk_delta/1024/1024}"
+        )
+        self.assertLessEqual(int(disk_delta / 1024 / 1024), 10)
+        self.lib_k8s.delete_namespace(namespace)
+
     def test_select_services_by_label(self):
         namespace = "test-" + self.get_random_string(10)
         service_name_1 = "krkn-syn-flood-" + self.get_random_string(10)