Chaos AI updates

utils/chaos_ai/README.md

@@ -3,10 +3,11 @@ Enhancing Chaos Engineering with AI-assisted fault injection for better resilien
 
 ## Generate python package wheel file
 ```
+$ rm -rf build/ constraints.egg-info
 $ python3.9 generate_wheel_package.py sdist bdist_wheel
 $ cp dist/aichaos-0.0.1-py3-none-any.whl docker/
 ```
-This creates a python package file aichaos-0.0.1-py3-none-any.whl in the dist folder. 
+This creates a python package file aichaos-0.0.1-py3-none-any.whl in the dist folder.
 
 ## Build Image
 ```

utils/chaos_ai/docker/config/yml/chaosGen.yml

@@ -14,12 +14,12 @@ Generate chaos on an application deployed on a cluster.
         type: string
         default: robot-shop
         required: true
-        description: Namespace to test
+        description: Comma separated namespaces to test
       - name: podlabels
         in: formData
         type: string
         default: service=cart,service=payment
-        required: true
+        required: false
         description: Pod labels to test
       - name: nodelabels
         in: formData

utils/chaos_ai/docker/requirements.txt

@@ -1,6 +1,8 @@
-numpy
-pandas
-requests
+numpy==2.0.0
+pandas==2.2.2
+requests==2.32.3
 Flask==2.2.5
 Werkzeug==3.0.3
 flasgger==0.9.5
+kubernetes==30.1.0
+urllib3==2.2.3

utils/chaos_ai/docker/swagger_api.py

@@ -12,6 +12,7 @@
 
 # sys.path.append("..")
 from src.aichaos_main import AIChaos
+import src.utils as utils
 
 app = Flask(__name__)
 Swagger(app)
@@ -54,6 +55,8 @@ def startchaos(self, kubeconfigfile, file_id, params):
                 params['iterations'] = config_params['iterations']
                 params['maxfaults'] = config_params['maxfaults']
         # faults = [f + ':' + p for f in params['faults'].split(',') for p in params['podlabels'].split(',')]
+        if params['podlabels'] is None or params['podlabels'] == '':
+            params['podlabels'] = ','.join(utils.get_pods(kubeconfigfile))
         faults = []
         for f in params['faults'].split(','):
             if f in ['pod-delete']:
@@ -119,6 +122,10 @@ def chaos_gen():
         # print('HEADER:', f.headers)
         print('[GenerateChaos] reqs:', request.form.to_dict())
         # print('[GenerateChaos]', f.filename, datetime.now())
+        if utils.is_cluster_accessible(kubeconfigfile):
+            print("Cluster is accessible!")
+        else:
+            return 'Cluster not accessible !!!'
         thread = threading.Thread(target=sw.startchaos, args=(kubeconfigfile, str(i), request.form.to_dict()))
         thread.daemon = True
         print(thread.getName())

utils/chaos_ai/src/kraken_utils.py

@@ -17,6 +17,7 @@ def __init__(self, namespace='robot-shop', chaos_dir='../config/',
         self.engines = []
         self.wait_checks = wait_checks
         self.command = command
+        self.ns_pods = utils.get_namespace_pods(namespace, kubeconfig)
 
     def exp_status(self, engine='engine-cartns3'):
         substring_list = ['Waiting for the specified duration','Waiting for wait_duration', 'Step workload started, waiting for response']
@@ -83,15 +84,16 @@ def inject_faults(self, fault, pod_name):
         self.logger.debug('[KRAKEN][INJECT_FAULT] ' + fault + ':' + pod_name)
         fault, load = utils.get_load(fault)
         engine = 'engine-' + pod_name.replace('=', '-').replace('/','-') + '-' + fault
+        ns = utils.get_ns_from_pod(self.ns_pods, pod_name)
         if fault == 'pod-delete':
-            cmd = self.command+' run  -d -e NAMESPACE='+self.namespace+' -e POD_LABEL='+pod_name+' --name='+engine+' --net=host -v '+self.kubeconfig+':/root/.kube/config:Z quay.io/redhat-chaos/krkn-hub:pod-scenarios >> temp'
+            cmd = self.command+' run  -d -e NAMESPACE='+ns+' -e POD_LABEL='+pod_name+' --name='+engine+' --net=host -v '+self.kubeconfig+':/root/.kube/config:Z quay.io/redhat-chaos/krkn-hub:pod-scenarios >> temp'
         elif fault == 'network-chaos':
             # 'docker run -e NODE_NAME=minikube-m03 -e DURATION=10  --name=knetwork --net=host -v /home/chaos/.kube/kube-config-raw:/root/.kube/config:Z -d quay.io/redhat-chaos/krkn-hub:network-chaos >> temp'        
             cmd = self.command+' run -d -e NODE_NAME='+pod_name+' -e DURATION=120  --name='+engine+' --net=host -v '+self.kubeconfig+':/root/.kube/config:Z -d quay.io/redhat-chaos/krkn-hub:network-chaos >> temp'
         elif fault == 'node-memory-hog':
             cmd = self.command+' run -d -e NODE_NAME='+pod_name+' -e DURATION=120 -e NODES_AFFECTED_PERC=100 --name='+engine+' --net=host -v '+self.kubeconfig+':/root/.kube/config:Z -d quay.io/redhat-chaos/krkn-hub:node-memory-hog >> temp'
         elif fault == 'node-cpu-hog':
-            cmd = self.command+'  run -e NODE_SELECTORS='+pod_name+' -e NODE_CPU_PERCENTAGE=100 -e NAMESPACE='+self.namespace+' -e TOTAL_CHAOS_DURATION=120 -e NODE_CPU_CORE=100 --name='+engine+' --net=host -env-host=true -v '+self.kubeconfig+':/root/.kube/config:Z -d quay.io/redhat-chaos/krkn-hub:node-cpu-hog'
+            cmd = self.command+'  run -e NODE_SELECTORS='+pod_name+' -e NODE_CPU_PERCENTAGE=100 -e NAMESPACE='+ns+' -e TOTAL_CHAOS_DURATION=120 -e NODE_CPU_CORE=100 --name='+engine+' --net=host -env-host=true -v '+self.kubeconfig+':/root/.kube/config:Z -d quay.io/redhat-chaos/krkn-hub:node-cpu-hog'
         else:
             cmd = 'echo'
         self.logger.debug('[KRAKEN][INJECT_FAULT] ' + cmd)

utils/chaos_ai/src/utils.py

@@ -1,5 +1,9 @@
 import re
 
+import urllib3
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+
 
 def get_load(fault):
     params = re.findall(r'\(.*?\)', fault)
@@ -8,3 +12,90 @@ def get_load(fault):
         load = params[0].strip('()')
         fault = fault.strip(params[0])
     return fault, load
+
+
+def is_cluster_accessible(kubeconfig_path: str) -> bool:
+    try:
+        config.load_kube_config(config_file=kubeconfig_path)
+        v1 = client.CoreV1Api()
+
+        # Try to list nodes in the cluster
+        nodes = v1.list_node()
+        print("#Nodes in Cluster: ", len(nodes.items))
+
+        return True
+    except (FileNotFoundError, ApiException, Exception) as e:
+        print(f"Cluster is not accessible: {e}")
+        return False
+
+def get_namespace_pods(namespaces: str, kubeconfig_path: str):
+    ns_list = namespaces.split(",")
+    ns_pods = {}
+    for ns in ns_list:
+        pods = get_pod_labels(ns, kubeconfig_path)
+        if len(pods) > 0:
+            ns_pods[ns] = pods
+    return ns_pods
+
+def get_ns_from_pod(ns_pods, podlabel):
+    for ns in ns_pods:
+        if podlabel in ns:
+            return ns
+    return ''
+
+# get all pod labels from a namespace
+def get_pod_labels(namespace: str, kubeconfig_path: str = None):
+    pods = []
+    try:
+        if kubeconfig_path:
+            config.load_kube_config(config_file=kubeconfig_path)
+        else:
+            config.load_kube_config()  # Load default kubeconfig file
+
+        v1 = client.CoreV1Api()
+
+        pods = v1.list_namespaced_pod(namespace)
+
+        print(f"Pod labels in namespace '{namespace}':")
+        for pod in pods.items:
+            print(f"Pod Name: {pod.metadata.name}, Labels: {pod.metadata.labels}")
+
+    except FileNotFoundError:
+        print(f"Kubeconfig file not found at {kubeconfig_path}")
+    except ApiException as e:
+        print(f"API exception occurred: {e}")
+    except urllib3.exceptions.MaxRetryError as e:
+        print(f"Max retries exceeded: {e}")
+    except urllib3.exceptions.NewConnectionError as e:
+        print(f"New connection error: {e}")
+    except urllib3.exceptions.NameResolutionError as e:
+        print(f"Name resolution error: {e}")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+    return pods
+
+
+def get_pods(kubeconfig_path: str = None):
+    pods = []
+    try:
+        # Load the kubeconfig file
+        if kubeconfig_path:
+            config.load_kube_config(config_file=kubeconfig_path)
+        else:
+            config.load_kube_config()  # Load default kubeconfig file
+        v1 = client.CoreV1Api()
+        pods = v1.list_pod_for_all_namespaces()
+
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+    return pods
+
+# Example usage
+namespace = 'default'
+kubeconfig_path = '/path/to/your/kubeconfig'  # Provide the path to your kubeconfig file, or set to None to use the default
+get_pod_labels(namespace, kubeconfig_path)
+
+if __name__ == '__main__':
+    # print(is_cluster_accessible("~/Downloads/chaos/kraken/kubeconfig"))
+    print(is_cluster_accessible("~/Downloads/kube-config-raw"))
+