wip

config/config.yaml

@@ -87,6 +87,9 @@ telemetry:
      - "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+"         # Sep 9 11:20:36.123425532
      - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+"          # kinit 2023/09/15 11:20:36 log
      - "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+"      # 2023-09-15T11:20:36.123425532Z log
+
+
+    logs_filter_patterns: [ "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+","kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+","(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z).+" ]
     oc_cli_path: /usr/bin/oc                                # optional, if not specified will be search in $PATH
     events_backup: True                                     # enables/disables cluster events collection
 

kraken/prometheus/client.py

@@ -47,3 +47,6 @@ def instance(distribution, prometheus_url, prometheus_bearer_token):
             "|| oc sa new-token -n openshift-monitoring prometheus-k8s"
         )
     return prometheus_url, prometheus_bearer_token
+
+
+def scrape_metrics()

requirements.txt

@@ -19,14 +19,13 @@ ibm_cloud_sdk_core
 ibm_vpc
 itsdangerous==2.0.1
 jinja2==3.0.3
-krkn-lib>=1.4.2
+krkn-lib@git+https://github.com/redhat-chaos/krkn-lib.git@prometheus
 kubernetes
 lxml >= 4.3.0
 oauth2client>=4.1.3
 openshift-client
 paramiko
 podman-compose
-prometheus_api_client
 pyVmomi >= 6.7
 pyfiglet
 pytest

run_kraken.py

@@ -8,6 +8,7 @@
 import pyfiglet
 import uuid
 import time
+from krkn_lib.prometheus.krkn_prometheus import KrknPrometheus
 import kraken.time_actions.common_time_functions as time_actions
 import kraken.performance_dashboards.setup as performance_dashboards
 import kraken.pod_scenarios.setup as pod_scenarios
@@ -22,7 +23,6 @@
 import kraken.network_chaos.actions as network_chaos
 import kraken.arcaflow_plugin as arcaflow_plugin
 import server as server
-import kraken.prometheus.client as promcli
 from kraken import plugins
 
 from krkn_lib.k8s import KrknKubernetes
@@ -150,7 +150,7 @@ def main(cfg):
         # KrknTelemetry init
         telemetry_k8s = KrknTelemetryKubernetes(safe_logger, kubecli)
         telemetry_ocp = KrknTelemetryOpenshift(safe_logger, ocpcli)
-
+        prometheus = KrknPrometheus(prometheus_url, prometheus_bearer_token)
 
         # find node kraken might be running on
         kubecli.find_kraken_node()
@@ -351,9 +351,10 @@ def main(cfg):
                         # Check for critical alerts when enabled
                         if check_critical_alerts:
                             logging.info("Checking for critical alerts firing post choas")
-                            promcli.initialize_prom_client(distribution, prometheus_url, prometheus_bearer_token)
+
+                            ##PROM
                             query = r"""ALERTS{severity="critical"}"""
-                            critical_alerts = promcli.process_prom_query(query)
+                            critical_alerts = prometheus.process_prom_query(query)
                             critical_alerts_count = len(critical_alerts)
                             if critical_alerts_count > 0:
                                 logging.error("Critical alerts are firing: %s", critical_alerts)
@@ -406,6 +407,7 @@ def main(cfg):
 
         # Capture metrics for the run
         if capture_metrics:
+            if config
             logging.info("Capturing metrics")
             kube_burner.setup(kube_burner_url)
             kube_burner.scrape_metrics(