Merge pull request #552 from armosec/ff

jira_integration - added retry mechnism in case of hitting rate limit

tests_scripts/helm/jira_integration.py

@@ -1,3 +1,4 @@
+import time
 from .base_helm import BaseHelm
 from ..kubescape.base_kubescape import BaseKubescape
 from systest_utils import statics, Logger, TestUtil
@@ -147,6 +148,22 @@ def setup_cluster_and_run_posture_scan(self):
         self.namespace = namespace
         self.cluster = cluster
 
+    def create_jira_issue(self, issue, retries=3, sleep=45):
+        for i in range(retries):
+            Logger.logger.info(f"Create Jira issue attempt {i+1}")
+            try:
+                ticket = self.backend.create_jira_issue(issue)
+                assert ticket, "Jira ticket is empty"
+                return ticket
+            except (Exception, AssertionError) as e:
+                # we can get RetryAfter error, so we will retry
+                if "RetryAfter".lower() in str(e).lower():
+                    Logger.logger.info(f"Jira issue creation failed with RetryAfter, retrying in {sleep} seconds")
+                    time.sleep(sleep)
+                else:
+                    raise e
+
+
     def create_jira_issue_for_posture(self):
         resource = self.get_posture_resource()
         controlId = resource['failedControls'][0]
@@ -161,8 +178,7 @@ def create_jira_issue_for_posture(self):
         issue['owner'] = {"resourceHash": resourceHash}
         issue['subjects'] = [{"controlId": controlId}]
         issue['fields']['summary'] = f"Jira System Test control Issue cluster:{self.cluster} namespace:{self.namespace} resource:{resourceHash}"
-        ticket = self.backend.create_jira_issue(issue)
-        assert ticket, "Jira ticket is empty"
+        ticket = self.create_jira_issue(issue)
         self.postureTicket = ticket
         assert ticket['owner']['resourceHash'] == resourceHash, "Resource hash is not matching"
         assert ticket['subjects'][0]['controlID'] == controlId, "Control id is not matching"
@@ -206,7 +222,7 @@ def create_jira_issue_for_security_risks(self):
         issue['owner'] = {"resourceHash": resourceHash}
         issue['subjects'] = [{"securityRiskID": security_risk_id}]
         issue['fields']['summary'] = f"Jira System Test security risks Issue cluster:{self.cluster} namespace:{self.namespace} resource:{resourceHash}"
-        ticket = self.backend.create_jira_issue(issue)
+        ticket = self.create_jira_issue(issue)
         assert ticket, "Jira ticket is empty"
         self.securityTicket = ticket
         assert ticket['owner']['resourceHash'] == resourceHash, "Resource hash is not matching"
@@ -296,14 +312,14 @@ def create_vuln_tickets(self):
         issue['issueTypeId'] = self.issueType['id']        
         issue['subjects'] = [{"cveName": self.vuln['name'],"severity": self.vuln['severity'] , "component": self.vuln['componentInfo']['name'], "componentVersion": self.vuln['componentInfo']['version']}]
         issue['fields']['summary'] = f"Jira System Test global Issue CVE:{self.vuln['name']}"
-        globalCVEicket = self.backend.create_jira_issue(issue)
+        globalCVEicket = self.create_jira_issue(issue)
         assert globalCVEicket, "Jira ticket is empty"
 
         Logger.logger.info('create  ticket for workload CVE')
         issue["collaborationGUID"] = self.backend.get_jira_collaboration_guid_by_site_name(self.site_name)
         issue['owner'] = {"cluster": self.vulnWL['cluster'], "namespace": self.vulnWL['namespace'], "kind": self.vulnWL['kind'], "name": self.vulnWL['name']}
         issue['fields']['summary'] = f"Jira System Test CVE Issue for workload cluster:{self.cluster} namespace:{self.namespace} image:{self.vulnImage['repository']}"
-        workloadCVEicket = self.backend.create_jira_issue(issue)
+        workloadCVEicket = self.create_jira_issue(issue)
         assert workloadCVEicket, "Jira ticket is empty"
         assert workloadCVEicket, "Jira ticket is empty"
 
@@ -315,14 +331,14 @@ def create_vuln_tickets(self):
         issue['issueType'] = "image"
         issue['subjects'] = [{"imageRepository": self.vulnImage['repository']}]
         issue['fields']['summary'] = f"Jira System Test global Issue image:{self.vulnImage['repository']}"
-        globalImageTicket = self.backend.create_jira_issue(issue)
+        globalImageTicket = self.create_jira_issue(issue)
         assert globalImageTicket, "Jira ticket is empty"
 
         Logger.logger.info('create ticket for image in workload')
         issue["collaborationGUID"] = self.backend.get_jira_collaboration_guid_by_site_name(self.site_name)
         issue['owner'] = {"cluster": self.vulnWL['cluster'], "namespace": self.vulnWL['namespace'], "kind": self.vulnWL['kind'], "name": self.vulnWL['name']}
         issue['fields']['summary'] = f"Jira System Test image Issue for workload cluster:{self.cluster} namespace:{self.namespace} image:{self.vulnImage['repository']}"
-        workloadImageTicket = self.backend.create_jira_issue(issue)
+        workloadImageTicket = self.create_jira_issue(issue)
         assert workloadImageTicket, "Jira ticket is empty"
 
 

tests_scripts/kubernetes/base_k8s.py

@@ -769,7 +769,11 @@ def verify_running_pods(self, namespace: str, replicas: int = None, name: str =
                 Logger.logger.info(f"all pods are running after {delta_t} seconds")
                 result = subprocess.run("kubectl get pods -A", timeout=300, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 result = " ".join(result.stdout.splitlines())
-                Logger.logger.info(f"cluster state {result}")
+                Logger.logger.info(
+                    "cluster state\n"
+                    f"{result}"
+                )
+
                 return
             delta_t = (datetime.now() - start).total_seconds()
             time.sleep(10)
@@ -781,7 +785,10 @@ def verify_running_pods(self, namespace: str, replicas: int = None, name: str =
 
         result = subprocess.run("kubectl get pods -A", timeout=300, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         result = " ".join(result.stdout.splitlines())
-        Logger.logger.info(f"cluster state {result}")
+        Logger.logger.info(
+                    "cluster state\n"
+                    f"{result}"
+                )
         raise Exception("wrong number of pods are running after {} seconds. expected: {}, running: {}, pods:{}"
                         .format(delta_t, replicas, len(running_pods), running_pods))  # , len(total_pods)))