diff --git a/src/krkn_lib/tests/base_test.py b/src/krkn_lib/tests/base_test.py index 19403848..423c1d8d 100644 --- a/src/krkn_lib/tests/base_test.py +++ b/src/krkn_lib/tests/base_test.py @@ -1,6 +1,7 @@ import cProfile import logging import os +import queue import random import string import sys @@ -32,6 +33,7 @@ class BaseTest(unittest.TestCase): lib_telemetry_ocp: KrknTelemetryOpenshift lib_elastic: KrknElastic pr: cProfile.Profile + pod_delete_queue: queue.Queue @classmethod def setUpClass(cls): @@ -57,6 +59,17 @@ def setUpClass(cls): # cls.pr = cProfile.Profile() # cls.pr.enable() # print("\n<<<---") + + # starting pod_delete_queue + cls.pod_delete_queue = queue.Queue() + worker = threading.Thread( + target=cls.pod_delete_worker, + args=[cls] + ) + + worker.daemon = True + worker.start() + try: requests.get(host, timeout=2, verify=False) except ConnectTimeout: @@ -69,6 +82,20 @@ def setUpClass(cls): ) sys.exit(1) + def pod_delete_worker(self): + while True: + pod_namespace = self.pod_delete_queue.get() + if pod_namespace[0] == "exit" and pod_namespace == "exit": + print("pod killing thread exiting....") + return + try: + self.lib_k8s.delete_pod(pod_namespace[0], pod_namespace[1]) + self.lib_k8s.delete_namespace(pod_namespace[1]) + print(f"[POD DELETE WORKER] deleted pod: {pod_namespace[0]} namespace:{pod_namespace[1]}") + except Exception as e: + print(f"[POD DELETE WORKER] exception raised but continuing: {e}") + continue + @classmethod def tearDownClass(cls) -> None: # PROFILER @@ -79,6 +106,7 @@ def tearDownClass(cls) -> None: # p.print_stats() # print # "\n--->>>" + cls.pod_delete_queue.put(["exit", "exit"]) pass def wait_delete_namespace( diff --git a/src/krkn_lib/tests/test_krkn_kubernetes.py b/src/krkn_lib/tests/test_krkn_kubernetes.py index d2683069..c2ec6482 100644 --- a/src/krkn_lib/tests/test_krkn_kubernetes.py +++ b/src/krkn_lib/tests/test_krkn_kubernetes.py @@ -7,6 +7,7 @@ import time import unittest import uuid +from queue import Queue import yaml from jinja2 import Environment, FileSystemLoader @@ -20,7 +21,8 @@ class KrknKubernetesTests(BaseTest): - def test_zzzexec_command(self): + + def test_exec_command(self): namespace = "test-ec-" + self.get_random_string(10) alpine_name = "alpine-" + self.get_random_string(10) self.deploy_namespace(namespace, []) @@ -81,11 +83,13 @@ def test_zzzexec_command(self): try: self.lib_k8s.exec_cmd_in_pod(["ls", "-al"], alpine_name, namespace) - self.lib_k8s.delete_namespace(namespace) except Exception: - self.lib_k8s.delete_namespace(namespace) self.fail() + self.pod_delete_queue.put(["fedtools", namespace]) + self.pod_delete_queue.put([alpine_name, namespace]) + + def test_pod_shell(self): namespace = "test-ps-" + self.get_random_string(10) alpine_name = "alpine-" + self.get_random_string(10) @@ -118,7 +122,7 @@ def test_pod_shell(self): continue shell = self.lib_k8s.get_pod_shell("fedtools", namespace) self.assertEqual(shell, "bash") - self.lib_k8s.delete_namespace(namespace) + self.pod_delete_queue.put([alpine_name, namespace]) def test_command_on_node(self): try: @@ -850,14 +854,13 @@ def test_download_folder_from_pod_as_archive(self): self.assertTrue(os.stat(file[1]).st_size > 0) def test_exists_path_in_pod(self): - namespace = "test-" + self.get_random_string(10) + namespace="default" pod_name = "alpine-"+self.get_random_string(10) - self.deploy_namespace(namespace, []) # to ensure that the namespace is fully deployed time.sleep(5) self.depoy_alpine(pod_name, namespace) count = 0 - MAX_RETRIES = 20 + MAX_RETRIES = 100 while not self.lib_k8s.is_pod_running(pod_name, namespace): if count > MAX_RETRIES: self.assertFalse(True, "container failed to become ready") @@ -867,13 +870,13 @@ def test_exists_path_in_pod(self): self.assertTrue( self.lib_k8s.path_exists_in_pod( - "fedtools", "fedtools", namespace, "/home" + "alpine", "alpine", namespace, "/home" ) ) self.assertFalse( self.lib_k8s.path_exists_in_pod( - "fedtools", "fedtools", namespace, "/does_not_exist" + "alpine", "alpine", namespace, "/does_not_exist" ) ) self.lib_k8s.delete_namespace(namespace)