split ingresses test marker (#4870)

* split ingresses test marker

.github/data/matrix-regression.json

@@ -2,9 +2,11 @@
   "k8s": [],
   "images": [
     {
+      "label": "regression",
       "image": "debian"
     },
     {
+      "label": "regression",
       "image": "debian-plus"
     }
   ]

.github/data/matrix-smoke.json

@@ -1,90 +1,122 @@
 {
   "images": [
     {
+      "label": "ingresses 1/2",
       "image": "debian",
-      "marker": "ingresses"
+      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'"
     },
     {
+      "label": "ingresses 2/2",
+      "image": "debian",
+      "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'"
+    },
+    {
+      "label": "VSR",
       "image": "alpine",
       "marker": "vsr"
     },
     {
+      "label": "policies 1/2",
       "image": "alpine",
       "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'"
     },
     {
+      "label": "policies 2/2",
       "image": "alpine",
       "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'"
     },
     {
+      "label": "VS 1/3",
       "image": "debian",
       "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'"
     },
     {
+      "label": "VS 2/3",
       "image": "debian",
       "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'"
     },
     {
+      "label": "VS 3/3",
       "image": "debian",
       "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'"
     },
     {
+      "label": "TS",
       "image": "ubi",
       "marker": "ts"
     },
     {
+      "label": "VS 1/3",
       "image": "debian-plus",
       "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'"
     },
     {
+      "label": "VS 2/3",
       "image": "debian-plus",
       "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'"
     },
     {
+      "label": "VS 3/3",
       "image": "debian-plus",
       "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'"
     },
     {
+      "label": "TS",
       "image": "debian-plus",
       "marker": "ts"
     },
     {
+      "label": "ingresses 1/2",
+      "image": "alpine-plus",
+      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'"
+    },
+    {
+      "label": "ingresses 2/2",
       "image": "alpine-plus",
-      "marker": "ingresses"
+      "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'"
     },
     {
+      "label": "VSR",
       "image": "alpine-plus",
       "marker": "vsr"
     },
     {
+      "label": "policies 1/2",
       "image": "ubi-plus",
       "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls'"
     },
     {
+      "label": "policies 2/2",
       "image": "ubi-plus",
       "marker": "'policies_ac or policies_jwt or policies_mtls'"
     },
     {
+      "label": "AP_WAF 1/4",
       "image": "debian-plus-nap",
       "marker": "appprotect_waf_policies_allow"
     },
     {
+      "label": "AP_WAF 2/4",
       "image": "debian-plus-nap",
       "marker": "'appprotect_waf_policies and not appprotect_waf_policies_allow'"
     },
     {
+      "label": "AP_WAF 3/4",
       "image": "debian-plus-nap",
       "marker": "appprotect_waf_policies_grpc"
     },
     {
+      "label": "AP_WAF 4/4",
       "image": "debian-plus-nap",
       "marker": "'appprotect_watch or appprotect_batch or appprotect_integration'"
     },
     {
+      "label": "AP_DOS",
       "image": "debian-plus-nap",
       "marker": "'dos and not dos_learning'"
     },
     {
+      "label": "AP_DOS_Learning",
       "image": "debian-plus-nap",
       "marker": "dos_learning"
     }

.github/workflows/ci.yml

@@ -339,7 +339,7 @@ jobs:
           load: true
 
   smoke-tests:
-    name: Smoke Tests
+    name: ${{ matrix.images.label }} ${{ matrix.images.image }} smoke tests
     runs-on: ubuntu-22.04
     needs: [checks, setup-matrix]
     strategy:

tests/suite/test_annotations.py

@@ -210,6 +210,7 @@ def fin():
 
 
 @pytest.mark.ingresses
+@pytest.mark.annotations
 @pytest.mark.parametrize("annotations_setup", ["standard", "mergeable"], indirect=True)
 class TestAnnotations:
     def test_nginx_config_defaults(self, kube_apis, annotations_setup, ingress_controller_prerequisites, cli_arguments):
@@ -484,6 +485,7 @@ def test_validation(self, kube_apis, annotations_setup, ingress_controller_prere
 
 
 @pytest.mark.ingresses
+@pytest.mark.annotations
 @pytest.mark.parametrize("annotations_setup", ["mergeable"], indirect=True)
 class TestMergeableFlows:
     @pytest.mark.parametrize(
@@ -527,6 +529,7 @@ def test_minion_overrides_master(
 
 
 @pytest.mark.ingresses
+@pytest.mark.annotations
 class TestGrpcFlows:
     @pytest.mark.parametrize(
         "annotations, expected_strings, unexpected_strings",

tests/suite/test_auth_basic_auth_mergeable.py

@@ -158,6 +158,7 @@ def get_credentials_from_file(creds_type) -> str:
 
 
 @pytest.mark.ingresses
+@pytest.mark.basic_auth
 class TestAuthBasicAuthMergeableMinions:
     def test_auth_basic_auth_response_codes(self, kube_apis, auth_basic_auth_setup, test_namespace):
         print("Step 1: execute check after secrets creation")

tests/suite/test_auth_basic_secrets.py

@@ -107,6 +107,7 @@ def fin():
 
 
 @pytest.mark.ingresses
+@pytest.mark.basic_auth
 class TestAuthBasicSecrets:
     def test_response_code_200_and_server_name(self, auth_basic_secrets_setup, auth_basic_secret):
         req_url = f"http://{auth_basic_secrets_setup.public_endpoint.public_ip}:{auth_basic_secrets_setup.public_endpoint.port}/backend2"

tests/suite/test_hsts.py

@@ -78,6 +78,7 @@ def fin():
 
 
 @pytest.mark.ingresses
+@pytest.mark.hsts
 @pytest.mark.parametrize("hsts_setup", ["standard-tls", "mergeable-tls"], indirect=True)
 class TestTLSHSTSFlows:
     def test_headers(self, kube_apis, hsts_setup, ingress_controller_prerequisites):
@@ -115,6 +116,7 @@ def test_headers(self, kube_apis, hsts_setup, ingress_controller_prerequisites):
 
 
 @pytest.mark.ingresses
+@pytest.mark.hsts
 @pytest.mark.parametrize("hsts_setup", ["tls-no-secret"], indirect=True)
 class TestBrokenTLSHSTSFlows:
     def test_headers_without_secret(self, kube_apis, hsts_setup, ingress_controller_prerequisites):
@@ -129,6 +131,7 @@ def test_headers_without_secret(self, kube_apis, hsts_setup, ingress_controller_
 
 
 @pytest.mark.ingresses
+@pytest.mark.hsts
 @pytest.mark.parametrize("hsts_setup", ["standard", "mergeable"], indirect=True)
 class TestNoTLSHSTS:
     def test_headers(self, kube_apis, hsts_setup, ingress_controller_prerequisites):

tests/suite/test_watch_namespace.py

@@ -75,6 +75,7 @@ def fin():
 
 
 @pytest.mark.ingresses
+@pytest.mark.watch_namespace
 @pytest.mark.parametrize(
     "ingress_controller, expected_responses",
     [
@@ -95,6 +96,7 @@ def test_response_codes(self, ingress_controller, backend_setup, expected_respon
 
 
 @pytest.mark.ingresses
+@pytest.mark.watch_namespace
 @pytest.mark.parametrize(
     "ingress_controller, expected_responses",
     [

tests/suite/test_watch_namespace_label.py

@@ -121,6 +121,7 @@ def create_and_setup_namespace(kube_apis, ingress_controller_endpoint, ns_name,
 
 
 @pytest.mark.ingresses
+@pytest.mark.watch_namespace
 @pytest.mark.parametrize(
     "ingress_controller, expected_responses",
     [

tests/suite/test_wildcard_tls_secret.py

@@ -115,6 +115,7 @@ def fin():
 
 @pytest.mark.ingresses
 @pytest.mark.smoke
+@pytest.mark.wildcard_tls
 class TestTLSWildcardSecrets:
     @pytest.mark.parametrize("path", paths)
     def test_response_code_200(self, wildcard_tls_secret_ingress_controller, wildcard_tls_secret_setup, path):