Jira replacing bz for customer scenarios check (#15878)

* Jira replacing bz for customer scenarios check

* Jira module restructured and dynamicitic  improvements

* Nitpicking and docstrings

* Customer Scenarios script and makefile has dual support

(cherry picked from commit 3d4eac2)

.github/workflows/weekly.yml

@@ -48,7 +48,7 @@ jobs:
           echo "assignees: pondrejk " >> .env.md
           echo "labels: Documentation" >> .env.md
           echo "---" >> .env.md
-          echo CS_TAGS="$(make customer-scenario-check)" >> .env.md
+          echo CS_TAGS="$(make customer-scenario-check-jira)" >> .env.md
           if grep 'The following tests need customerscenario tags' .env.md; then
             echo "::set-output name=result::0"
           fi

Makefile

@@ -169,8 +169,11 @@ clean-cache:
 
 clean-all: docs-clean logs-clean pyc-clean clean-cache clean-shared
 
-customer-scenario-check:
-	@scripts/customer_scenarios.py
+customer-scenario-check-bz:
+	@scripts/customer_scenarios.py --bz
+
+customer-scenario-check-jira:
+	@scripts/customer_scenarios.py --jira
 
 vault-login:
 	@scripts/vault_login.py --login

conf/dynaconf_hooks.py

@@ -53,7 +53,7 @@ def config_migrations(settings, data):
     :type data: dict
     """
     logger.info('Running config migration hooks')
-    sys.path.append(str(Path(__file__).parent))
+    sys.path.append(str(Path(__file__).parent.parent))
     from conf import migrations
 
     migration_functions = [

robottelo/utils/issue_handlers/jira.py

@@ -20,6 +20,31 @@
 # The .version group being a `d.d` string that can be casted to Version()
 VERSION_RE = re.compile(r'(?:sat-)*?(?P<version>\d\.\d)\.\w*')
 
+common_jira_fields = ['key', 'summary', 'status', 'labels', 'resolution', 'fixVersions']
+
+mapped_response_fields = {
+    'key': "{obj_name}['key']",
+    'summary': "{obj_name}['fields']['summary']",
+    'status': "{obj_name}['fields']['status']['name']",
+    'labels': "{obj_name}['fields']['labels']",
+    'resolution': "{obj_name}['fields']['resolution']['name'] if {obj_name}['fields']['resolution'] else ''",
+    'fixVersions': "[ver['name'] for ver in {obj_name}['fields']['fixVersions']] if {obj_name}['fields']['fixVersions'] else []",
+    # Custom Field - SFDC Cases Counter
+    'customfield_12313440': "{obj_name}['fields']['customfield_12313440']",
+}
+
+
+def sanitized_issue_data(issue, out_fields):
+    """fetches the value for all the given fields from a given jira issue
+
+    Arguments:
+        issue {dict} -- The json data for a jira issue
+        out_fields {list} -- The list of fields for which data to be retrieved from jira issue
+    """
+    return {
+        field: eval(mapped_response_fields[field].format(obj_name='issue')) for field in out_fields
+    }
+
 
 def is_open_jira(issue_id, data=None):
     """Check if specific Jira is open consulting a cached `data` dict or
@@ -131,8 +156,7 @@ def collect_data_jira(collected_data, cached_data):  # pragma: no cover
     """
     jira_data = (
         get_data_jira(
-            [item for item in collected_data if item.startswith('SAT-')],
-            cached_data=cached_data,
+            [item for item in collected_data if item.startswith('SAT-')], cached_data=cached_data
         )
         or []
     )
@@ -169,16 +193,41 @@ def collect_dupes(jira, collected_data, cached_data=None):  # pragma: no cover
     stop=stop_after_attempt(4),  # Retry 3 times before raising
     wait=wait_fixed(20),  # Wait seconds between retries
 )
-def get_data_jira(issue_ids, cached_data=None):  # pragma: no cover
+def get_jira(jql, fields=None):
+    """Accepts the jql to retrieve the data from Jira for the given fields
+
+    Arguments:
+        jql {str} -- The query for retrieving the issue(s) details from jira
+        fields {list} -- The custom fields in query to retrieve the data for
+
+    Returns: Jira object of response after status check
+    """
+    params = {"jql": jql}
+    if fields:
+        params.update({"fields": ",".join(fields)})
+    response = requests.get(
+        f"{settings.jira.url}/rest/api/latest/search/",
+        params=params,
+        headers={"Authorization": f"Bearer {settings.jira.api_key}"},
+    )
+    response.raise_for_status()
+    return response
+
+
+def get_data_jira(issue_ids, cached_data=None, jira_fields=None):  # pragma: no cover
     """Get a list of marked Jira data and query Jira REST API.
 
     Arguments:
         issue_ids {list of str} -- ['SAT-12345', ...]
         cached_data {dict} -- Cached data previous loaded from API
+        jira_fields {list of str} -- List of fields to be retrieved by a jira issue GET request
 
     Returns:
         [list of dicts] -- [{'id':..., 'status':..., 'resolution': ...}]
     """
+    if not jira_fields:
+        jira_fields = common_jira_fields
+
     if not issue_ids:
         return []
 
@@ -204,48 +253,18 @@ def get_data_jira(issue_ids, cached_data=None):  # pragma: no cover
 
     # No cached data so Call Jira API
     logger.debug(f"Calling Jira API for {set(issue_ids)}")
-    jira_fields = [
-        "key",
-        "summary",
-        "status",
-        "labels",
-        "resolution",
-        "fixVersions",
-    ]
     # Following fields are dynamically calculated/loaded
     for field in ('is_open', 'version'):
         assert field not in jira_fields
 
     # Generate jql
+    if isinstance(issue_ids, str):
+        issue_ids = [issue_id.strip() for issue_id in issue_ids.split(',')]
     jql = ' OR '.join([f"id = {issue_id}" for issue_id in issue_ids])
-
-    response = requests.get(
-        f"{settings.jira.url}/rest/api/latest/search/",
-        params={
-            "jql": jql,
-            "fields": ",".join(jira_fields),
-        },
-        headers={"Authorization": f"Bearer {settings.jira.api_key}"},
-    )
-    response.raise_for_status()
+    response = get_jira(jql, jira_fields)
     data = response.json().get('issues')
     # Clean the data, only keep the required info.
-    data = [
-        {
-            'key': issue['key'],
-            'summary': issue['fields']['summary'],
-            'status': issue['fields']['status']['name'],
-            'labels': issue['fields']['labels'],
-            'resolution': issue['fields']['resolution']['name']
-            if issue['fields']['resolution']
-            else '',
-            'fixVersions': [ver['name'] for ver in issue['fields']['fixVersions']]
-            if issue['fields']['fixVersions']
-            else [],
-        }
-        for issue in data
-        if issue is not None
-    ]
+    data = [sanitized_issue_data(issue, jira_fields) for issue in data if issue is not None]
     CACHED_RESPONSES['get_data'][str(sorted(issue_ids))] = data
     return data
 

scripts/customer_scenarios.py

@@ -6,6 +6,7 @@
 import testimony
 
 from robottelo.config import settings
+from robottelo.utils.issue_handlers.jira import get_data_jira
 
 
 @click.group()
@@ -30,10 +31,39 @@ def get_bz_data(paths):
         for test in tests:
             test_dict = test.to_dict()
             test_data = {**test_dict['tokens'], **test_dict['invalid-tokens']}
-            if 'bz' in test_data and (
-                'customerscenario' not in test_data or test_data['customerscenario'] == 'false'
+            lowered_test_data = {name.lower(): val for name, val in test_data.items()}
+            if 'bz' in lowered_test_data and (
+                'customerscenario' not in lowered_test_data
+                or lowered_test_data['customerscenario'] == 'false'
             ):
-                path_result.append([test.name, test_data['bz']])
+                path_result.append([test.name, lowered_test_data['bz']])
+        if path_result:
+            result[path] = path_result
+    return result
+
+
+def get_tests_path_without_customer_tag(paths):
+    """Returns the path and test name that does not have customerscenario token even
+    though it has verifies token when necessary
+
+    Arguments:
+        paths {list} -- List of test modules paths
+    """
+    testcases = testimony.get_testcases(paths)
+    result = {}
+    for path, tests in testcases.items():
+        path_result = []
+        for test in tests:
+            test_dict = test.to_dict()
+            test_data = {**test_dict['tokens'], **test_dict['invalid-tokens']}
+            # 1st level lowering should be good enough as `verifies` and `customerscenario`
+            # tokens are at 1st level
+            lowered_test_data = {name.lower(): val for name, val in test_data.items()}
+            if 'verifies' in lowered_test_data and (
+                'customerscenario' not in lowered_test_data
+                or lowered_test_data['customerscenario'] == 'false'
+            ):
+                path_result.append([test.name, lowered_test_data['verifies']])
         if path_result:
             result[path] = path_result
     return result
@@ -82,11 +112,41 @@ def query_bz(data):
     return set(output)
 
 
+def query_jira(data):
+    """Returns the list of path and test name for missing customerscenario token
+
+    Arguments:
+         data {dict} -- The list of test modules and tests without customerscenario tags
+    """
+    output = []
+    sfdc_counter_field = 'customfield_12313440'
+    with click.progressbar(data.items()) as bar:
+        for path, tests in bar:
+            for test in tests:
+                jira_data = get_data_jira(test[1], jira_fields=[sfdc_counter_field])
+                for data in jira_data:
+                    customer_cases = int(float(data[sfdc_counter_field]))
+                    if customer_cases and customer_cases >= 1:
+                        output.append(f'{path} {test}')
+                        break
+    return set(output)
+
+
 @main.command()
-def run(paths=None):
-    path_list = make_path_list(paths)
-    values = get_bz_data(path_list)
-    results = query_bz(values)
+@click.option('--jira', is_flag=True, help='Run the customer scripting for Jira')
+@click.option('--bz', is_flag=True, help='Run the customer scripting for BZ')
+def run(jira, bz, paths=None):
+    if jira:
+        path_list = make_path_list(paths)
+        values = get_tests_path_without_customer_tag(path_list)
+        results = query_jira(values)
+    elif bz:
+        path_list = make_path_list(paths)
+        values = get_bz_data(path_list)
+        results = query_bz(values)
+    else:
+        raise UserWarning('Choose either `--jira` or `--bz` option')
+
     if len(results) == 0:
         click.echo('No action needed for customerscenario tags')
     else: