diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 92fa9f329a9..b07e6f1d85e 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -48,7 +48,7 @@ jobs: echo "assignees: pondrejk " >> .env.md echo "labels: Documentation" >> .env.md echo "---" >> .env.md - echo CS_TAGS="$(make customer-scenario-check)" >> .env.md + echo CS_TAGS="$(make customer-scenario-check-jira)" >> .env.md if grep 'The following tests need customerscenario tags' .env.md; then echo "::set-output name=result::0" fi diff --git a/Makefile b/Makefile index ff0920c70ee..ac14dc3a0eb 100644 --- a/Makefile +++ b/Makefile @@ -169,8 +169,11 @@ clean-cache: clean-all: docs-clean logs-clean pyc-clean clean-cache clean-shared -customer-scenario-check: - @scripts/customer_scenarios.py +customer-scenario-check-bz: + @scripts/customer_scenarios.py --bz + +customer-scenario-check-jira: + @scripts/customer_scenarios.py --jira vault-login: @scripts/vault_login.py --login diff --git a/conf/dynaconf_hooks.py b/conf/dynaconf_hooks.py index b3d7a8e219c..0fb3734eeee 100644 --- a/conf/dynaconf_hooks.py +++ b/conf/dynaconf_hooks.py @@ -53,7 +53,7 @@ def config_migrations(settings, data): :type data: dict """ logger.info('Running config migration hooks') - sys.path.append(str(Path(__file__).parent)) + sys.path.append(str(Path(__file__).parent.parent)) from conf import migrations migration_functions = [ diff --git a/robottelo/utils/issue_handlers/jira.py b/robottelo/utils/issue_handlers/jira.py index f9ec6e47bfc..0c50a47c9a8 100644 --- a/robottelo/utils/issue_handlers/jira.py +++ b/robottelo/utils/issue_handlers/jira.py @@ -20,6 +20,31 @@ # The .version group being a `d.d` string that can be casted to Version() VERSION_RE = re.compile(r'(?:sat-)*?(?P\d\.\d)\.\w*') +common_jira_fields = ['key', 'summary', 'status', 'labels', 'resolution', 'fixVersions'] + +mapped_response_fields = { + 'key': "{obj_name}['key']", + 'summary': "{obj_name}['fields']['summary']", + 'status': "{obj_name}['fields']['status']['name']", + 'labels': "{obj_name}['fields']['labels']", + 'resolution': "{obj_name}['fields']['resolution']['name'] if {obj_name}['fields']['resolution'] else ''", + 'fixVersions': "[ver['name'] for ver in {obj_name}['fields']['fixVersions']] if {obj_name}['fields']['fixVersions'] else []", + # Custom Field - SFDC Cases Counter + 'customfield_12313440': "{obj_name}['fields']['customfield_12313440']", +} + + +def sanitized_issue_data(issue, out_fields): + """fetches the value for all the given fields from a given jira issue + + Arguments: + issue {dict} -- The json data for a jira issue + out_fields {list} -- The list of fields for which data to be retrieved from jira issue + """ + return { + field: eval(mapped_response_fields[field].format(obj_name='issue')) for field in out_fields + } + def is_open_jira(issue_id, data=None): """Check if specific Jira is open consulting a cached `data` dict or @@ -131,8 +156,7 @@ def collect_data_jira(collected_data, cached_data): # pragma: no cover """ jira_data = ( get_data_jira( - [item for item in collected_data if item.startswith('SAT-')], - cached_data=cached_data, + [item for item in collected_data if item.startswith('SAT-')], cached_data=cached_data ) or [] ) @@ -169,16 +193,41 @@ def collect_dupes(jira, collected_data, cached_data=None): # pragma: no cover stop=stop_after_attempt(4), # Retry 3 times before raising wait=wait_fixed(20), # Wait seconds between retries ) -def get_data_jira(issue_ids, cached_data=None): # pragma: no cover +def get_jira(jql, fields=None): + """Accepts the jql to retrieve the data from Jira for the given fields + + Arguments: + jql {str} -- The query for retrieving the issue(s) details from jira + fields {list} -- The custom fields in query to retrieve the data for + + Returns: Jira object of response after status check + """ + params = {"jql": jql} + if fields: + params.update({"fields": ",".join(fields)}) + response = requests.get( + f"{settings.jira.url}/rest/api/latest/search/", + params=params, + headers={"Authorization": f"Bearer {settings.jira.api_key}"}, + ) + response.raise_for_status() + return response + + +def get_data_jira(issue_ids, cached_data=None, jira_fields=None): # pragma: no cover """Get a list of marked Jira data and query Jira REST API. Arguments: issue_ids {list of str} -- ['SAT-12345', ...] cached_data {dict} -- Cached data previous loaded from API + jira_fields {list of str} -- List of fields to be retrieved by a jira issue GET request Returns: [list of dicts] -- [{'id':..., 'status':..., 'resolution': ...}] """ + if not jira_fields: + jira_fields = common_jira_fields + if not issue_ids: return [] @@ -204,48 +253,18 @@ def get_data_jira(issue_ids, cached_data=None): # pragma: no cover # No cached data so Call Jira API logger.debug(f"Calling Jira API for {set(issue_ids)}") - jira_fields = [ - "key", - "summary", - "status", - "labels", - "resolution", - "fixVersions", - ] # Following fields are dynamically calculated/loaded for field in ('is_open', 'version'): assert field not in jira_fields # Generate jql + if isinstance(issue_ids, str): + issue_ids = [issue_id.strip() for issue_id in issue_ids.split(',')] jql = ' OR '.join([f"id = {issue_id}" for issue_id in issue_ids]) - - response = requests.get( - f"{settings.jira.url}/rest/api/latest/search/", - params={ - "jql": jql, - "fields": ",".join(jira_fields), - }, - headers={"Authorization": f"Bearer {settings.jira.api_key}"}, - ) - response.raise_for_status() + response = get_jira(jql, jira_fields) data = response.json().get('issues') # Clean the data, only keep the required info. - data = [ - { - 'key': issue['key'], - 'summary': issue['fields']['summary'], - 'status': issue['fields']['status']['name'], - 'labels': issue['fields']['labels'], - 'resolution': issue['fields']['resolution']['name'] - if issue['fields']['resolution'] - else '', - 'fixVersions': [ver['name'] for ver in issue['fields']['fixVersions']] - if issue['fields']['fixVersions'] - else [], - } - for issue in data - if issue is not None - ] + data = [sanitized_issue_data(issue, jira_fields) for issue in data if issue is not None] CACHED_RESPONSES['get_data'][str(sorted(issue_ids))] = data return data diff --git a/scripts/customer_scenarios.py b/scripts/customer_scenarios.py index b81137042b9..adee83132ff 100755 --- a/scripts/customer_scenarios.py +++ b/scripts/customer_scenarios.py @@ -6,6 +6,7 @@ import testimony from robottelo.config import settings +from robottelo.utils.issue_handlers.jira import get_data_jira @click.group() @@ -30,10 +31,39 @@ def get_bz_data(paths): for test in tests: test_dict = test.to_dict() test_data = {**test_dict['tokens'], **test_dict['invalid-tokens']} - if 'bz' in test_data and ( - 'customerscenario' not in test_data or test_data['customerscenario'] == 'false' + lowered_test_data = {name.lower(): val for name, val in test_data.items()} + if 'bz' in lowered_test_data and ( + 'customerscenario' not in lowered_test_data + or lowered_test_data['customerscenario'] == 'false' ): - path_result.append([test.name, test_data['bz']]) + path_result.append([test.name, lowered_test_data['bz']]) + if path_result: + result[path] = path_result + return result + + +def get_tests_path_without_customer_tag(paths): + """Returns the path and test name that does not have customerscenario token even + though it has verifies token when necessary + + Arguments: + paths {list} -- List of test modules paths + """ + testcases = testimony.get_testcases(paths) + result = {} + for path, tests in testcases.items(): + path_result = [] + for test in tests: + test_dict = test.to_dict() + test_data = {**test_dict['tokens'], **test_dict['invalid-tokens']} + # 1st level lowering should be good enough as `verifies` and `customerscenario` + # tokens are at 1st level + lowered_test_data = {name.lower(): val for name, val in test_data.items()} + if 'verifies' in lowered_test_data and ( + 'customerscenario' not in lowered_test_data + or lowered_test_data['customerscenario'] == 'false' + ): + path_result.append([test.name, lowered_test_data['verifies']]) if path_result: result[path] = path_result return result @@ -82,11 +112,41 @@ def query_bz(data): return set(output) +def query_jira(data): + """Returns the list of path and test name for missing customerscenario token + + Arguments: + data {dict} -- The list of test modules and tests without customerscenario tags + """ + output = [] + sfdc_counter_field = 'customfield_12313440' + with click.progressbar(data.items()) as bar: + for path, tests in bar: + for test in tests: + jira_data = get_data_jira(test[1], jira_fields=[sfdc_counter_field]) + for data in jira_data: + customer_cases = int(float(data[sfdc_counter_field])) + if customer_cases and customer_cases >= 1: + output.append(f'{path} {test}') + break + return set(output) + + @main.command() -def run(paths=None): - path_list = make_path_list(paths) - values = get_bz_data(path_list) - results = query_bz(values) +@click.option('--jira', is_flag=True, help='Run the customer scripting for Jira') +@click.option('--bz', is_flag=True, help='Run the customer scripting for BZ') +def run(jira, bz, paths=None): + if jira: + path_list = make_path_list(paths) + values = get_tests_path_without_customer_tag(path_list) + results = query_jira(values) + elif bz: + path_list = make_path_list(paths) + values = get_bz_data(path_list) + results = query_bz(values) + else: + raise UserWarning('Choose either `--jira` or `--bz` option') + if len(results) == 0: click.echo('No action needed for customerscenario tags') else: