diff --git a/setup.cfg b/setup.cfg index e11d9b6..ddf10fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -89,6 +89,7 @@ python = 3.9: py39 pypy-2: pypy2 pypy-3: pypy3 +envs_are_optional = true [testenv] description = run test suite under {basepython} @@ -100,7 +101,7 @@ commands = pytest --cov=tox_gh_actions --cov-branch --cov-report=term --cov-repo [testenv:black] description = run black with check-only under {basepython} -commands = black --check src/ tests/ setup.py +commands = black --check --diff src/ tests/ setup.py extras = testing [testenv:flake8] diff --git a/src/tox_gh_actions/plugin.py b/src/tox_gh_actions/plugin.py index f16a1f9..8f89f35 100644 --- a/src/tox_gh_actions/plugin.py +++ b/src/tox_gh_actions/plugin.py @@ -1,11 +1,11 @@ -from itertools import product +from itertools import combinations, product import os import sys from typing import Any, Dict, Iterable, List import pluggy from tox.config import Config, TestenvConfig, _split_env as split_env -from tox.reporter import verbosity1, verbosity2 +from tox.reporter import verbosity1, verbosity2, warning from tox.venv import VirtualEnv @@ -39,10 +39,21 @@ def tox_configure(config): gh_actions_config = parse_config(config._cfg.sections) verbosity2("tox-gh-actions config: {}".format(gh_actions_config)) + if gh_actions_config["envs_are_optional"] is None: + warning( + "Config 'gh-actions.envs_are_optional' will become the default in a " + "future release. Set explicitly to 'true' or 'false' to disable this " + "warning." + ) + factors = get_factors(gh_actions_config, versions) verbosity2("using the following factors to decide envlist: {}".format(factors)) - envlist = get_envlist_from_factors(config.envlist, factors) + envlist = get_envlist_from_factors( + config.envlist, + factors, + envs_are_optional=gh_actions_config["envs_are_optional"], + ) config.envlist_default = config.envlist = envlist verbosity1("overriding envlist with: {}".format(envlist)) @@ -65,24 +76,32 @@ def tox_runtest_post(venv): print("::endgroup::") +def parse_env_config(value): + # type: (str) -> Dict[str, Dict[str, List[str]]] + return {k: split_env(v) for k, v in parse_dict(value).items()} + + def parse_config(config): - # type: (Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, Any]] + # type: (Dict[str, Dict[str, str]]) -> Dict[str, Any] """Parse gh-actions section in tox.ini""" - config_python = parse_dict(config.get("gh-actions", {}).get("python", "")) - config_env = { - name: {k: split_env(v) for k, v in parse_dict(conf).items()} - for name, conf in config.get("gh-actions:env", {}).items() - } + action_config = config.get("gh-actions", {}) + envs_are_optional = action_config.get("envs_are_optional") # Example of split_env: # "py{27,38}" => ["py27", "py38"] return { - "python": {k: split_env(v) for k, v in config_python.items()}, - "env": config_env, + "python": parse_env_config(action_config.get("python", "")), + "envs_are_optional": ( + None if envs_are_optional is None else envs_are_optional.lower() == "true" + ), + "env": { + name: parse_env_config(conf) + for name, conf in config.get("gh-actions:env", {}).items() + }, } def get_factors(gh_actions_config, versions): - # type: (Dict[str, Dict[str, Any]], Iterable[str]) -> List[str] + # type: (Dict[str, Any], Iterable[str]) -> List[List[str]] """Get a list of factors""" factors = [] # type: List[List[str]] for version in versions: @@ -95,20 +114,48 @@ def get_factors(gh_actions_config, versions): env_value = os.environ[env] if env_value in env_config: factors.append(env_config[env_value]) - return [x for x in map(lambda f: "-".join(f), product(*factors)) if x] + return factors -def get_envlist_from_factors(envlist, factors): - # type: (Iterable[str], Iterable[str]) -> List[str] +def get_envlist_from_factors(envlist, grouped_factors, envs_are_optional=False): + # type: (Iterable[str], Iterable[List[List[str]]], bool) -> List[str] """Filter envlist using factors""" - result = [] - for env in envlist: - for factor in factors: - env_facts = env.split("-") - if all(f in env_facts for f in factor.split("-")): - result.append(env) - break - return result + if not grouped_factors: + return [] + + result = set() + all_env_factors = [(set(e.split("-")), e) for e in envlist] + + if not envs_are_optional: + for env_factors, env in all_env_factors: + for factors in product(*grouped_factors): + if env_factors.issuperset(factors): + result.add(env) + else: + # The first factors come from the python config and are required + for required_factor in grouped_factors[0]: + env_factors = [(f, e) for f, e in all_env_factors if required_factor in f] + + # The remaining factors come from the env and will be tried exactly at + # first, and then will be tried again after a single factor is removed + # until there is only 1 factor left. All matches after removing N factors + # are added to the result set. + matches = set() + for optional_factors in product(*grouped_factors[1:]): + for count in range(len(optional_factors), 0, -1): + for factors in combinations(optional_factors, count): + factors = set(factors) + matches.update(e for f, e in env_factors if f >= factors) + + if matches: + result |= matches + break + + # if none of the optional factors matched add all required matches + if not matches: + result.update(e for f, e in env_factors) + + return [i for i in envlist if i in result] def get_python_version_keys(): diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 1b5b19b..912529e 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -23,6 +23,28 @@ "3.6": ["py36"], "3.7": ["py37", "flake8"], }, + "envs_are_optional": None, + "env": {}, + }, + ), + ( + { + "gh-actions": { + "python": """2.7: py27 +3.5: py35 +3.6: py36 +3.7: py37, flake8""", + "envs_are_optional": "true", + } + }, + { + "python": { + "2.7": ["py27"], + "3.5": ["py35"], + "3.6": ["py36"], + "3.7": ["py37", "flake8"], + }, + "envs_are_optional": True, "env": {}, }, ), @@ -43,6 +65,7 @@ "2.7": ["py27"], "3.8": ["py38"], }, + "envs_are_optional": None, "env": { "PLATFORM": { "ubuntu-latest": ["linux"], @@ -56,6 +79,31 @@ {"gh-actions": {}}, { "python": {}, + "envs_are_optional": None, + "env": {}, + }, + ), + ( + { + "gh-actions": { + "envs_are_optional": "false", + } + }, + { + "python": {}, + "envs_are_optional": False, + "env": {}, + }, + ), + ( + { + "gh-actions": { + "unknown": "unknown", + } + }, + { + "python": {}, + "envs_are_optional": None, "env": {}, }, ), @@ -63,6 +111,7 @@ {}, { "python": {}, + "envs_are_optional": None, "env": {}, }, ), @@ -85,7 +134,7 @@ def test_parse_config(config, expected): }, ["2.7", "2"], {}, - ["py27", "flake8"], + [["py27", "flake8"]], ), # Get factors using less precise Python version ( @@ -98,7 +147,7 @@ def test_parse_config(config, expected): }, ["3.8", "3"], {}, - ["py3", "flake8"], + [["py3", "flake8"]], ), # Get factors only from the most precise Python version ( @@ -112,7 +161,7 @@ def test_parse_config(config, expected): }, ["3.9", "3"], {}, - ["py39"], + [["py39"]], ), ( { @@ -132,7 +181,7 @@ def test_parse_config(config, expected): "SAMPLE": "VALUE1", "HOGE": "VALUE3", }, - ["py27-fact1", "py27-fact2", "flake8-fact1", "flake8-fact2"], + [["py27", "flake8"], ["fact1", "fact2"]], ), ( { @@ -156,16 +205,7 @@ def test_parse_config(config, expected): "SAMPLE": "VALUE1", "HOGE": "VALUE3", }, - [ - "py27-fact1-fact5", - "py27-fact1-fact6", - "py27-fact2-fact5", - "py27-fact2-fact6", - "flake8-fact1-fact5", - "flake8-fact1-fact6", - "flake8-fact2-fact5", - "flake8-fact2-fact6", - ], + [["py27", "flake8"], ["fact1", "fact2"], ["fact5", "fact6"]], ), ( { @@ -185,12 +225,7 @@ def test_parse_config(config, expected): "SAMPLE": "VALUE1", "HOGE": "VALUE3", }, - [ - "py27-django18", - "py27-flake8", - "flake8-django18", - "flake8-flake8", - ], + [["py27", "flake8"], ["django18", "flake8"]], ), ( { @@ -210,7 +245,7 @@ def test_parse_config(config, expected): { "SAMPLE": "VALUE3", }, - ["py27", "flake8"], + [["py27", "flake8"]], ), ( { @@ -229,7 +264,7 @@ def test_parse_config(config, expected): { "SAMPLE": "VALUE2", }, - ["py38", "flake8"], + [["py38", "flake8"]], ), ( { @@ -260,48 +295,74 @@ def test_get_factors(mocker, config, version, environ, expected): def normalize_factors_list(factors): """Utility to make it compare equality of a list of factors""" - result = [tuple(sorted(f.split("-"))) for f in factors] - result.sort() - return result + return [factors[:1], {frozenset(f) for f in factors[1:]}] @pytest.mark.parametrize( - "envlist,factors,expected", + "envlist,factors,lax,expected", [ ( ["py27", "py37", "flake8"], - ["py37", "flake8"], + [["py37", "flake8"]], + [True, False], ["py37", "flake8"], ), ( ["py27", "py37", "flake8"], [], + [True, False], [], ), ( [], - ["py37", "flake8"], + [["py37", "flake8"]], + [True, False], [], ), ( ["py27-dj111", "py37-dj111", "py37-dj20", "flake8"], - ["py37", "flake8"], + [["py37", "flake8"]], + [True, False], ["py37-dj111", "py37-dj20", "flake8"], ), ( ["py27-django18", "py37-django18", "flake8"], - [ - "py27-django18", - "py27-flake8", - "flake8-django18", - "flake8-flake8", - ], + [["py27", "flake8"], ["django18", "flake8"]], + [True, False], ["py27-django18", "flake8"], ), + # The following two show the difference between lax and non lax selection: + ( + ["py27-dj111", "py37-dj111", "py37-dj20", "flake8"], + [["py37", "flake8"], ["dj111"]], + [True], + ["py37-dj111", "flake8"], + ), + ( + ["py27-dj111", "py37-dj111", "py37-dj20", "flake8"], + [["py37", "flake8"], ["dj111"]], + [False], + ["py37-dj111"], + ), + # When lax selection is enabled the most specific match is used rather than + # selecting any match: + ( + ["py27-dj111", "py37-dj111", "py37-dj20", "flake8", "flake8-dj111"], + [["py37", "flake8"], ["dj111"]], + [True], + ["py37-dj111", "flake8-dj111"], + ), + ( + ["py27-dj111", "py37-dj111", "py37-dj20", "flake8", "flake8-dj111"], + [["py37", "flake8"], ["dj111"]], + [False], + ["py37-dj111", "flake8-dj111"], + ), ], ) -def test_get_envlist_from_factors(envlist, factors, expected): - assert plugin.get_envlist_from_factors(envlist, factors) == expected +def test_get_envlist_from_factors(envlist, factors, lax, expected): + for _lax in lax: + assert plugin.get_envlist_from_factors(envlist, factors, _lax) == expected @pytest.mark.parametrize(