From f12f31c11bb798229f14286bd4304bd94e901b5a Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Sun, 8 Sep 2024 16:29:00 +0200 Subject: [PATCH] - use pyproject.toml to store coverage configuration --- config/buildout-recipe/tox.ini.j2 | 4 +- config/c-code/tox.ini.j2 | 6 +- config/config-package.py | 73 ++++++++--------- config/default/tox-coverage-config.j2 | 22 ------ config/pure-python/tox.ini.j2 | 7 +- config/shared/packages.py | 108 ++++++++++++++++++++++++++ config/zope-product/tox.ini.j2 | 5 +- 7 files changed, 152 insertions(+), 73 deletions(-) delete mode 100644 config/default/tox-coverage-config.j2 diff --git a/config/buildout-recipe/tox.ini.j2 b/config/buildout-recipe/tox.ini.j2 index a1c2b796..6bf86440 100644 --- a/config/buildout-recipe/tox.ini.j2 +++ b/config/buildout-recipe/tox.ini.j2 @@ -17,7 +17,7 @@ setenv = COVERAGE_PROCESS_START={toxinidir}/.coveragerc {% endif %} deps = - coverage + coverage[toml] {% for line in testenv_deps %} %(line)s {% endfor %} @@ -36,7 +36,7 @@ commands = {% endif %} coverage combine coverage html - coverage report -m --fail-under=%(coverage_fail_under)s + coverage report {% for line in coverage_additional %} %(line)s {% endfor %} diff --git a/config/c-code/tox.ini.j2 b/config/c-code/tox.ini.j2 index 66d36e8a..b2f2f995 100644 --- a/config/c-code/tox.ini.j2 +++ b/config/c-code/tox.ini.j2 @@ -33,7 +33,7 @@ basepython = %(coverage_basepython)s allowlist_externals = mkdir deps = - coverage + coverage[toml] {% for line in testenv_deps %} %(line)s {% endfor %} @@ -50,8 +50,8 @@ commands = {% else %} coverage run -m zope.testrunner --test-path=src {posargs:-vc} {% endif %} - coverage html -i - coverage report -i -m --fail-under=%(coverage_fail_under)s + coverage html + coverage report {% for line in coverage_additional %} %(line)s {% endfor %} diff --git a/config/config-package.py b/config/config-package.py index 9150bfac..caa55a24 100755 --- a/config/config-package.py +++ b/config/config-package.py @@ -19,12 +19,13 @@ from shared.git import get_commit_id from shared.git import git_branch from shared.packages import FUTURE_PYTHON_VERSION +from shared.packages import get_pyproject_toml_defaults from shared.packages import MANYLINUX_AARCH64 from shared.packages import MANYLINUX_I686 from shared.packages import MANYLINUX_PYTHON_VERSION from shared.packages import MANYLINUX_X86_64 from shared.packages import OLDEST_PYTHON_VERSION -from shared.packages import PYPROJECT_TOML_DEFAULTS +from shared.packages import parse_additional_config from shared.packages import PYPY_VERSION from shared.packages import SETUPTOOLS_VERSION_SPEC from shared.path import change_dir @@ -151,8 +152,6 @@ def prepend_space(text): class PackageConfiguration: - add_coveragerc = False - rm_coveragerc = False add_manylinux = False def __init__(self, args): @@ -357,21 +356,6 @@ def readthedocs(self): rtd_build_extra=rtd_build_extra, ) - def coveragerc(self): - coverage_run_additional_config = self.meta_cfg['coverage-run'].get( - 'additional-config', []) - if (self.config_type_path / 'coveragerc.j2').exists(): - self.copy_with_meta( - 'coveragerc.j2', - self.path / '.coveragerc', - self.config_type, - coverage_run_source=self.coverage_run_source, - coverage_run_additional_config=coverage_run_additional_config, - ) - self.add_coveragerc = True - elif (self.path / '.coveragerc').exists(): - self.rm_coveragerc = True - def manylinux_sh(self): """Add the scripts to produce binary wheels""" manylinux_install_setup = self.meta_cfg['c-code'].get( @@ -553,32 +537,46 @@ def manifest_in(self): def pyproject_toml(self): """Modify pyproject.toml with meta options.""" - pyproject_toml_path = self.path / 'pyproject.toml' - pyproject_data = {} + toml_path = self.path / 'pyproject.toml' + + if toml_path.exists(): + with open(toml_path, 'rb') as fp: + toml_doc = tomlkit.load(fp) + else: + toml_doc = tomlkit.document() + preamble = f'\n{META_HINT.format(config_type=self.config_type)}' + toml_doc.add(tomlkit.comment(preamble)) + toml_data = collections.defaultdict(dict, **toml_doc) - if pyproject_toml_path.exists(): - with open(pyproject_toml_path, 'rb') as fp: - pyproject_data = tomlkit.load(fp) - pyproject_toml = collections.defaultdict(dict, **pyproject_data) - old_requires = pyproject_toml['build-system'].get('requires', []) + # Capture some pre-transformation data + old_requires = toml_data['build-system'].get('requires', []) - # Update/overwrite existing values with our defaults - pyproject_toml.update(PYPROJECT_TOML_DEFAULTS) + # Apply template-dependent defaults + toml_data.update(get_pyproject_toml_defaults(self.config_type)) - # Add prior requires values back + # Create or update section "build-system" if old_requires: setuptools_requirement = [ x for x in old_requires if x.startswith('setuptools')] for setuptools_req in setuptools_requirement: old_requires.remove(setuptools_req) - pyproject_toml['build-system']['requires'].extend(old_requires) + toml_data['build-system']['requires'].extend(old_requires) + + # Update coverage-related data + coverage = toml_data['tool']['coverage'] + coverage['run']['source'] = self.coverage_run_source.split() + coverage['report']['fail_under'] = self.coverage_fail_under + add_cfg = self.meta_cfg['coverage-run'].get( 'additional-config', []) + for key, value in parse_additional_config(add_cfg).items(): + coverage['run'][key] = value + + # Remove empty sections + toml_data = {k: v for k, v in toml_data.items() if v} - # Remove empty sections before writing to disk - pyproject_toml = {k: v for k, v in pyproject_toml.items() if v} - with open(pyproject_toml_path, 'w') as fp: - fp.write(META_HINT.format(config_type=self.config_type)) - fp.write('\n') - tomlkit.dump(pyproject_toml, fp, sort_keys=True) + # Update and write out the document + toml_doc.update(toml_data) + with open(toml_path, 'w') as fp: + tomlkit.dump(toml_doc, fp, sort_keys=True) def copy_with_meta( self, template_name, destination, config_type, @@ -631,7 +629,6 @@ def configure(self): if self.args.commit: call('git', 'add', *early_add) - self.coveragerc() self.manylinux_sh() self.tox() self.tests_yml() @@ -643,10 +640,8 @@ def configure(self): call('git', 'rm', 'bootstrap.py') if pathlib.Path('.travis.yml').exists(): call('git', 'rm', '.travis.yml') - if self.rm_coveragerc: + if pathlib.Path('.coveragerc').exists(): call('git', 'rm', '.coveragerc') - if self.add_coveragerc and self.args.commit: - call('git', 'add', '.coveragerc') if pathlib.Path('appveyor.yml').exists(): call('git', 'rm', 'appveyor.yml') if self.with_docs and self.args.commit: diff --git a/config/default/tox-coverage-config.j2 b/config/default/tox-coverage-config.j2 deleted file mode 100644 index cd221c33..00000000 --- a/config/default/tox-coverage-config.j2 +++ /dev/null @@ -1,22 +0,0 @@ - -[coverage:run] -branch = True -source = %(coverage_run_source)s -{% for line in coverage_run_additional_config %} -%(line)s -{% endfor %} - -[coverage:report] -precision = 2 -ignore_errors = True -exclude_lines = - pragma: no cover - pragma: nocover - except ImportError: - raise NotImplementedError - if __name__ == '__main__': - self.fail - raise AssertionError - -[coverage:html] -directory = parts/htmlcov diff --git a/config/pure-python/tox.ini.j2 b/config/pure-python/tox.ini.j2 index 3b0780fb..1e2b1fad 100644 --- a/config/pure-python/tox.ini.j2 +++ b/config/pure-python/tox.ini.j2 @@ -9,7 +9,7 @@ basepython = %(coverage_basepython)s allowlist_externals = mkdir deps = - coverage + coverage[toml] {% for line in testenv_deps %} %(line)s {% endfor %} @@ -34,9 +34,8 @@ commands = {% if with_sphinx_doctests %} coverage run -a -m sphinx -b doctest -d {envdir}/.cache/doctrees docs {envdir}/.cache/doctest {% endif %} - coverage html --ignore-errors - coverage report --show-missing --fail-under=%(coverage_fail_under)s + coverage html + coverage report {% for line in coverage_additional %} %(line)s {% endfor %} -{% include 'tox-coverage-config.j2' %} diff --git a/config/shared/packages.py b/config/shared/packages.py index d927d22b..ed0a432e 100644 --- a/config/shared/packages.py +++ b/config/shared/packages.py @@ -10,6 +10,7 @@ # FOR A PARTICULAR PURPOSE. # ############################################################################## +import configparser import itertools import pathlib @@ -35,9 +36,116 @@ 'requires': [f'setuptools{SETUPTOOLS_VERSION_SPEC}'], 'build-backend': 'setuptools.build_meta', }, + 'tool': { + 'coverage': { + 'run': { + 'branch': True, + 'source': 'src', + }, + 'report': { + 'fail_under': 0, + 'precision': 2, + 'ignore_errors': True, + 'show_missing': True, + 'exclude_lines': ['pragma: no cover', + 'pragma: nocover', + 'except ImportError:', + 'raise NotImplementedError', + "if __name__ == '__main__':", + 'self.fail', + 'raise AssertionError', + 'raise unittest.Skip', + ], + }, + 'html': { + 'directory': 'parts/htmlcov', + }, + }, + }, + +} +PYPROJECT_TOML_OVERRIDES = { + 'buildout-recipe': { + 'tool': { + 'coverage': { + 'run': { + 'parallel': True, + }, + 'paths': { + 'source': ['src/', + '.tox/*/lib/python*/site-packages/', + '.tox/pypy*/site-packages/', + ], + }, + }, + }, + }, + 'c-code': { + 'tool': { + 'coverage': { + 'run': { + 'relative_files': True, + }, + 'paths': { + 'source': ['src/', + '.tox/*/lib/python*/site-packages/', + '.tox/pypy*/site-packages/', + ], + }, + }, + }, + }, } +def get_pyproject_toml_defaults(template_name): + """ Get pyproject.toml default data for a given template name""" + return merge_dicts(PYPROJECT_TOML_DEFAULTS, + PYPROJECT_TOML_OVERRIDES.get(template_name, {})) + + +def merge_dicts(dict1, dict2): + for key, value in dict2.items(): + if key in dict1 and \ + isinstance(dict1[key], dict) and \ + isinstance(value, dict): + # Recursively merge nested dictionaries + dict1[key] = merge_dicts(dict1[key], value) + else: + # Merge non-dictionary values + dict1[key] = value + return dict1 + + +def parse_additional_config(cfg): + """Attempt to parse "additional-config" data + + "additional-config" sections usually contain ini-style key/value pairs + packaged into a sequence of lines. + + Returns a mapping of keys and values found + """ + data = {} + + if cfg: + parser = configparser.ConfigParser() + parser.read_string('[dummysection]\n' + '\n'.join(cfg)) + for key, value in parser.items('dummysection'): + if '\n' in value: + value = value.split() + else: + for func in (parser.getboolean, + parser.getint, + parser.getfloat): + try: + value = func('dummysection', key) + break + except: + pass + data[key] = value + + return data + def list_packages(path: pathlib.Path) -> list: """List the packages in ``path``. diff --git a/config/zope-product/tox.ini.j2 b/config/zope-product/tox.ini.j2 index 786200f2..23ec5cc4 100644 --- a/config/zope-product/tox.ini.j2 +++ b/config/zope-product/tox.ini.j2 @@ -65,7 +65,7 @@ setenv = {% endif %} deps = {[testenv]deps} - coverage + coverage[toml] commands = mkdir -p {toxinidir}/parts/htmlcov {% if coverage_command %} @@ -76,8 +76,7 @@ commands = coverage run {envbindir}/test {posargs:-cv} {% endif %} coverage html - coverage report -m --fail-under=%(coverage_fail_under)s + coverage report {% for line in coverage_additional %} %(line)s {% endfor %} -{% include 'tox-coverage-config.j2' %}