From f631b4806f524bd659be484081777d3c9a54815e Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Fri, 22 Mar 2024 17:12:53 +0000 Subject: [PATCH] Use tomlkit for TOML read/write operations (#215) * small cleanups * clean up argparse defaults --------- Co-authored-by: Michael Howitz --- config/config-package.py | 17 +++-- config/drop-legacy-python.py | 5 +- config/meta-cfg-to-toml.py | 114 ---------------------------------- config/requirements.txt | 10 +-- config/shared/toml_encoder.py | 47 -------------- 5 files changed, 15 insertions(+), 178 deletions(-) delete mode 100644 config/meta-cfg-to-toml.py delete mode 100644 config/shared/toml_encoder.py diff --git a/config/config-package.py b/config/config-package.py index 0c3ec1e..e28ebaa 100755 --- a/config/config-package.py +++ b/config/config-package.py @@ -18,13 +18,12 @@ from shared.git import get_commit_id from shared.git import git_branch from shared.path import change_dir -from shared.toml_encoder import TomlArraySeparatorEncoderWithNewline import argparse import collections import jinja2 import pathlib import shutil -import toml +import tomlkit META_HINT = """\ @@ -67,13 +66,13 @@ def handle_command_line_arguments(): '--no-flake8', dest='use_flake8', action='store_false', - default=None, + default=True, help='Do not include flake8 and isort in the linting configuration.') parser.add_argument( '--with-appveyor', dest='with_appveyor', action='store_true', - default=None, + default=False, help='Activate running tests on AppVeyor, too, ' 'if not already configured in .meta.toml.') parser.add_argument( @@ -111,7 +110,7 @@ def handle_command_line_arguments(): '--with-sphinx', dest='with_docs', action='store_true', - default=None, + default=False, help='Activate building docs if not already configured in .meta.toml.') parser.add_argument( '--with-sphinx-doctests', @@ -178,7 +177,8 @@ def _read_meta_configuration(self): """Read and update meta configuration""" meta_toml_path = self.path / '.meta.toml' if meta_toml_path.exists(): - meta_cfg = toml.load(meta_toml_path) + with open(meta_toml_path, 'rb') as meta_f: + meta_cfg = tomlkit.load(meta_f) meta_cfg = collections.defaultdict(dict, **meta_cfg) else: meta_cfg = collections.defaultdict(dict) @@ -636,10 +636,7 @@ def configure(self): with open('.meta.toml', 'w') as meta_f: meta_f.write(META_HINT.format(config_type=self.config_type)) meta_f.write('\n') - toml.dump( - meta_cfg, meta_f, - TomlArraySeparatorEncoderWithNewline( - separator=',\n ', indent_first_line=True)) + tomlkit.dump(meta_cfg, meta_f) tox_path = shutil.which('tox') or ( pathlib.Path(cwd) / 'bin' / 'tox') diff --git a/config/drop-legacy-python.py b/config/drop-legacy-python.py index e24ac7a..50ab1de 100644 --- a/config/drop-legacy-python.py +++ b/config/drop-legacy-python.py @@ -22,7 +22,7 @@ import pathlib import shutil import sys -import toml +import tomlkit parser = argparse.ArgumentParser( @@ -56,7 +56,8 @@ with change_dir(path) as cwd_str: cwd = pathlib.Path(cwd_str) bin_dir = cwd / 'bin' - meta_cfg = collections.defaultdict(dict, **toml.load('.meta.toml')) + with open('.meta.toml', 'rb') as meta_f: + meta_cfg = collections.defaultdict(dict, **tomlkit.load(meta_f)) config_type = meta_cfg['meta']['template'] branch_name = get_branch_name(args.branch_name, config_type) updating = git_branch(branch_name) diff --git a/config/meta-cfg-to-toml.py b/config/meta-cfg-to-toml.py deleted file mode 100644 index 5d079f6..0000000 --- a/config/meta-cfg-to-toml.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -############################################################################## -# -# Copyright (c) 2020 Zope Foundation and Contributors. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -from configparser import ConfigParser -from shared.call import call -from shared.path import change_dir -from shared.toml_encoder import TomlArraySeparatorEncoderWithNewline -import argparse -import collections -import pathlib -import sys -import toml - - -parser = argparse.ArgumentParser( - description='Convert the .meta.cfg in a package to a .meta.toml.') -parser.add_argument( - 'path', type=pathlib.Path, help='path to the repository to be changed') -parser.add_argument( - '--no-push', - dest='no_push', - action='store_true', - help='Prevent direct push.') - - -args = parser.parse_args() -path = args.path -meta_cfg_path = path / '.meta.cfg' - -if not (path / '.git').exists(): - raise ValueError('`path` does not point to a git clone of a repository!') - -if not meta_cfg_path.exists(): - raise ValueError('`path` does have a `.meta.cfg`!') - - -src = ConfigParser() -src.read(meta_cfg_path) -src = src['meta'] - -dest = collections.defaultdict(dict) -dest['meta']['template'] = src['template'] -dest['meta']['commit-id'] = src['commit-id'] - -dest['python']['with-pypy'] = src.getboolean('with-pypy', False) -dest['python']['with-docs'] = src.getboolean('with-docs', False) -dest['python']['with-sphinx-doctests'] = src.getboolean( - 'with-sphinx-doctests', False) - -dest['coverage']['fail-under'] = int(src.setdefault('fail-under', 0)) - -flake8_config = src.get('additional-flake8-config', '').strip() -if flake8_config: - dest['flake8']['additional-config'] = flake8_config.splitlines() - -manifest_rules = src.get('additional-manifest-rules', '').strip() -if manifest_rules: - dest['manifest']['additional-rules'] = manifest_rules.splitlines() - -check_manifest = src.get('additional-check-manifest-ignores', '').strip() -if check_manifest: - dest['check-manifest']['additional-ignores'] = check_manifest.splitlines() - - -branch_name = 'covert.meta.cfg-to-.meta.toml' -with change_dir(path) as cwd: - with open('.meta.toml', 'w') as meta_f: - meta_f.write( - '# Generated from:\n' - '# https://github.com/zopefoundation/meta/tree/master/config/' - f'{src["template"]}\n') - toml.dump( - dest, meta_f, - TomlArraySeparatorEncoderWithNewline( - separator=',\n ', indent_first_line=True)) - - branches = call( - 'git', 'branch', '--format', '%(refname:short)', - capture_output=True).stdout.splitlines() - if branch_name in branches: - call('git', 'checkout', branch_name) - updating = True - else: - call('git', 'checkout', '-b', branch_name) - updating = False - call('git', 'rm', '.meta.cfg') - call('git', 'add', '.meta.toml') - call('git', 'commit', '-m', 'Switching from .meta.cfg to .meta.toml.') - config_package_args = [ - sys.executable, - 'config-package.py', - path, - f'--branch={branch_name}', - ] - if args.no_push: - config_package_args.append('--no-push') - call(*config_package_args, cwd=cwd) - print() - print('Created resp. updated branch', end='') - if args.no_push: - print(', but did not push upstream.') - else: - call('git', 'push', '--set-upstream', 'origin', branch_name) - print('and pushed upstream.') diff --git a/config/requirements.txt b/config/requirements.txt index d419f51..b0ec9b3 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -1,6 +1,6 @@ -check-python-versions==0.20.0 +check-python-versions==0.21.3 Jinja2==3.1.3 -pyupgrade==3.3.1 -toml==0.10.2 -tox==4.0.14 -zest.releaser==7.2.0 +pyupgrade==3.3.2 +tomlkit==0.12.1 +tox==4.8.0 +zest.releaser==8.0.0 diff --git a/config/shared/toml_encoder.py b/config/shared/toml_encoder.py deleted file mode 100644 index e84625f..0000000 --- a/config/shared/toml_encoder.py +++ /dev/null @@ -1,47 +0,0 @@ -############################################################################## -# -# Copyright (c) 2020 Zope Foundation and Contributors. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -import toml - - -class TomlArraySeparatorEncoderWithNewline(toml.TomlArraySeparatorEncoder): - """Special version indenting the first element of and array. - - In https://github.com/zopefoundation/meta/issues/118 we suggest to switch - to Python 3.11 and its built-in toml support. We'll see if this path is - still needed then. - """ - - def __init__(self, _dict=dict, preserve=False, separator=",", - indent_first_line=False): - super(TomlArraySeparatorEncoderWithNewline, self).__init__( - _dict=_dict, preserve=preserve, separator=separator) - self.indent_first_line = indent_first_line - - def dump_list(self, v): - t = [] - retval = "[" - if self.indent_first_line: - retval += self.separator.strip(',') - for u in v: - t.append(self.dump_value(u)) - while t != []: - s = [] - for u in t: - if isinstance(u, list): - for r in u: - s.append(r) - else: - retval += " " + u + self.separator - t = s - retval += " ]" - return retval