From 770456fa4fd20f6ba8d0bd858a812db5a725b295 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Fri, 22 Sep 2023 15:50:11 +0200 Subject: [PATCH 1/9] WIP: Set branch protection. --- config/c-code/tests.yml.j2 | 6 +- config/config-package.py | 2 + config/re-enable-actions.py | 23 ++--- config/requirements.txt | 1 + config/set-branch-protection-rules.py | 124 ++++++++++++++++++++++++++ config/shared/packages.py | 14 +++ 6 files changed, 153 insertions(+), 17 deletions(-) create mode 100644 config/set-branch-protection-rules.py diff --git a/config/c-code/tests.yml.j2 b/config/c-code/tests.yml.j2 index 32557a1..c0e378e 100644 --- a/config/c-code/tests.yml.j2 +++ b/config/c-code/tests.yml.j2 @@ -312,7 +312,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.9"] + python-version: ["%(manylinux_python_version)s"] os: [ubuntu-20.04] steps: @@ -338,7 +338,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.9"] + python-version: ["%(manylinux_python_version)s"] os: [ubuntu-20.04] steps: @@ -363,7 +363,7 @@ jobs: # We use a regular Python matrix entry to share as much code as possible. strategy: matrix: - python-version: ["3.9"] + python-version: ["%(manylinux_python_version)s"] image: [manylinux2014_x86_64, manylinux2014_i686, manylinux2014_aarch64] steps: diff --git a/config/config-package.py b/config/config-package.py index 5cfa33d..bd4f0de 100755 --- a/config/config-package.py +++ b/config/config-package.py @@ -5,6 +5,7 @@ from shared.git import get_branch_name from shared.git import get_commit_id from shared.git import git_branch +from shared.packages import MANYLINUX_PYTHON_VERSION from shared.path import change_dir from shared.toml_encoder import TomlArraySeparatorEncoderWithNewline import argparse @@ -488,6 +489,7 @@ def tests_yml(self): with_pypy=self.with_pypy, with_macos=self.with_macos, with_windows=self.with_windows, + manylinux_python_version=MANYLINUX_PYTHON_VERSION, ) def manifest_in(self): diff --git a/config/re-enable-actions.py b/config/re-enable-actions.py index f43e192..4905108 100644 --- a/config/re-enable-actions.py +++ b/config/re-enable-actions.py @@ -1,15 +1,13 @@ #!/bin/env python3 from shared.call import call -from shared.packages import list_packages +from shared.packages import ALL_REPOS +from shared.packages import ORG import argparse -import itertools import pathlib -org = 'zopefoundation' -base_url = f'https://github.com/{org}' -base_path = pathlib.Path(__file__).parent -types = ['buildout-recipe', 'c-code', 'pure-python', 'zope-product'] +base_url = f'https://github.com/{ORG}' +BASE_PATH = pathlib.Path(__file__).parent parser = argparse.ArgumentParser( @@ -21,9 +19,6 @@ action='store_true') args = parser.parse_args() -repos = itertools.chain( - *[list_packages(base_path / type / 'packages.txt') - for type in types]) def run_workflow(base_url, org, repo): @@ -38,18 +33,18 @@ def run_workflow(base_url, org, repo): return True -for repo in repos: +for repo in ALL_REPOS: print(repo) wfs = call( - 'gh', 'workflow', 'list', '--all', '-R', f'{org}/{repo}', + 'gh', 'workflow', 'list', '--all', '-R', f'{ORG}/{repo}', capture_output=True).stdout test_line = [x for x in wfs.splitlines() if x.startswith('test')][0] if 'disabled_inactivity' not in test_line: print(' ☑️ already enabled') if args.force_run: - run_workflow(base_url, org, repo) + run_workflow(base_url, ORG, repo) continue test_id = test_line.split()[-1] - call('gh', 'workflow', 'enable', test_id, '-R', f'{org}/{repo}') - if run_workflow(base_url, org, repo): + call('gh', 'workflow', 'enable', test_id, '-R', f'{ORG}/{repo}') + if run_workflow(base_url, ORG, repo): print(' ✅ enabled') diff --git a/config/requirements.txt b/config/requirements.txt index 0d4dadf..b9270b7 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -4,3 +4,4 @@ pyupgrade==3.3.1 toml==0.10.2 tox==4.0.14 zest.releaser==7.2.0 +requests==2.31.0 diff --git a/config/set-branch-protection-rules.py b/config/set-branch-protection-rules.py new file mode 100644 index 0000000..ba748f5 --- /dev/null +++ b/config/set-branch-protection-rules.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +from shared.call import abort +from shared.call import call +from shared.packages import ALL_REPOS +from shared.packages import MANYLINUX_PYTHON_VERSION +from shared.packages import NEWEST_PYTHON_VERSION +from shared.packages import OLDEST_PYTHON_VERSION +from shared.packages import ORG +import argparse +import json +import os +import requests +import tempfile +import tomllib + + +BASE_URL = f'https://raw.githubusercontent.com/{ORG}' +OLDEST_PYTHON = f'py{OLDEST_PYTHON_VERSION.replace(".", "")}' +NEWEST_PYTHON = f'py{NEWEST_PYTHON_VERSION.replace(".", "")}' +DEFAULT_BRANCH = 'master' + + +parser = argparse.ArgumentParser( + description='Set the branch protection rules for all known packages.\n' + 'Prerequsites: `gh auth login`.') +parser.add_argument( + '--I-am-authenticated', + help='If you are authenticated via `gh auth login`, use this required' + ' parameter.', + action='store_true', + required=True) + +args = parser.parse_args() + + +def call_gh( + method, path, *args, capture_output=False, allowed_return_codes=(0, )): + """Call the gh api command.""" + return call( + 'gh', 'api', + '--method', method, + '-H', 'Accept: application/vnd.github+json', + '-H', 'X-GitHub-Api-Version: 2022-11-28', + f'/repos/{ORG}/{repo}/branches/{DEFAULT_BRANCH}/{path}', + *args, capture_output=capture_output, + allowed_return_codes=allowed_return_codes) + + +for repo in ALL_REPOS: + print(repo, end="") + result = call_gh( + 'GET', 'protection/required_pull_request_reviews', + capture_output=True, allowed_return_codes=(0, 1)) + required_pull_request_reviews = None + if result.returncode == 1: + if json.loads(result.stdout)['message'] != "Branch not protected": + # If there is no branch protection we create it later on using the + # PUT call, but if there is another error we show it: + print(result.stdout) + abort(result.returncode) + else: + required_approving_review_count = json.loads( + result.stdout)['required_approving_review_count'] + required_pull_request_reviews = { + 'required_approving_review_count': required_approving_review_count + } + print(f' required reviews={required_approving_review_count}', end='') + + response = requests.get( + f'{BASE_URL}/{repo}/{DEFAULT_BRANCH}/.meta.toml', timeout=30) + meta_toml = tomllib.loads(response.text) + if meta_toml['python']['with-windows']: + required = [] + print('TBI') + import sys + sys.exit() + elif meta_toml['meta']['template'] == 'c-code': + required = [ + f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_aarch64)', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_i686)', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_x86_64)', + f'lint ({MANYLINUX_PYTHON_VERSION}, ubuntu-20.04)', + f'test ({OLDEST_PYTHON_VERSION}, macos-11)', + f'test ({NEWEST_PYTHON_VERSION}, macos-11)', + f'test ({OLDEST_PYTHON_VERSION}, ubuntu-20.04)', + f'test ({NEWEST_PYTHON_VERSION}, ubuntu-20.04)', + ] + if meta_toml['python'].get('with-docs', False): + required.append(f'docs ({MANYLINUX_PYTHON_VERSION}, ubuntu-20.04)') + if meta_toml['python']['with-pypy']: + required.append('test (pypy-3.9, ubuntu-20.04)') + elif meta_toml['meta']['template'] in ('c-code', 'toolkit'): + print('TBI') + import sys + sys.exit() + else: # default for most packages + required = ['coverage', 'lint', OLDEST_PYTHON, NEWEST_PYTHON] + if meta_toml['python'].get('with-docs', False): + required.append('docs') + if meta_toml['python']['with-pypy']: + required.append('pypy3') + + data = { + 'allow_deletions': False, + 'allow_force_pushes': False, + 'allow_fork_syncing': True, + 'lock_branch': False, + 'enforce_admins': None, + 'restrictions': None, + 'required_conversation_resolution': True, + 'required_linear_history': False, + 'required_pull_request_reviews': required_pull_request_reviews, + 'required_status_checks': {'contexts': required, 'strict': False} + } + fd, filename = tempfile.mkstemp('config.json', 'meta', text=True) + try: + file = os.fdopen(fd, 'w') + json.dump(data, file) + file.close() + call_gh( + 'PUT', 'protection', '--input', filename, capture_output=True) + finally: + os.unlink(filename) + print(' ✅') diff --git a/config/shared/packages.py b/config/shared/packages.py index 83b1dcf..6416f78 100644 --- a/config/shared/packages.py +++ b/config/shared/packages.py @@ -1,6 +1,15 @@ +import itertools import pathlib +TYPES = ['buildout-recipe', 'c-code', 'pure-python', 'zope-product', 'toolkit'] +ORG = 'zopefoundation' +BASE_PATH = pathlib.Path(__file__).parent.parent +OLDEST_PYTHON_VERSION = '3.7' +NEWEST_PYTHON_VERSION = '3.11' +MANYLINUX_PYTHON_VERSION = '3.9' + + def list_packages(path: pathlib.Path) -> list: """List the packages in ``path``. @@ -11,3 +20,8 @@ def list_packages(path: pathlib.Path) -> list: for p in path.read_text().split('\n') if p and not p.startswith('#') ] + + +ALL_REPOS = itertools.chain( + *[list_packages(BASE_PATH / type / 'packages.txt') + for type in TYPES]) From 4d3c7df28366672020cc4ae7534345c996beb096 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Sat, 13 Apr 2024 07:07:58 +0200 Subject: [PATCH 2/9] WIP: requirements for c-code and ability for selected repos. --- config/c-code/tests-strategy.j2 | 10 ++-- config/c-code/tests.yml.j2 | 16 +++---- config/config-package.py | 10 +++- config/default/tests.yml.j2 | 18 +++---- config/set-branch-protection-rules.py | 67 +++++++++++++++++---------- config/shared/packages.py | 5 ++ 6 files changed, 78 insertions(+), 48 deletions(-) diff --git a/config/c-code/tests-strategy.j2 b/config/c-code/tests-strategy.j2 index 2a15505..bbf5f9a 100644 --- a/config/c-code/tests-strategy.j2 +++ b/config/c-code/tests-strategy.j2 @@ -3,7 +3,7 @@ matrix: python-version: {% if with_pypy %} - - "pypy-3.10" + - "pypy-%(pypy_version)s" {% endif %} - "3.7" - "3.8" @@ -15,16 +15,16 @@ - "%(future_python_version)s" {% endif %} {% if with_windows %} - os: [ubuntu-20.04, macos-11, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] {% else %} - os: [ubuntu-20.04, macos-11] + os: [ubuntu-latest, macos-latest] {% endif %} {% if with_pypy or gha_additional_exclude %} exclude: {% endif %} {% if with_pypy %} - - os: macos-11 - python-version: "pypy-3.10" + - os: macos-latest + python-version: "pypy-%(pypy_version)s" {% endif %} {% for line in gha_additional_exclude %} %(line)s diff --git a/config/c-code/tests.yml.j2 b/config/c-code/tests.yml.j2 index b32b65b..76bca97 100644 --- a/config/c-code/tests.yml.j2 +++ b/config/c-code/tests.yml.j2 @@ -311,7 +311,7 @@ jobs: coveralls_finish: needs: test - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Coveralls Finished uses: AndreMiras/coveralls-python-action@develop @@ -325,7 +325,7 @@ jobs: strategy: matrix: python-version: ["%(manylinux_python_version)s"] - os: [ubuntu-20.04] + os: [ubuntu-latest] steps: {% include 'tests-cache.j2' %} @@ -351,7 +351,7 @@ jobs: strategy: matrix: python-version: ["%(manylinux_python_version)s"] - os: [ubuntu-20.04] + os: [ubuntu-latest] steps: {% include 'tests-cache.j2' %} @@ -370,13 +370,13 @@ jobs: # python -m pylint --limit-inference-results=1 --rcfile=.pylintrc %(package_name)s -f parseable -r n manylinux: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name # We use a regular Python matrix entry to share as much code as possible. strategy: matrix: python-version: ["%(manylinux_python_version)s"] - image: [manylinux2014_x86_64, manylinux2014_i686, manylinux2014_aarch64] + image: [%(manylinux_x86_64)s, %(manylinux_i686)s, %(manylinux_aarch64)s] steps: {% set cache_key = "${{ runner.os }}-pip_manylinux-${{ matrix.image }}-${{ matrix.python-version }}" %} @@ -385,7 +385,7 @@ jobs: - name: Update pip run: pip install -U pip - name: Build %(package_name)s (x86_64) - if: matrix.image == 'manylinux2014_x86_64' + if: matrix.image == '%(manylinux_x86_64)s' # An alternate way to do this is to run the container directly with a uses: # and then the script runs inside it. That may work better with caching. # See https://github.com/pyca/bcrypt/blob/f6b5ee2eda76d077c531362ac65e16f045cf1f29/.github/workflows/wheel-builder.yml @@ -394,14 +394,14 @@ jobs: run: | bash .manylinux.sh - name: Build %(package_name)s (i686) - if: matrix.image == 'manylinux2014_i686' + if: matrix.image == '%(manylinux_i686)s' env: DOCKER_IMAGE: quay.io/pypa/${{ matrix.image }} PRE_CMD: linux32 run: | bash .manylinux.sh - name: Build %(package_name)s (aarch64) - if: matrix.image == 'manylinux2014_aarch64' + if: matrix.image == '%(manylinux_aarch64)s' env: DOCKER_IMAGE: quay.io/pypa/${{ matrix.image }} run: | diff --git a/config/config-package.py b/config/config-package.py index 1e0a4f8..43fdda6 100755 --- a/config/config-package.py +++ b/config/config-package.py @@ -17,7 +17,12 @@ from shared.git import get_branch_name from shared.git import get_commit_id from shared.git import git_branch +from shared.packages import FUTURE_PYTHON_VERSION +from shared.packages import MANYLINUX_AARCH64 +from shared.packages import MANYLINUX_I686 from shared.packages import MANYLINUX_PYTHON_VERSION +from shared.packages import MANYLINUX_X86_64 +from shared.packages import PYPY_VERSION from shared.path import change_dir import argparse import collections @@ -35,7 +40,6 @@ Generated from: https://github.com/zopefoundation/meta/tree/master/config/{config_type} --> """ -FUTURE_PYTHON_VERSION = "3.13.0-alpha - 3.13.0" DEFAULT = object() @@ -501,6 +505,10 @@ def tests_yml(self): with_macos=self.with_macos, with_windows=self.with_windows, manylinux_python_version=MANYLINUX_PYTHON_VERSION, + manylinux_aarch64=MANYLINUX_AARCH64, + manylinux_i686=MANYLINUX_I686, + manylinux_x86_64=MANYLINUX_X86_64, + pypy_version=PYPY_VERSION, ) def manifest_in(self): diff --git a/config/default/tests.yml.j2 b/config/default/tests.yml.j2 index 92dd6a6..c80a4c5 100644 --- a/config/default/tests.yml.j2 +++ b/config/default/tests.yml.j2 @@ -26,12 +26,12 @@ jobs: fail-fast: false matrix: os: - - ["ubuntu", "ubuntu-20.04"] + - ["ubuntu", "ubuntu-latest"] {% if with_windows %} - ["windows", "windows-latest"] {% endif %} {% if with_macos %} - - ["macos", "macos-11"] + - ["macos", "macos-latest"] {% endif %} config: # [Python version, tox env] @@ -70,18 +70,18 @@ jobs: - { os: ["windows", "windows-latest"], config: ["3.9", "coverage"] } {% endif %} {% if with_macos %} - - { os: ["macos", "macos-11"], config: ["3.9", "release-check"] } - - { os: ["macos", "macos-11"], config: ["3.9", "lint"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "release-check"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "lint"] } {% if with_docs %} - - { os: ["macos", "macos-11"], config: ["3.9", "docs"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "docs"] } {% endif %} - - { os: ["macos", "macos-11"], config: ["3.9", "coverage"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "coverage"] } # macOS/Python 3.11+ is set up for universal2 architecture # which causes build and package selection issues. - - { os: ["macos", "macos-11"], config: ["3.11", "py311"] } - - { os: ["macos", "macos-11"], config: ["3.12", "py312"] } + - { os: ["macos", "macos-latest"], config: ["3.11", "py311"] } + - { os: ["macos", "macos-latest"], config: ["3.12", "py312"] } {% if with_future_python %} - - { os: ["macos", "macos-11"], config: ["%(future_python_version)s", "py313"] } + - { os: ["macos", "macos-latest"], config: ["%(future_python_version)s", "py313"] } {% endif %} {% endif %} {% for line in gha_additional_exclude %} diff --git a/config/set-branch-protection-rules.py b/config/set-branch-protection-rules.py index ba748f5..1587c7c 100644 --- a/config/set-branch-protection-rules.py +++ b/config/set-branch-protection-rules.py @@ -2,10 +2,14 @@ from shared.call import abort from shared.call import call from shared.packages import ALL_REPOS +from shared.packages import MANYLINUX_AARCH64 +from shared.packages import MANYLINUX_I686 from shared.packages import MANYLINUX_PYTHON_VERSION +from shared.packages import MANYLINUX_X86_64 from shared.packages import NEWEST_PYTHON_VERSION from shared.packages import OLDEST_PYTHON_VERSION from shared.packages import ORG +from shared.packages import PYPY_VERSION import argparse import json import os @@ -29,12 +33,18 @@ ' parameter.', action='store_true', required=True) +parser.add_argument( + '-r', '--repos', + help='Run the script only for the given repos instead of all.', + metavar='NAME', nargs='*', default=[]) args = parser.parse_args() +repos = args.repos if args.repos else ALL_REPOS def call_gh( - method, path, *args, capture_output=False, allowed_return_codes=(0, )): + method, path, repo, *args, capture_output=False, + allowed_return_codes=(0, )): """Call the gh api command.""" return call( 'gh', 'api', @@ -46,10 +56,10 @@ def call_gh( allowed_return_codes=allowed_return_codes) -for repo in ALL_REPOS: +for repo in repos: print(repo, end="") result = call_gh( - 'GET', 'protection/required_pull_request_reviews', + 'GET', 'protection/required_pull_request_reviews', repo, capture_output=True, allowed_return_codes=(0, 1)) required_pull_request_reviews = None if result.returncode == 1: @@ -69,35 +79,41 @@ def call_gh( response = requests.get( f'{BASE_URL}/{repo}/{DEFAULT_BRANCH}/.meta.toml', timeout=30) meta_toml = tomllib.loads(response.text) - if meta_toml['python']['with-windows']: - required = [] - print('TBI') - import sys - sys.exit() - elif meta_toml['meta']['template'] == 'c-code': + template = meta_toml['meta']['template'] + with_docs = meta_toml['python'].get('with-docs', False) + with_pypy = meta_toml['python']['with-pypy'] + with_windows = meta_toml['python']['with-windows'] + if template == 'c-code': required = [ - f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_aarch64)', - f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_i686)', - f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_x86_64)', - f'lint ({MANYLINUX_PYTHON_VERSION}, ubuntu-20.04)', - f'test ({OLDEST_PYTHON_VERSION}, macos-11)', - f'test ({NEWEST_PYTHON_VERSION}, macos-11)', - f'test ({OLDEST_PYTHON_VERSION}, ubuntu-20.04)', - f'test ({NEWEST_PYTHON_VERSION}, ubuntu-20.04)', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_AARCH64})', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_I686})', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_X86_64})', + f'lint ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)', + f'test ({OLDEST_PYTHON_VERSION}, macos-latest)', + f'test ({NEWEST_PYTHON_VERSION}, macos-latest)', + f'test ({OLDEST_PYTHON_VERSION}, ubuntu-latest)', + f'test ({NEWEST_PYTHON_VERSION}, ubuntu-latest)', ] - if meta_toml['python'].get('with-docs', False): - required.append(f'docs ({MANYLINUX_PYTHON_VERSION}, ubuntu-20.04)') - if meta_toml['python']['with-pypy']: - required.append('test (pypy-3.9, ubuntu-20.04)') - elif meta_toml['meta']['template'] in ('c-code', 'toolkit'): + if with_docs: + required.append( + f'docs ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)') + if with_pypy: + required.append(f'test (pypy-{PYPY_VERSION}, ubuntu-latest)') + if with_windows: + required.extend([ + f'test ({OLDEST_PYTHON_VERSION}, windows-latest)', + f'test ({NEWEST_PYTHON_VERSION}, windows-latest)', + ]) + elif with_windows: + required = [] print('TBI') import sys sys.exit() else: # default for most packages required = ['coverage', 'lint', OLDEST_PYTHON, NEWEST_PYTHON] - if meta_toml['python'].get('with-docs', False): + if with_docs: required.append('docs') - if meta_toml['python']['with-pypy']: + if with_pypy: required.append('pypy3') data = { @@ -118,7 +134,8 @@ def call_gh( json.dump(data, file) file.close() call_gh( - 'PUT', 'protection', '--input', filename, capture_output=True) + 'PUT', 'protection', repo, '--input', filename, + capture_output=True) finally: os.unlink(filename) print(' ✅') diff --git a/config/shared/packages.py b/config/shared/packages.py index b87fda1..5e380ee 100644 --- a/config/shared/packages.py +++ b/config/shared/packages.py @@ -19,7 +19,12 @@ BASE_PATH = pathlib.Path(__file__).parent.parent OLDEST_PYTHON_VERSION = '3.7' NEWEST_PYTHON_VERSION = '3.11' +FUTURE_PYTHON_VERSION = "3.13.0-alpha - 3.13.0" +PYPY_VERSION = '3.10' MANYLINUX_PYTHON_VERSION = '3.9' +MANYLINUX_AARCH64 = 'manylinux2014_aarch64' +MANYLINUX_I686 = 'manylinux2014_i686' +MANYLINUX_X86_64 = 'manylinux2014_i686' def list_packages(path: pathlib.Path) -> list: From 74c9c31cf6f8bc98674987bc01b2573da23fc468 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Tue, 16 Apr 2024 08:48:15 +0200 Subject: [PATCH 3/9] Add ability to run off a local .meta.toml file. --- config/set-branch-protection-rules.py | 20 +++++++++++++++++--- config/shared/packages.py | 4 ++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/config/set-branch-protection-rules.py b/config/set-branch-protection-rules.py index 1587c7c..2e7c5b2 100644 --- a/config/set-branch-protection-rules.py +++ b/config/set-branch-protection-rules.py @@ -13,6 +13,7 @@ import argparse import json import os +import pathlib import requests import tempfile import tomllib @@ -37,9 +38,17 @@ '-r', '--repos', help='Run the script only for the given repos instead of all.', metavar='NAME', nargs='*', default=[]) +parser.add_argument( + '-m', '--meta', + help='Use this .meta.toml instead the one on `master` of the repos.', + metavar='PATH', default=None, type=pathlib.Path) args = parser.parse_args() repos = args.repos if args.repos else ALL_REPOS +meta_path = args.meta + +if meta_path and len(repos) > 1: + print('--meta can only be used together with a single repos.') def call_gh( @@ -76,9 +85,13 @@ def call_gh( } print(f' required reviews={required_approving_review_count}', end='') - response = requests.get( - f'{BASE_URL}/{repo}/{DEFAULT_BRANCH}/.meta.toml', timeout=30) - meta_toml = tomllib.loads(response.text) + if meta_path is None: + response = requests.get( + f'{BASE_URL}/{repo}/{DEFAULT_BRANCH}/.meta.toml', timeout=30) + meta_toml = tomllib.loads(response.text) + else: + with open(meta_path) as f: + meta_toml = tomllib.loads(f.read()) template = meta_toml['meta']['template'] with_docs = meta_toml['python'].get('with-docs', False) with_pypy = meta_toml['python']['with-pypy'] @@ -93,6 +106,7 @@ def call_gh( f'test ({NEWEST_PYTHON_VERSION}, macos-latest)', f'test ({OLDEST_PYTHON_VERSION}, ubuntu-latest)', f'test ({NEWEST_PYTHON_VERSION}, ubuntu-latest)', + 'coverage/coveralls', ] if with_docs: required.append( diff --git a/config/shared/packages.py b/config/shared/packages.py index 5e380ee..4a3d992 100644 --- a/config/shared/packages.py +++ b/config/shared/packages.py @@ -18,13 +18,13 @@ ORG = 'zopefoundation' BASE_PATH = pathlib.Path(__file__).parent.parent OLDEST_PYTHON_VERSION = '3.7' -NEWEST_PYTHON_VERSION = '3.11' +NEWEST_PYTHON_VERSION = '3.12' FUTURE_PYTHON_VERSION = "3.13.0-alpha - 3.13.0" PYPY_VERSION = '3.10' MANYLINUX_PYTHON_VERSION = '3.9' MANYLINUX_AARCH64 = 'manylinux2014_aarch64' MANYLINUX_I686 = 'manylinux2014_i686' -MANYLINUX_X86_64 = 'manylinux2014_i686' +MANYLINUX_X86_64 = 'manylinux2014_x86_64' def list_packages(path: pathlib.Path) -> list: From 3a81d045ed269bcceb9c0996c0a6da2cc2d599bc Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Tue, 16 Apr 2024 08:50:42 +0200 Subject: [PATCH 4/9] Fix coveralls requirement. --- config/set-branch-protection-rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/set-branch-protection-rules.py b/config/set-branch-protection-rules.py index 2e7c5b2..2ad5fc8 100644 --- a/config/set-branch-protection-rules.py +++ b/config/set-branch-protection-rules.py @@ -106,7 +106,7 @@ def call_gh( f'test ({NEWEST_PYTHON_VERSION}, macos-latest)', f'test ({OLDEST_PYTHON_VERSION}, ubuntu-latest)', f'test ({NEWEST_PYTHON_VERSION}, ubuntu-latest)', - 'coverage/coveralls', + 'coveralls_finish', ] if with_docs: required.append( From e6285ccef1d53188c19332a441869fa12c8a1cb4 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Thu, 18 Apr 2024 08:57:09 +0200 Subject: [PATCH 5/9] Make set_branch_protection_rules usable from python code. --- ...ules.py => set_branch_protection_rules.py} | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) rename config/{set-branch-protection-rules.py => set_branch_protection_rules.py} (77%) diff --git a/config/set-branch-protection-rules.py b/config/set_branch_protection_rules.py similarity index 77% rename from config/set-branch-protection-rules.py rename to config/set_branch_protection_rules.py index 2ad5fc8..fcf0e67 100644 --- a/config/set-branch-protection-rules.py +++ b/config/set_branch_protection_rules.py @@ -25,33 +25,7 @@ DEFAULT_BRANCH = 'master' -parser = argparse.ArgumentParser( - description='Set the branch protection rules for all known packages.\n' - 'Prerequsites: `gh auth login`.') -parser.add_argument( - '--I-am-authenticated', - help='If you are authenticated via `gh auth login`, use this required' - ' parameter.', - action='store_true', - required=True) -parser.add_argument( - '-r', '--repos', - help='Run the script only for the given repos instead of all.', - metavar='NAME', nargs='*', default=[]) -parser.add_argument( - '-m', '--meta', - help='Use this .meta.toml instead the one on `master` of the repos.', - metavar='PATH', default=None, type=pathlib.Path) - -args = parser.parse_args() -repos = args.repos if args.repos else ALL_REPOS -meta_path = args.meta - -if meta_path and len(repos) > 1: - print('--meta can only be used together with a single repos.') - - -def call_gh( +def _call_gh( method, path, repo, *args, capture_output=False, allowed_return_codes=(0, )): """Call the gh api command.""" @@ -65,9 +39,8 @@ def call_gh( allowed_return_codes=allowed_return_codes) -for repo in repos: - print(repo, end="") - result = call_gh( +def set_branch_protection(repo: str, meta_path: pathlib.Path | None) -> bool: + result = _call_gh( 'GET', 'protection/required_pull_request_reviews', repo, capture_output=True, allowed_return_codes=(0, 1)) required_pull_request_reviews = None @@ -83,7 +56,6 @@ def call_gh( required_pull_request_reviews = { 'required_approving_review_count': required_approving_review_count } - print(f' required reviews={required_approving_review_count}', end='') if meta_path is None: response = requests.get( @@ -113,6 +85,7 @@ def call_gh( f'docs ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)') if with_pypy: required.append(f'test (pypy-{PYPY_VERSION}, ubuntu-latest)') + required.append(f'test (pypy-{PYPY_VERSION}, windows-latest)') if with_windows: required.extend([ f'test ({OLDEST_PYTHON_VERSION}, windows-latest)', @@ -147,9 +120,41 @@ def call_gh( file = os.fdopen(fd, 'w') json.dump(data, file) file.close() - call_gh( + _call_gh( 'PUT', 'protection', repo, '--input', filename, capture_output=True) finally: os.unlink(filename) - print(' ✅') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Set the branch protection rules for all known packages.\n' + 'Prerequsites: `gh auth login`.') + parser.add_argument( + '--I-am-authenticated', + help='If you are authenticated via `gh auth login`, use this required' + ' parameter.', + action='store_true', + required=True) + parser.add_argument( + '-r', '--repos', + help='Run the script only for the given repos instead of all.', + metavar='NAME', nargs='*', default=[]) + parser.add_argument( + '-m', '--meta', + help='Use this .meta.toml instead the one on `master` of the repos.', + metavar='PATH', default=None, type=pathlib.Path) + + args = parser.parse_args() + repos = args.repos if args.repos else ALL_REPOS + meta_path = args.meta + + if meta_path and len(repos) > 1: + print('--meta can only be used together with a single repos.') + abort(-1) + + for repo in repos: + print(repo, end="") + set_branch_protection(repo, meta_path) + print(' ✅') From a2af77a555577c12312ee51cfe15be093b7e592d Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Thu, 25 Apr 2024 09:45:22 +0200 Subject: [PATCH 6/9] Integrate set-branch-protection into config-package. --- config/README.rst | 2 +- config/config-package.py | 15 +++++++++++++++ config/set_branch_protection_rules.py | 9 ++++----- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/config/README.rst b/config/README.rst index e5a67c6..fc3f4a2 100644 --- a/config/README.rst +++ b/config/README.rst @@ -334,7 +334,7 @@ updated. Example: "\"${PYBIN}/tox\" -e py", "cd ..", ] - require-cffi = True + require-cffi = true [zest-releaser] options = [ diff --git a/config/config-package.py b/config/config-package.py index a8d9752..b14809c 100755 --- a/config/config-package.py +++ b/config/config-package.py @@ -12,6 +12,7 @@ # ############################################################################## from functools import cached_property +from set_branch_protection_rules import set_branch_protection from shared.call import abort from shared.call import call from shared.git import get_branch_name @@ -635,6 +636,20 @@ def configure(self): call('git', 'push', '--set-upstream', 'origin', self.branch_name) print() + print('If you are an admin and are logged in via `gh auth login`') + print('update branch protection rules? (y/N)?', end=' ') + if input().lower() == 'y': + remote_url = call( + 'git', 'config', '--get', 'remote.origin.url', + capture_output=True).stdout.strip() + package_name = remote_url.rsplit('/')[-1].removesuffix('.git') + success = set_branch_protection( + package_name, self.path / '.meta.toml') + if success: + print('Successfully updated branch protection rules.') + else: + abort(-1) + print() print('If everything went fine up to here:') if updating: print('Updated the previously created PR.') diff --git a/config/set_branch_protection_rules.py b/config/set_branch_protection_rules.py index fcf0e67..1413cb7 100644 --- a/config/set_branch_protection_rules.py +++ b/config/set_branch_protection_rules.py @@ -26,7 +26,7 @@ def _call_gh( - method, path, repo, *args, capture_output=False, + method, path, repo, *args, capture_output=True, allowed_return_codes=(0, )): """Call the gh api command.""" return call( @@ -42,7 +42,7 @@ def _call_gh( def set_branch_protection(repo: str, meta_path: pathlib.Path | None) -> bool: result = _call_gh( 'GET', 'protection/required_pull_request_reviews', repo, - capture_output=True, allowed_return_codes=(0, 1)) + allowed_return_codes=(0, 1)) required_pull_request_reviews = None if result.returncode == 1: if json.loads(result.stdout)['message'] != "Branch not protected": @@ -120,11 +120,10 @@ def set_branch_protection(repo: str, meta_path: pathlib.Path | None) -> bool: file = os.fdopen(fd, 'w') json.dump(data, file) file.close() - _call_gh( - 'PUT', 'protection', repo, '--input', filename, - capture_output=True) + _call_gh('PUT', 'protection', repo, '--input', filename) finally: os.unlink(filename) + return True if __name__ == '__main__': From c0bfa85b9df330d7571221c3dfb1cf44166bb80c Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Fri, 26 Apr 2024 09:10:57 +0200 Subject: [PATCH 7/9] - the GHA runner macos-latest has no Python 3.7 --- config/c-code/tests-strategy.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/c-code/tests-strategy.j2 b/config/c-code/tests-strategy.j2 index bbf5f9a..a06c70a 100644 --- a/config/c-code/tests-strategy.j2 +++ b/config/c-code/tests-strategy.j2 @@ -22,6 +22,8 @@ {% if with_pypy or gha_additional_exclude %} exclude: {% endif %} + - os: macos-latest + python-version: "3.7" {% if with_pypy %} - os: macos-latest python-version: "pypy-%(pypy_version)s" @@ -29,3 +31,6 @@ {% for line in gha_additional_exclude %} %(line)s {% endfor %} + include: + - os: macos-12 + python-version: "3.7" From 7713fd8644f6644d4373a285849f0cf5b3090711 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Fri, 26 Apr 2024 09:30:53 +0200 Subject: [PATCH 8/9] Implement branch protection rules for with_windows. Fix rule for mac-os. --- .pre-commit-config.yaml | 8 ++++---- config/set_branch_protection_rules.py | 22 +++++++++++++++++----- config/shared/call.py | 10 +++++++++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 318910e..fb4e246 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,16 +10,16 @@ repos: - id: check-yaml - id: debug-statements language_version: python3 + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v2.0.4 + hooks: + - id: autopep8 - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: - id: flake8 language_version: python3 additional_dependencies: [flake8-typing-imports==1.15.0] - - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.4 - hooks: - - id: autopep8 - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: diff --git a/config/set_branch_protection_rules.py b/config/set_branch_protection_rules.py index 1413cb7..2de7025 100644 --- a/config/set_branch_protection_rules.py +++ b/config/set_branch_protection_rules.py @@ -74,7 +74,7 @@ def set_branch_protection(repo: str, meta_path: pathlib.Path | None) -> bool: f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_I686})', f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_X86_64})', f'lint ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)', - f'test ({OLDEST_PYTHON_VERSION}, macos-latest)', + f'test ({OLDEST_PYTHON_VERSION}, macos-12)', f'test ({NEWEST_PYTHON_VERSION}, macos-latest)', f'test ({OLDEST_PYTHON_VERSION}, ubuntu-latest)', f'test ({NEWEST_PYTHON_VERSION}, ubuntu-latest)', @@ -92,10 +92,22 @@ def set_branch_protection(repo: str, meta_path: pathlib.Path | None) -> bool: f'test ({NEWEST_PYTHON_VERSION}, windows-latest)', ]) elif with_windows: - required = [] - print('TBI') - import sys - sys.exit() + required = [ + 'coverage/coveralls', + 'ubuntu-lint', + 'ubuntu-coverage', + f'ubuntu-{OLDEST_PYTHON}', + f'ubuntu-{NEWEST_PYTHON}', + f'windows-{OLDEST_PYTHON}', + f'windows-{NEWEST_PYTHON}', + ] + if with_pypy: + required.extend([ + 'ubuntu-pypy3', + 'windows-pypy3', + ]) + if with_docs: + required.append('ubuntu-docs') else: # default for most packages required = ['coverage', 'lint', OLDEST_PYTHON, NEWEST_PYTHON] if with_docs: diff --git a/config/shared/call.py b/config/shared/call.py index fd16b5c..c2752f7 100644 --- a/config/shared/call.py +++ b/config/shared/call.py @@ -12,6 +12,7 @@ ############################################################################## import subprocess import sys +import textwrap def abort(exitcode): @@ -36,5 +37,12 @@ def call(*args, capture_output=False, cwd=None, allowed_return_codes=(0, )): result = subprocess.run( args, capture_output=capture_output, text=True, cwd=cwd) if result.returncode not in allowed_return_codes: - abort(result.returncode) + if capture_output: + abort_text = textwrap.defent(f''' + error code: {result.returncode} + stderr: {result.stderr} + stdout: {result.stdout}''') + else: + abort_text = result.returncode + abort(abort_text) return result From 352ee9f410b4878fa0e9c08fa42dde4a590f1bcf Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Mon, 6 May 2024 08:37:21 +0200 Subject: [PATCH 9/9] Update config/shared/call.py Co-authored-by: Jens Vagelpohl --- config/shared/call.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/shared/call.py b/config/shared/call.py index c2752f7..be5f6df 100644 --- a/config/shared/call.py +++ b/config/shared/call.py @@ -38,7 +38,7 @@ def call(*args, capture_output=False, cwd=None, allowed_return_codes=(0, )): args, capture_output=capture_output, text=True, cwd=cwd) if result.returncode not in allowed_return_codes: if capture_output: - abort_text = textwrap.defent(f''' + abort_text = textwrap.dedent(f''' error code: {result.returncode} stderr: {result.stderr} stdout: {result.stdout}''')