From 4d3c7df28366672020cc4ae7534345c996beb096 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Sat, 13 Apr 2024 07:07:58 +0200 Subject: [PATCH] WIP: requirements for c-code and ability for selected repos. --- config/c-code/tests-strategy.j2 | 10 ++-- config/c-code/tests.yml.j2 | 16 +++---- config/config-package.py | 10 +++- config/default/tests.yml.j2 | 18 +++---- config/set-branch-protection-rules.py | 67 +++++++++++++++++---------- config/shared/packages.py | 5 ++ 6 files changed, 78 insertions(+), 48 deletions(-) diff --git a/config/c-code/tests-strategy.j2 b/config/c-code/tests-strategy.j2 index 2a15505..bbf5f9a 100644 --- a/config/c-code/tests-strategy.j2 +++ b/config/c-code/tests-strategy.j2 @@ -3,7 +3,7 @@ matrix: python-version: {% if with_pypy %} - - "pypy-3.10" + - "pypy-%(pypy_version)s" {% endif %} - "3.7" - "3.8" @@ -15,16 +15,16 @@ - "%(future_python_version)s" {% endif %} {% if with_windows %} - os: [ubuntu-20.04, macos-11, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] {% else %} - os: [ubuntu-20.04, macos-11] + os: [ubuntu-latest, macos-latest] {% endif %} {% if with_pypy or gha_additional_exclude %} exclude: {% endif %} {% if with_pypy %} - - os: macos-11 - python-version: "pypy-3.10" + - os: macos-latest + python-version: "pypy-%(pypy_version)s" {% endif %} {% for line in gha_additional_exclude %} %(line)s diff --git a/config/c-code/tests.yml.j2 b/config/c-code/tests.yml.j2 index b32b65b..76bca97 100644 --- a/config/c-code/tests.yml.j2 +++ b/config/c-code/tests.yml.j2 @@ -311,7 +311,7 @@ jobs: coveralls_finish: needs: test - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Coveralls Finished uses: AndreMiras/coveralls-python-action@develop @@ -325,7 +325,7 @@ jobs: strategy: matrix: python-version: ["%(manylinux_python_version)s"] - os: [ubuntu-20.04] + os: [ubuntu-latest] steps: {% include 'tests-cache.j2' %} @@ -351,7 +351,7 @@ jobs: strategy: matrix: python-version: ["%(manylinux_python_version)s"] - os: [ubuntu-20.04] + os: [ubuntu-latest] steps: {% include 'tests-cache.j2' %} @@ -370,13 +370,13 @@ jobs: # python -m pylint --limit-inference-results=1 --rcfile=.pylintrc %(package_name)s -f parseable -r n manylinux: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name # We use a regular Python matrix entry to share as much code as possible. strategy: matrix: python-version: ["%(manylinux_python_version)s"] - image: [manylinux2014_x86_64, manylinux2014_i686, manylinux2014_aarch64] + image: [%(manylinux_x86_64)s, %(manylinux_i686)s, %(manylinux_aarch64)s] steps: {% set cache_key = "${{ runner.os }}-pip_manylinux-${{ matrix.image }}-${{ matrix.python-version }}" %} @@ -385,7 +385,7 @@ jobs: - name: Update pip run: pip install -U pip - name: Build %(package_name)s (x86_64) - if: matrix.image == 'manylinux2014_x86_64' + if: matrix.image == '%(manylinux_x86_64)s' # An alternate way to do this is to run the container directly with a uses: # and then the script runs inside it. That may work better with caching. # See https://github.com/pyca/bcrypt/blob/f6b5ee2eda76d077c531362ac65e16f045cf1f29/.github/workflows/wheel-builder.yml @@ -394,14 +394,14 @@ jobs: run: | bash .manylinux.sh - name: Build %(package_name)s (i686) - if: matrix.image == 'manylinux2014_i686' + if: matrix.image == '%(manylinux_i686)s' env: DOCKER_IMAGE: quay.io/pypa/${{ matrix.image }} PRE_CMD: linux32 run: | bash .manylinux.sh - name: Build %(package_name)s (aarch64) - if: matrix.image == 'manylinux2014_aarch64' + if: matrix.image == '%(manylinux_aarch64)s' env: DOCKER_IMAGE: quay.io/pypa/${{ matrix.image }} run: | diff --git a/config/config-package.py b/config/config-package.py index 1e0a4f8..43fdda6 100755 --- a/config/config-package.py +++ b/config/config-package.py @@ -17,7 +17,12 @@ from shared.git import get_branch_name from shared.git import get_commit_id from shared.git import git_branch +from shared.packages import FUTURE_PYTHON_VERSION +from shared.packages import MANYLINUX_AARCH64 +from shared.packages import MANYLINUX_I686 from shared.packages import MANYLINUX_PYTHON_VERSION +from shared.packages import MANYLINUX_X86_64 +from shared.packages import PYPY_VERSION from shared.path import change_dir import argparse import collections @@ -35,7 +40,6 @@ Generated from: https://github.com/zopefoundation/meta/tree/master/config/{config_type} --> """ -FUTURE_PYTHON_VERSION = "3.13.0-alpha - 3.13.0" DEFAULT = object() @@ -501,6 +505,10 @@ def tests_yml(self): with_macos=self.with_macos, with_windows=self.with_windows, manylinux_python_version=MANYLINUX_PYTHON_VERSION, + manylinux_aarch64=MANYLINUX_AARCH64, + manylinux_i686=MANYLINUX_I686, + manylinux_x86_64=MANYLINUX_X86_64, + pypy_version=PYPY_VERSION, ) def manifest_in(self): diff --git a/config/default/tests.yml.j2 b/config/default/tests.yml.j2 index 92dd6a6..c80a4c5 100644 --- a/config/default/tests.yml.j2 +++ b/config/default/tests.yml.j2 @@ -26,12 +26,12 @@ jobs: fail-fast: false matrix: os: - - ["ubuntu", "ubuntu-20.04"] + - ["ubuntu", "ubuntu-latest"] {% if with_windows %} - ["windows", "windows-latest"] {% endif %} {% if with_macos %} - - ["macos", "macos-11"] + - ["macos", "macos-latest"] {% endif %} config: # [Python version, tox env] @@ -70,18 +70,18 @@ jobs: - { os: ["windows", "windows-latest"], config: ["3.9", "coverage"] } {% endif %} {% if with_macos %} - - { os: ["macos", "macos-11"], config: ["3.9", "release-check"] } - - { os: ["macos", "macos-11"], config: ["3.9", "lint"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "release-check"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "lint"] } {% if with_docs %} - - { os: ["macos", "macos-11"], config: ["3.9", "docs"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "docs"] } {% endif %} - - { os: ["macos", "macos-11"], config: ["3.9", "coverage"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "coverage"] } # macOS/Python 3.11+ is set up for universal2 architecture # which causes build and package selection issues. - - { os: ["macos", "macos-11"], config: ["3.11", "py311"] } - - { os: ["macos", "macos-11"], config: ["3.12", "py312"] } + - { os: ["macos", "macos-latest"], config: ["3.11", "py311"] } + - { os: ["macos", "macos-latest"], config: ["3.12", "py312"] } {% if with_future_python %} - - { os: ["macos", "macos-11"], config: ["%(future_python_version)s", "py313"] } + - { os: ["macos", "macos-latest"], config: ["%(future_python_version)s", "py313"] } {% endif %} {% endif %} {% for line in gha_additional_exclude %} diff --git a/config/set-branch-protection-rules.py b/config/set-branch-protection-rules.py index ba748f5..1587c7c 100644 --- a/config/set-branch-protection-rules.py +++ b/config/set-branch-protection-rules.py @@ -2,10 +2,14 @@ from shared.call import abort from shared.call import call from shared.packages import ALL_REPOS +from shared.packages import MANYLINUX_AARCH64 +from shared.packages import MANYLINUX_I686 from shared.packages import MANYLINUX_PYTHON_VERSION +from shared.packages import MANYLINUX_X86_64 from shared.packages import NEWEST_PYTHON_VERSION from shared.packages import OLDEST_PYTHON_VERSION from shared.packages import ORG +from shared.packages import PYPY_VERSION import argparse import json import os @@ -29,12 +33,18 @@ ' parameter.', action='store_true', required=True) +parser.add_argument( + '-r', '--repos', + help='Run the script only for the given repos instead of all.', + metavar='NAME', nargs='*', default=[]) args = parser.parse_args() +repos = args.repos if args.repos else ALL_REPOS def call_gh( - method, path, *args, capture_output=False, allowed_return_codes=(0, )): + method, path, repo, *args, capture_output=False, + allowed_return_codes=(0, )): """Call the gh api command.""" return call( 'gh', 'api', @@ -46,10 +56,10 @@ def call_gh( allowed_return_codes=allowed_return_codes) -for repo in ALL_REPOS: +for repo in repos: print(repo, end="") result = call_gh( - 'GET', 'protection/required_pull_request_reviews', + 'GET', 'protection/required_pull_request_reviews', repo, capture_output=True, allowed_return_codes=(0, 1)) required_pull_request_reviews = None if result.returncode == 1: @@ -69,35 +79,41 @@ def call_gh( response = requests.get( f'{BASE_URL}/{repo}/{DEFAULT_BRANCH}/.meta.toml', timeout=30) meta_toml = tomllib.loads(response.text) - if meta_toml['python']['with-windows']: - required = [] - print('TBI') - import sys - sys.exit() - elif meta_toml['meta']['template'] == 'c-code': + template = meta_toml['meta']['template'] + with_docs = meta_toml['python'].get('with-docs', False) + with_pypy = meta_toml['python']['with-pypy'] + with_windows = meta_toml['python']['with-windows'] + if template == 'c-code': required = [ - f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_aarch64)', - f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_i686)', - f'manylinux ({MANYLINUX_PYTHON_VERSION}, manylinux2014_x86_64)', - f'lint ({MANYLINUX_PYTHON_VERSION}, ubuntu-20.04)', - f'test ({OLDEST_PYTHON_VERSION}, macos-11)', - f'test ({NEWEST_PYTHON_VERSION}, macos-11)', - f'test ({OLDEST_PYTHON_VERSION}, ubuntu-20.04)', - f'test ({NEWEST_PYTHON_VERSION}, ubuntu-20.04)', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_AARCH64})', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_I686})', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_X86_64})', + f'lint ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)', + f'test ({OLDEST_PYTHON_VERSION}, macos-latest)', + f'test ({NEWEST_PYTHON_VERSION}, macos-latest)', + f'test ({OLDEST_PYTHON_VERSION}, ubuntu-latest)', + f'test ({NEWEST_PYTHON_VERSION}, ubuntu-latest)', ] - if meta_toml['python'].get('with-docs', False): - required.append(f'docs ({MANYLINUX_PYTHON_VERSION}, ubuntu-20.04)') - if meta_toml['python']['with-pypy']: - required.append('test (pypy-3.9, ubuntu-20.04)') - elif meta_toml['meta']['template'] in ('c-code', 'toolkit'): + if with_docs: + required.append( + f'docs ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)') + if with_pypy: + required.append(f'test (pypy-{PYPY_VERSION}, ubuntu-latest)') + if with_windows: + required.extend([ + f'test ({OLDEST_PYTHON_VERSION}, windows-latest)', + f'test ({NEWEST_PYTHON_VERSION}, windows-latest)', + ]) + elif with_windows: + required = [] print('TBI') import sys sys.exit() else: # default for most packages required = ['coverage', 'lint', OLDEST_PYTHON, NEWEST_PYTHON] - if meta_toml['python'].get('with-docs', False): + if with_docs: required.append('docs') - if meta_toml['python']['with-pypy']: + if with_pypy: required.append('pypy3') data = { @@ -118,7 +134,8 @@ def call_gh( json.dump(data, file) file.close() call_gh( - 'PUT', 'protection', '--input', filename, capture_output=True) + 'PUT', 'protection', repo, '--input', filename, + capture_output=True) finally: os.unlink(filename) print(' ✅') diff --git a/config/shared/packages.py b/config/shared/packages.py index b87fda1..5e380ee 100644 --- a/config/shared/packages.py +++ b/config/shared/packages.py @@ -19,7 +19,12 @@ BASE_PATH = pathlib.Path(__file__).parent.parent OLDEST_PYTHON_VERSION = '3.7' NEWEST_PYTHON_VERSION = '3.11' +FUTURE_PYTHON_VERSION = "3.13.0-alpha - 3.13.0" +PYPY_VERSION = '3.10' MANYLINUX_PYTHON_VERSION = '3.9' +MANYLINUX_AARCH64 = 'manylinux2014_aarch64' +MANYLINUX_I686 = 'manylinux2014_i686' +MANYLINUX_X86_64 = 'manylinux2014_i686' def list_packages(path: pathlib.Path) -> list: