Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
sentrivana committed Jan 8, 2025
1 parent 3407856 commit e08255b
Showing 1 changed file with 108 additions and 29 deletions.
137 changes: 108 additions & 29 deletions scripts/populate_tox/populate_tox.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
import configparser
import functools
import time
from datetime import datetime, timedelta
from pathlib import Path

import requests

from ..split_tox_gh_actions.split_tox_gh_actions import GROUPS

print(GROUPS)

# Only consider package versions going back this far
CUTOFF = datetime.now() - timedelta(days=365 * 3)
CUTOFF = datetime.now() - timedelta(days=365 * 5)
LOWEST_SUPPORTED_PY_VERSION = "3.6"

TOX_FILE = Path(__file__).resolve().parent.parent.parent / "tox.ini"

PYPI_PROJECT_URL = "https://pypi.python.org/pypi/{project}/json"
PYPI_VERSION_URL = "https://pypi.python.org/pypi/{project}/{version}/json"

CLASSIFIER_PREFIX = "Programming Language :: Python :: "

EXCLUDE = {
"common",
}
Expand All @@ -27,7 +27,7 @@

@functools.total_ordering
class Version:
def __init__(self, version, metadata):
def __init__(self, version, metadata=None):
self.raw = version
self.metadata = metadata

Expand All @@ -36,10 +36,14 @@ def __init__(self, version, metadata):
self.patch = None
self.parsed = None

self.python_versions = []

try:
parsed = version.split(".")
if parsed[2].isnumeric():
if len(parsed) == 3 and parsed[2].isnumeric():
self.major, self.minor, self.patch = (int(p) for p in parsed)
elif len(parsed) == 2 and parsed[1].isnumeric():
self.major, self.minor = (int(p) for p in parsed)
except Exception:
# This will fail for e.g. prereleases, but we don't care about those
# for now
Expand Down Expand Up @@ -97,50 +101,125 @@ def parse_tox():
print(f"ERROR reading line {line}")


def fetch_metadata(package):
def fetch_package(package: str) -> dict:
"""Fetch package metadata from PYPI."""
url = PYPI_PROJECT_URL.format(project=package)
pypi_data = requests.get(url)

if pypi_data.status_code != 200:
print(f"{package} not found")

import pprint

pprint.pprint(package)
pprint.pprint(pypi_data.json())
return pypi_data.json()


def parse_metadata(data):
package = data["info"]["name"]
def get_releases(pypi_data: dict) -> list[Version]:
package = pypi_data["info"]["name"]

majors = {}
versions = []

for release, metadata in pypi_data["releases"].items():
if not metadata:
continue

for release, metadata in data["releases"].items():
meta = metadata[0]
if datetime.fromisoformat(meta["upload_time"]) < CUTOFF:
continue

version = Version(release, meta)
if not version.valid:
print(f"Failed to parse version {release} of package {package}")
print(
f"Failed to parse version {release} of package {package}. Ignoring..."
)
continue

if version.major not in majors:
# 0 -> [min 0.x version, max 0.x version]
majors[version.major] = [version, version]
continue
versions.append(version)

if version < majors[version.major][0]:
majors[version.major][0] = version
if version > majors[version.major][1]:
majors[version.major][1] = version
return sorted(versions)


def pick_releases_to_test(releases: list[Version]) -> list[Version]:
indexes = [
0, # oldest version younger than CUTOFF
len(releases) // 3,
len(releases) // 3 * 2,
-1, # latest
]
return [releases[i] for i in indexes]


def fetch_release(package: str, version: Version) -> dict:
url = PYPI_VERSION_URL.format(project=package, version=version)
pypi_data = requests.get(url)

if pypi_data.status_code != 200:
print(f"{package} not found")

return pypi_data.json()


def determine_python_versions(
package: str, version: Version, pypi_data: dict
) -> list[str]:
try:
classifiers = pypi_data["info"]["classifiers"]
except (AttributeError, IndexError):
print(f"{package} {version} has no classifiers")
return []

python_versions = []
for classifier in classifiers:
if classifier.startswith(CLASSIFIER_PREFIX):
python_version = classifier[len(CLASSIFIER_PREFIX) :]
if "." in python_version:
# we don't care about stuff like
# Programming Language :: Python :: 3 :: Only
# Programming Language :: Python :: 3
# etc., we're only interested in specific versions like 3.13
python_versions.append(python_version)

python_versions = [
version
for version in python_versions
if Version(version) >= Version(LOWEST_SUPPORTED_PY_VERSION)
]

return python_versions


def write_tox_file(package, versions):
for version in versions:
print(
"{python_versions}-{package}-v{version}".format(
python_versions=",".join([f"py{v}" for v in version.python_versions]),
package=package,
version=version,
)
)

print()

for version in versions:
print(f"{package}-v{version}: {package}=={version.raw}")


if __name__ == "__main__":
for package in ("celery", "django"):
pypi_data = fetch_package(package)
releases = get_releases(pypi_data)
test_releases = pick_releases_to_test(releases)
for release in test_releases:
release_pypi_data = fetch_release(package, release)
release.python_versions = determine_python_versions(
package, release, release_pypi_data
)
# XXX if no supported python versions -> delete

print(release, "not too old", meta["upload_time"])
print(release, " on ", release.python_versions)
time.sleep(0.1)

return majors
print(releases)
print(test_releases)

write_tox_file(package, test_releases)

print(parse_tox())
print(packages)
print(parse_metadata(fetch_metadata("celery")))
print(parse_tox())

0 comments on commit e08255b

Please sign in to comment.