From 642ab757f58139b7fe91221a2c6e10414d197711 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 15 Mar 2024 18:55:36 -0400 Subject: [PATCH] cargo: Remove duplicated type definitions Typing of Cargo manifest is really hard and requires to duplicate every fields in a TypedDict, with odd syntax to allow fields with dash. In the end it does not even add any type safety because we cast the raw toml object into whatever we want with no verifications. Instead, disable type checking of raw manifest by using T.Any in `from_raw()` class method that converts untyped raw object into typed python class. While at it, also remove incomplete attempt at supporting workspaces, a follow up PR will restore that functionality. --- mesonbuild/cargo/interpreter.py | 135 ++++++------------- mesonbuild/cargo/manifest.py | 227 -------------------------------- 2 files changed, 43 insertions(+), 319 deletions(-) delete mode 100644 mesonbuild/cargo/manifest.py diff --git a/mesonbuild/cargo/interpreter.py b/mesonbuild/cargo/interpreter.py index f1ed23953850..094cb5101ba9 100644 --- a/mesonbuild/cargo/interpreter.py +++ b/mesonbuild/cargo/interpreter.py @@ -27,12 +27,15 @@ if T.TYPE_CHECKING: from types import ModuleType + from typing_extensions import Literal - from . import manifest from .. import mparser from ..environment import Environment from ..coredata import KeyedOptionDictType + EDITION = Literal['2015', '2018', '2021'] + CRATE_TYPE = Literal['bin', 'lib', 'dylib', 'staticlib', 'cdylib', 'rlib', 'proc-macro'] + # tomllib is present in python 3.11, before that it is a pypi module called tomli, # we try to import tomllib, then tomli, # TODO: add a fallback to toml2json? @@ -52,7 +55,7 @@ toml2json = shutil.which('toml2json') -def load_toml(filename: str) -> T.Dict[object, object]: +def load_toml(filename: str) -> T.Any: if tomllib: with open(filename, 'rb') as f: raw = tomllib.load(f) @@ -81,35 +84,17 @@ def fixup_meson_varname(name: str) -> str: return name.replace('-', '_') -# Pylance can figure out that these do not, in fact, overlap, but mypy can't -@T.overload -def _fixup_raw_mappings(d: manifest.BuildTarget) -> manifest.FixedBuildTarget: ... # type: ignore - -@T.overload -def _fixup_raw_mappings(d: manifest.LibTarget) -> manifest.FixedLibTarget: ... # type: ignore - -@T.overload -def _fixup_raw_mappings(d: manifest.Dependency) -> manifest.FixedDependency: ... - -def _fixup_raw_mappings(d: T.Union[manifest.BuildTarget, manifest.LibTarget, manifest.Dependency] - ) -> T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, - manifest.FixedDependency]: +def _fixup_raw_mappings(d: T.Any) -> T.Dict[str, T.Any]: """Fixup raw cargo mappings to ones more suitable for python to consume. This does the following: * replaces any `-` with `_`, cargo likes the former, but python dicts make keys with `-` in them awkward to work with - * Convert Dependndency versions from the cargo format to something meson - understands :param d: The mapping to fix :return: the fixed string """ - raw = {fixup_meson_varname(k): v for k, v in d.items()} - if 'version' in raw: - assert isinstance(raw['version'], str), 'for mypy' - raw['version'] = version.convert(raw['version']) - return T.cast('T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, manifest.FixedDependency]', raw) + return {fixup_meson_varname(k): v for k, v in d.items()} @dataclasses.dataclass @@ -122,7 +107,7 @@ class Package: description: T.Optional[str] = None resolver: T.Optional[str] = None authors: T.List[str] = dataclasses.field(default_factory=list) - edition: manifest.EDITION = '2015' + edition: EDITION = '2015' rust_version: T.Optional[str] = None documentation: T.Optional[str] = None readme: T.Optional[str] = None @@ -181,18 +166,19 @@ def __post_init__(self, name: str) -> None: raise MesonException(f'Cannot determine minimum API version from {self.version}.') @classmethod - def from_raw(cls, name: str, raw: manifest.DependencyV) -> Dependency: + def from_raw(cls, name: str, raw: T.Any) -> Dependency: """Create a dependency from a raw cargo dictionary""" if isinstance(raw, str): return cls(name, version.convert(raw)) - return cls(name, **_fixup_raw_mappings(raw)) + v = version.convert(raw.pop('version', '')) + return cls(name, v, **_fixup_raw_mappings(raw)) @dataclasses.dataclass class BuildTarget: name: str - crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) path: dataclasses.InitVar[T.Optional[str]] = None # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-test-field @@ -212,7 +198,7 @@ class BuildTarget: doc: bool = False harness: bool = True - edition: manifest.EDITION = '2015' + edition: EDITION = '2015' required_features: T.List[str] = dataclasses.field(default_factory=list) plugin: bool = False @@ -226,7 +212,7 @@ class Library(BuildTarget): doc: bool = True path: str = os.path.join('src', 'lib.rs') proc_macro: bool = False - crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) doc_scrape_examples: bool = True @@ -259,7 +245,7 @@ class Example(BuildTarget): """Representation of a Cargo Example Entry.""" - crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) + crate_type: T.List[CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) @dataclasses.dataclass @@ -289,78 +275,43 @@ class Manifest: features: T.Dict[str, T.List[str]] target: T.Dict[str, T.Dict[str, Dependency]] subdir: str - path: str = '' def __post_init__(self) -> None: self.features.setdefault('default', []) - -def _convert_manifest(raw_manifest: manifest.Manifest, subdir: str, path: str = '') -> Manifest: - # This cast is a bit of a hack to deal with proc-macro - lib = _fixup_raw_mappings(raw_manifest.get('lib', {})) - - # We need to set the name field if it's not set manually, - # including if other fields are set in the lib section - lib.setdefault('name', raw_manifest['package']['name']) - - pkg = T.cast('manifest.FixedPackage', - {fixup_meson_varname(k): v for k, v in raw_manifest['package'].items()}) - - return Manifest( - Package(**pkg), - {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('dependencies', {}).items()}, - {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('dev-dependencies', {}).items()}, - {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('build-dependencies', {}).items()}, - Library(**lib), - [Binary(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bin', {})], - [Test(**_fixup_raw_mappings(b)) for b in raw_manifest.get('test', {})], - [Benchmark(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bench', {})], - [Example(**_fixup_raw_mappings(b)) for b in raw_manifest.get('example', {})], - raw_manifest.get('features', {}), - {k: {k2: Dependency.from_raw(k2, v2) for k2, v2 in v.get('dependencies', {}).items()} - for k, v in raw_manifest.get('target', {}).items()}, - subdir, - path, - ) + @classmethod + def from_raw(cls, raw_manifest: T.Any, subdir: str) -> Manifest: + # We need to set the name field if it's not set manually, + # including if other fields are set in the lib section + lib = raw_manifest.get('lib', {}) + lib.setdefault('name', raw_manifest['package']['name']) + + return Manifest( + Package(**_fixup_raw_mappings(raw_manifest['package'])), + {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('dependencies', {}).items()}, + {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('dev-dependencies', {}).items()}, + {k: Dependency.from_raw(k, v) for k, v in raw_manifest.get('build-dependencies', {}).items()}, + Library(**_fixup_raw_mappings(lib)), + [Binary(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bin', {})], + [Test(**_fixup_raw_mappings(b)) for b in raw_manifest.get('test', {})], + [Benchmark(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bench', {})], + [Example(**_fixup_raw_mappings(b)) for b in raw_manifest.get('example', {})], + raw_manifest.get('features', {}), + {k: {k2: Dependency.from_raw(k2, v2) for k2, v2 in v.get('dependencies', {}).items()} + for k, v in raw_manifest.get('target', {}).items()}, + subdir, + ) def _load_manifests(subdir: str) -> T.Dict[str, Manifest]: filename = os.path.join(subdir, 'Cargo.toml') - raw = load_toml(filename) + raw_manifest = load_toml(filename) manifests: T.Dict[str, Manifest] = {} - raw_manifest: T.Union[manifest.Manifest, manifest.VirtualManifest] - if 'package' in raw: - raw_manifest = T.cast('manifest.Manifest', raw) - manifest_ = _convert_manifest(raw_manifest, subdir) + if 'package' in raw_manifest: + manifest_ = Manifest.from_raw(raw_manifest, subdir) manifests[manifest_.package.name] = manifest_ - else: - raw_manifest = T.cast('manifest.VirtualManifest', raw) - - if 'workspace' in raw_manifest: - # XXX: need to verify that python glob and cargo globbing are the - # same and probably write a glob implementation. Blarg - - # We need to chdir here to make the glob work correctly - pwd = os.getcwd() - os.chdir(subdir) - members: T.Iterable[str] - try: - members = itertools.chain.from_iterable( - glob.glob(m) for m in raw_manifest['workspace']['members']) - finally: - os.chdir(pwd) - if 'exclude' in raw_manifest['workspace']: - members = (x for x in members if x not in raw_manifest['workspace']['exclude']) - - for m in members: - filename = os.path.join(subdir, m, 'Cargo.toml') - raw = load_toml(filename) - - raw_manifest = T.cast('manifest.Manifest', raw) - man = _convert_manifest(raw_manifest, subdir, m) - manifests[man.package.name] = man return manifests @@ -618,7 +569,7 @@ def _create_meson_subdir(cargo: Manifest, build: builder.Builder) -> T.List[mpar ] -def _create_lib(cargo: Manifest, build: builder.Builder, crate_type: manifest.CRATE_TYPE) -> T.List[mparser.BaseNode]: +def _create_lib(cargo: Manifest, build: builder.Builder, crate_type: CRATE_TYPE) -> T.List[mparser.BaseNode]: dependencies: T.List[mparser.BaseNode] = [] dependency_map: T.Dict[mparser.BaseNode, mparser.BaseNode] = {} for name, dep in cargo.dependencies.items(): @@ -708,7 +659,7 @@ def interpret(subp_name: str, subdir: str, env: Environment) -> T.Tuple[mparser. if not cargo: raise MesonException(f'Cargo package {package_name!r} not found in {subdir}') - filename = os.path.join(cargo.subdir, cargo.path, 'Cargo.toml') + filename = os.path.join(cargo.subdir, 'Cargo.toml') build = builder.Builder(filename) # Generate project options @@ -726,7 +677,7 @@ def interpret(subp_name: str, subdir: str, env: Environment) -> T.Tuple[mparser. # Libs are always auto-discovered and there's no other way to handle them, # which is unfortunate for reproducability - if os.path.exists(os.path.join(env.source_dir, cargo.subdir, cargo.path, cargo.lib.path)): + if os.path.exists(os.path.join(env.source_dir, cargo.subdir, cargo.lib.path)): for crate_type in cargo.lib.crate_type: ast.extend(_create_lib(cargo, build, crate_type)) diff --git a/mesonbuild/cargo/manifest.py b/mesonbuild/cargo/manifest.py deleted file mode 100644 index e6192d03cd98..000000000000 --- a/mesonbuild/cargo/manifest.py +++ /dev/null @@ -1,227 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright © 2022-2023 Intel Corporation - -"""Type definitions for cargo manifest files.""" - -from __future__ import annotations -import typing as T - -from typing_extensions import Literal, TypedDict, Required - -EDITION = Literal['2015', '2018', '2021'] -CRATE_TYPE = Literal['bin', 'lib', 'dylib', 'staticlib', 'cdylib', 'rlib', 'proc-macro'] - -Package = TypedDict( - 'Package', - { - 'name': Required[str], - 'version': Required[str], - 'authors': T.List[str], - 'edition': EDITION, - 'rust-version': str, - 'description': str, - 'readme': str, - 'license': str, - 'license-file': str, - 'keywords': T.List[str], - 'categories': T.List[str], - 'workspace': str, - 'build': str, - 'links': str, - 'include': T.List[str], - 'exclude': T.List[str], - 'publish': bool, - 'metadata': T.Dict[str, T.Dict[str, str]], - 'default-run': str, - 'autobins': bool, - 'autoexamples': bool, - 'autotests': bool, - 'autobenches': bool, - }, - total=False, -) -"""A description of the Package Dictionary.""" - -class FixedPackage(TypedDict, total=False): - - """A description of the Package Dictionary, fixed up.""" - - name: Required[str] - version: Required[str] - authors: T.List[str] - edition: EDITION - rust_version: str - description: str - readme: str - license: str - license_file: str - keywords: T.List[str] - categories: T.List[str] - workspace: str - build: str - links: str - include: T.List[str] - exclude: T.List[str] - publish: bool - metadata: T.Dict[str, T.Dict[str, str]] - default_run: str - autobins: bool - autoexamples: bool - autotests: bool - autobenches: bool - - -class Badge(TypedDict): - - """An entry in the badge section.""" - - status: Literal['actively-developed', 'passively-developed', 'as-is', 'experimental', 'deprecated', 'none'] - - -Dependency = TypedDict( - 'Dependency', - { - 'version': str, - 'registry': str, - 'git': str, - 'branch': str, - 'rev': str, - 'path': str, - 'optional': bool, - 'package': str, - 'default-features': bool, - 'features': T.List[str], - }, - total=False, -) -"""An entry in the *dependencies sections.""" - - -class FixedDependency(TypedDict, total=False): - - """An entry in the *dependencies sections, fixed up.""" - - version: T.List[str] - registry: str - git: str - branch: str - rev: str - path: str - optional: bool - package: str - default_features: bool - features: T.List[str] - - -DependencyV = T.Union[Dependency, str] -"""A Dependency entry, either a string or a Dependency Dict.""" - - -_BaseBuildTarget = TypedDict( - '_BaseBuildTarget', - { - 'path': str, - 'test': bool, - 'doctest': bool, - 'bench': bool, - 'doc': bool, - 'plugin': bool, - 'proc-macro': bool, - 'harness': bool, - 'edition': EDITION, - 'crate-type': T.List[CRATE_TYPE], - 'required-features': T.List[str], - }, - total=False, -) - - -class BuildTarget(_BaseBuildTarget, total=False): - - name: Required[str] - -class LibTarget(_BaseBuildTarget, total=False): - - name: str - - -class _BaseFixedBuildTarget(TypedDict, total=False): - path: str - test: bool - doctest: bool - bench: bool - doc: bool - plugin: bool - harness: bool - edition: EDITION - crate_type: T.List[CRATE_TYPE] - required_features: T.List[str] - - -class FixedBuildTarget(_BaseFixedBuildTarget, total=False): - - name: str - -class FixedLibTarget(_BaseFixedBuildTarget, total=False): - - name: Required[str] - proc_macro: bool - - -class Target(TypedDict): - - """Target entry in the Manifest File.""" - - dependencies: T.Dict[str, DependencyV] - - -class Workspace(TypedDict): - - """The representation of a workspace. - - In a vritual manifest the :attribute:`members` is always present, but in a - project manifest, an empty workspace may be provided, in which case the - workspace is implicitly filled in by values from the path based dependencies. - - the :attribute:`exclude` is always optional - """ - - members: T.List[str] - exclude: T.List[str] - - -Manifest = TypedDict( - 'Manifest', - { - 'package': Package, - 'badges': T.Dict[str, Badge], - 'dependencies': T.Dict[str, DependencyV], - 'dev-dependencies': T.Dict[str, DependencyV], - 'build-dependencies': T.Dict[str, DependencyV], - 'lib': LibTarget, - 'bin': T.List[BuildTarget], - 'test': T.List[BuildTarget], - 'bench': T.List[BuildTarget], - 'example': T.List[BuildTarget], - 'features': T.Dict[str, T.List[str]], - 'target': T.Dict[str, Target], - 'workspace': Workspace, - - # TODO: patch? - # TODO: replace? - }, - total=False, -) -"""The Cargo Manifest format.""" - - -class VirtualManifest(TypedDict): - - """The Representation of a virtual manifest. - - Cargo allows a root manifest that contains only a workspace, this is called - a virtual manifest. This doesn't really map 1:1 with any meson concept, - except perhaps the proposed "meta project". - """ - - workspace: Workspace