Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use poetry to manage deps for isolated build envs #9168

Merged
merged 1 commit into from
Mar 19, 2024
Merged

use poetry to manage deps for isolated build envs #9168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 11 additions & 42 deletions src/poetry/inspection/info.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import functools
import glob
import logging
import tempfile

from pathlib import Path
from typing import TYPE_CHECKING
Expand All @@ -13,7 +14,7 @@

import pkginfo

from poetry.core.factory import Factory
from build import BuildBackendException
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.package import Package
from poetry.core.pyproject.toml import PyProjectTOML
Expand All @@ -22,9 +23,9 @@
from poetry.core.version.markers import InvalidMarker
from poetry.core.version.requirements import InvalidRequirement

from poetry.utils.env import EnvCommandError
from poetry.utils.env import ephemeral_environment
from poetry.factory import Factory
from poetry.utils.helpers import extractall
from poetry.utils.isolated_build import isolated_builder
from poetry.utils.setup_reader import SetupReader


Expand All @@ -38,25 +39,6 @@

logger = logging.getLogger(__name__)

PEP517_META_BUILD = """\
import build
import build.env
import pyproject_hooks

source = '{source}'
dest = '{dest}'

with build.env.DefaultIsolatedEnv() as env:
builder = build.ProjectBuilder.from_isolated_env(
env, source, runner=pyproject_hooks.quiet_subprocess_runner
)
env.install(builder.build_system_requires)
env.install(builder.get_requires_for_build('wheel'))
builder.metadata_path(dest)
"""

PEP517_META_BUILD_DEPS = ["build==1.1.1", "pyproject_hooks==1.0.0"]


class PackageInfoError(ValueError):
def __init__(self, path: Path, *reasons: BaseException | str) -> None:
Expand Down Expand Up @@ -577,28 +559,15 @@ def get_pep517_metadata(path: Path) -> PackageInfo:
if all(x is not None for x in (info.version, info.name, info.requires_dist)):
return info

with ephemeral_environment(
flags={"no-pip": False, "no-setuptools": True, "no-wheel": True}
) as venv:
# TODO: cache PEP 517 build environment corresponding to each project venv
dest_dir = venv.path.parent / "dist"
dest_dir.mkdir()
with tempfile.TemporaryDirectory() as dist:
try:
dest = Path(dist)

pep517_meta_build_script = PEP517_META_BUILD.format(
source=path.as_posix(), dest=dest_dir.as_posix()
)
with isolated_builder(path, "wheel") as builder:
builder.metadata_path(dest)

try:
venv.run_pip(
"install",
"--disable-pip-version-check",
"--ignore-installed",
"--no-input",
*PEP517_META_BUILD_DEPS,
)
venv.run_python_script(pep517_meta_build_script)
info = PackageInfo.from_metadata_directory(dest_dir)
except EnvCommandError as e:
info = PackageInfo.from_metadata_directory(dest)
except BuildBackendException as e:
logger.debug("PEP517 build failed: %s", e)
abn marked this conversation as resolved.
Show resolved Hide resolved
raise PackageInfoError(path, e, "PEP517 build failed")

Expand Down
145 changes: 28 additions & 117 deletions src/poetry/installation/chef.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,20 @@
from __future__ import annotations

import os
import tempfile

from contextlib import redirect_stdout
from io import StringIO
from pathlib import Path
from typing import TYPE_CHECKING

from build import BuildBackendException
from build import ProjectBuilder
from build.env import IsolatedEnv as BaseIsolatedEnv
from poetry.core.utils.helpers import temporary_directory
from pyproject_hooks import quiet_subprocess_runner # type: ignore[import-untyped]

from poetry.utils._compat import decode
from poetry.utils.env import ephemeral_environment
from poetry.utils.helpers import extractall
from poetry.utils.isolated_build import IsolatedBuildError
from poetry.utils.isolated_build import isolated_builder


if TYPE_CHECKING:
from collections.abc import Collection

from poetry.repositories import RepositoryPool
from poetry.utils.cache import ArtifactCache
from poetry.utils.env import Env
Expand All @@ -30,78 +23,6 @@
class ChefError(Exception): ...


class ChefBuildError(ChefError): ...


class ChefInstallError(ChefError):
def __init__(self, requirements: Collection[str], output: str, error: str) -> None:
message = "\n\n".join(
(
f"Failed to install {', '.join(requirements)}.",
f"Output:\n{output}",
f"Error:\n{error}",
)
)
super().__init__(message)
self._requirements = requirements

@property
def requirements(self) -> Collection[str]:
return self._requirements


class IsolatedEnv(BaseIsolatedEnv):
def __init__(self, env: Env, pool: RepositoryPool) -> None:
self._env = env
self._pool = pool

@property
def python_executable(self) -> str:
return str(self._env.python)

def make_extra_environ(self) -> dict[str, str]:
path = os.environ.get("PATH")
scripts_dir = str(self._env._bin_dir)
return {
"PATH": (
os.pathsep.join([scripts_dir, path])
if path is not None
else scripts_dir
)
}

def install(self, requirements: Collection[str]) -> None:
from cleo.io.buffered_io import BufferedIO
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.project_package import ProjectPackage

from poetry.config.config import Config
from poetry.installation.installer import Installer
from poetry.packages.locker import Locker
from poetry.repositories.installed_repository import InstalledRepository

# We build Poetry dependencies from the requirements
package = ProjectPackage("__root__", "0.0.0")
package.python_versions = ".".join(str(v) for v in self._env.version_info[:3])
for requirement in requirements:
dependency = Dependency.create_from_pep_508(requirement)
package.add_dependency(dependency)

io = BufferedIO()
installer = Installer(
io,
self._env,
package,
Locker(self._env.path.joinpath("poetry.lock"), {}),
self._pool,
Config.create(),
InstalledRepository.load(self._env),
)
installer.update(True)
if installer.run() != 0:
raise ChefInstallError(requirements, io.fetch_output(), io.fetch_error())


class Chef:
def __init__(
self, artifact_cache: ArtifactCache, env: Env, pool: RepositoryPool
Expand All @@ -127,46 +48,36 @@ def _prepare(
) -> Path:
from subprocess import CalledProcessError

with ephemeral_environment(
self._env.python,
flags={"no-pip": True, "no-setuptools": True, "no-wheel": True},
) as venv:
env = IsolatedEnv(venv, self._pool)
builder = ProjectBuilder.from_isolated_env(
env, directory, runner=quiet_subprocess_runner
)
env.install(builder.build_system_requires)

stdout = StringIO()
error: Exception | None = None
try:
with redirect_stdout(stdout):
dist_format = "wheel" if not editable else "editable"
env.install(
builder.build_system_requires
| builder.get_requires_for_build(dist_format)
)
path = Path(
builder.build(
dist_format,
destination.as_posix(),
)
distribution = "wheel" if not editable else "editable"
error: Exception | None = None

try:
with isolated_builder(
source=directory,
distribution=distribution,
python_executable=self._env.python,
pool=self._pool,
) as builder:
return Path(
builder.build(
distribution,
destination.as_posix(),
)
except BuildBackendException as e:
message_parts = [str(e)]
if isinstance(e.exception, CalledProcessError):
text = e.exception.stderr or e.exception.stdout
if text is not None:
message_parts.append(decode(text))
else:
message_parts.append(str(e.exception))
)
except BuildBackendException as e:
message_parts = [str(e)]

error = ChefBuildError("\n\n".join(message_parts))
if isinstance(e.exception, CalledProcessError):
text = e.exception.stderr or e.exception.stdout
if text is not None:
message_parts.append(decode(text))
else:
message_parts.append(str(e.exception))

if error is not None:
raise error from None
error = IsolatedBuildError("\n\n".join(message_parts))

return path
if error is not None:
raise error from None

def _prepare_sdist(self, archive: Path, destination: Path | None = None) -> Path:
from poetry.core.packages.utils.link import Link
Expand Down
8 changes: 4 additions & 4 deletions src/poetry/installation/executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@
from poetry.core.packages.utils.link import Link

from poetry.installation.chef import Chef
from poetry.installation.chef import ChefBuildError
from poetry.installation.chef import ChefInstallError
from poetry.installation.chooser import Chooser
from poetry.installation.operations import Install
from poetry.installation.operations import Uninstall
Expand All @@ -34,6 +32,8 @@
from poetry.utils.helpers import get_highest_priority_hash_type
from poetry.utils.helpers import pluralize
from poetry.utils.helpers import remove_directory
from poetry.utils.isolated_build import IsolatedBuildError
from poetry.utils.isolated_build import IsolatedBuildInstallError
from poetry.utils.pip import pip_install


Expand Down Expand Up @@ -310,7 +310,7 @@ def _execute_operation(self, operation: Operation) -> None:
trace = ExceptionTrace(e)
trace.render(io)
pkg = operation.package
if isinstance(e, ChefBuildError):
if isinstance(e, IsolatedBuildError):
pip_command = "pip wheel --no-cache-dir --use-pep517"
if pkg.develop:
requirement = pkg.source_url
Expand All @@ -328,7 +328,7 @@ def _execute_operation(self, operation: Operation) -> None:
f" running '{pip_command} \"{requirement}\"'."
"</info>"
)
elif isinstance(e, ChefInstallError):
elif isinstance(e, IsolatedBuildInstallError):
message = (
"<error>"
"Cannot install build-system.requires"
Expand Down
Loading
Loading