From 4c161d284bac47104db1ccd2e73daeb2e4d0d182 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 8 Jan 2025 12:38:17 -0500 Subject: [PATCH] Respect `requires-python` when installing tools (#10401) ## Summary This PR revives https://github.com/astral-sh/uv/pull/7827 to improve tool resolutions such that, if the resolution fails, and the selected interpreter doesn't match the required Python version from the solve, we attempt to re-solve with a newly-discovered interpreter that _does_ match the required Python version. For now, we attempt to choose a Python interpreter that's greater than the inferred `requires-python`, but compatible with the same Python minor. This helps avoid successive failures for cases like Posting, where choosing Python 3.13 fails because it has a dependency that lacks source distributions and doesn't publish any Python 3.13 wheels. We should further improve the strategy to solve _that_ case too, but this is at least the more conservative option... In short, if you do `uv tool instal posting`, and we find Python 3.8 on your machine, we'll detect that `requires-python: >=3.11`, then search for the latest Python 3.11 interpreter and re-resolve. Closes https://github.com/astral-sh/uv/issues/6381. Closes https://github.com/astral-sh/uv/issues/10282. ## Test Plan The following should succeed: ``` cargo run python uninstall --all cargo run python install 3.8 cargo run tool install posting ``` In the logs, we see: ``` ... DEBUG No compatible version found for: posting DEBUG Refining interpreter with: Python >=3.11, <3.12 DEBUG Searching for Python >=3.11, <3.12 in managed installations or search path DEBUG Searching for managed installations at `/Users/crmarsh/.local/share/uv/python` DEBUG Skipping incompatible managed installation `cpython-3.8.20-macos-aarch64-none` DEBUG Found `cpython-3.13.1-macos-aarch64-none` at `/opt/homebrew/bin/python3` (search path) DEBUG Skipping interpreter at `/opt/homebrew/opt/python@3.13/bin/python3.13` from search path: does not satisfy request `>=3.11, <3.12` DEBUG Found `cpython-3.11.7-macos-aarch64-none` at `/opt/homebrew/bin/python3.11` (search path) DEBUG Re-resolving with Python 3.11.7 DEBUG Using request timeout of 30s DEBUG Solving with installed Python version: 3.11.7 DEBUG Solving with target Python version: >=3.11.7 DEBUG Adding direct dependency: posting* DEBUG Searching for a compatible version of posting (*) ... ``` --- crates/uv-requirements/src/specification.rs | 2 +- crates/uv-resolver/src/error.rs | 28 +++++ crates/uv-resolver/src/requires_python.rs | 66 ++++++----- crates/uv/src/commands/project/environment.rs | 4 +- crates/uv/src/commands/project/mod.rs | 2 +- crates/uv/src/commands/project/run.rs | 4 +- crates/uv/src/commands/tool/common.rs | 110 ++++++++++++++++-- crates/uv/src/commands/tool/install.rs | 84 ++++++++++--- crates/uv/src/commands/tool/run.rs | 80 +++++++++++-- crates/uv/src/lib.rs | 4 +- 10 files changed, 320 insertions(+), 64 deletions(-) diff --git a/crates/uv-requirements/src/specification.rs b/crates/uv-requirements/src/specification.rs index 61c5513e1079..2c8ec42a8651 100644 --- a/crates/uv-requirements/src/specification.rs +++ b/crates/uv-requirements/src/specification.rs @@ -50,7 +50,7 @@ use uv_workspace::pyproject::PyProjectToml; use crate::RequirementsSource; -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct RequirementsSpecification { /// The name of the project specifying requirements. pub project: Option, diff --git a/crates/uv-resolver/src/error.rs b/crates/uv-resolver/src/error.rs index 2cd333c40a82..de9c15646a9a 100644 --- a/crates/uv-resolver/src/error.rs +++ b/crates/uv-resolver/src/error.rs @@ -22,6 +22,7 @@ use crate::fork_urls::ForkUrls; use crate::prerelease::AllowPrerelease; use crate::pubgrub::{PubGrubPackage, PubGrubPackageInner, PubGrubReportFormatter}; use crate::python_requirement::PythonRequirement; +use crate::requires_python::LowerBound; use crate::resolution::ConflictingDistributionError; use crate::resolver::{ MetadataUnavailable, ResolverEnvironment, UnavailablePackage, UnavailableReason, @@ -294,6 +295,33 @@ impl NoSolutionError { strip(derivation_tree).expect("derivation tree should contain at least one term") } + /// Given a [`DerivationTree`], identify the largest required Python version that is missing. + pub fn find_requires_python(&self) -> LowerBound { + fn find(derivation_tree: &ErrorTree, minimum: &mut LowerBound) { + match derivation_tree { + DerivationTree::Derived(derived) => { + find(derived.cause1.as_ref(), minimum); + find(derived.cause2.as_ref(), minimum); + } + DerivationTree::External(External::FromDependencyOf(.., package, version)) => { + if let PubGrubPackageInner::Python(_) = &**package { + if let Some((lower, ..)) = version.bounding_range() { + let lower = LowerBound::new(lower.cloned()); + if lower > *minimum { + *minimum = lower; + } + } + } + } + DerivationTree::External(_) => {} + } + } + + let mut minimum = LowerBound::default(); + find(&self.error, &mut minimum); + minimum + } + /// Initialize a [`NoSolutionHeader`] for this error. pub fn header(&self) -> NoSolutionHeader { NoSolutionHeader::new(self.env.clone()) diff --git a/crates/uv-resolver/src/requires_python.rs b/crates/uv-resolver/src/requires_python.rs index 7c6b3b99b7aa..9fbaa36af390 100644 --- a/crates/uv-resolver/src/requires_python.rs +++ b/crates/uv-resolver/src/requires_python.rs @@ -586,6 +586,17 @@ impl SimplifiedMarkerTree { pub struct LowerBound(Bound); impl LowerBound { + /// Initialize a [`LowerBound`] with the given bound. + /// + /// These bounds use release-only semantics when comparing versions. + pub fn new(bound: Bound) -> Self { + Self(match bound { + Bound::Included(version) => Bound::Included(version.only_release()), + Bound::Excluded(version) => Bound::Excluded(version.only_release()), + Bound::Unbounded => Bound::Unbounded, + }) + } + /// Return the [`LowerBound`] truncated to the major and minor version. fn major_minor(&self) -> Self { match &self.0 { @@ -600,6 +611,15 @@ impl LowerBound { Bound::Unbounded => Self(Bound::Unbounded), } } + + /// Returns `true` if the lower bound contains the given version. + pub fn contains(&self, version: &Version) -> bool { + match self.0 { + Bound::Included(ref bound) => bound <= version, + Bound::Excluded(ref bound) => bound < version, + Bound::Unbounded => true, + } + } } impl PartialOrd for LowerBound { @@ -668,19 +688,6 @@ impl Default for LowerBound { } } -impl LowerBound { - /// Initialize a [`LowerBound`] with the given bound. - /// - /// These bounds use release-only semantics when comparing versions. - pub fn new(bound: Bound) -> Self { - Self(match bound { - Bound::Included(version) => Bound::Included(version.only_release()), - Bound::Excluded(version) => Bound::Excluded(version.only_release()), - Bound::Unbounded => Bound::Unbounded, - }) - } -} - impl Deref for LowerBound { type Target = Bound; @@ -699,6 +706,17 @@ impl From for Bound { pub struct UpperBound(Bound); impl UpperBound { + /// Initialize a [`UpperBound`] with the given bound. + /// + /// These bounds use release-only semantics when comparing versions. + pub fn new(bound: Bound) -> Self { + Self(match bound { + Bound::Included(version) => Bound::Included(version.only_release()), + Bound::Excluded(version) => Bound::Excluded(version.only_release()), + Bound::Unbounded => Bound::Unbounded, + }) + } + /// Return the [`UpperBound`] truncated to the major and minor version. fn major_minor(&self) -> Self { match &self.0 { @@ -721,6 +739,15 @@ impl UpperBound { Bound::Unbounded => Self(Bound::Unbounded), } } + + /// Returns `true` if the upper bound contains the given version. + pub fn contains(&self, version: &Version) -> bool { + match self.0 { + Bound::Included(ref bound) => bound >= version, + Bound::Excluded(ref bound) => bound > version, + Bound::Unbounded => true, + } + } } impl PartialOrd for UpperBound { @@ -787,19 +814,6 @@ impl Default for UpperBound { } } -impl UpperBound { - /// Initialize a [`UpperBound`] with the given bound. - /// - /// These bounds use release-only semantics when comparing versions. - pub fn new(bound: Bound) -> Self { - Self(match bound { - Bound::Included(version) => Bound::Included(version.only_release()), - Bound::Excluded(version) => Bound::Excluded(version.only_release()), - Bound::Unbounded => Bound::Unbounded, - }) - } -} - impl Deref for UpperBound { type Target = Bound; diff --git a/crates/uv/src/commands/project/environment.rs b/crates/uv/src/commands/project/environment.rs index e4dd28ec2c01..e6b696518e39 100644 --- a/crates/uv/src/commands/project/environment.rs +++ b/crates/uv/src/commands/project/environment.rs @@ -29,7 +29,7 @@ impl CachedEnvironment { /// interpreter. pub(crate) async fn get_or_create( spec: EnvironmentSpecification<'_>, - interpreter: Interpreter, + interpreter: &Interpreter, settings: &ResolverInstallerSettings, state: &SharedState, resolve: Box, @@ -56,7 +56,7 @@ impl CachedEnvironment { "Caching via interpreter: `{}`", interpreter.sys_executable().display() ); - interpreter + interpreter.clone() }; // Resolve the requirements with the interpreter. diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index 0c96a5b52ac2..c738eba18661 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -1086,7 +1086,7 @@ pub(crate) async fn resolve_names( Ok(requirements) } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct EnvironmentSpecification<'lock> { /// The requirements to include in the environment. requirements: RequirementsSpecification, diff --git a/crates/uv/src/commands/project/run.rs b/crates/uv/src/commands/project/run.rs index e08ecb95ff37..ac4e37694461 100644 --- a/crates/uv/src/commands/project/run.rs +++ b/crates/uv/src/commands/project/run.rs @@ -296,7 +296,7 @@ pub(crate) async fn run( RequirementsSpecification::from_overrides(requirements, constraints, overrides); let result = CachedEnvironment::get_or_create( EnvironmentSpecification::from(spec), - interpreter, + &interpreter, &settings, &state, if show_resolution { @@ -852,7 +852,7 @@ pub(crate) async fn run( lock.as_ref() .map(|(lock, install_path)| (lock, install_path.as_ref())), ), - base_interpreter.clone(), + &base_interpreter, &settings, &state, if show_resolution { diff --git a/crates/uv/src/commands/tool/common.rs b/crates/uv/src/commands/tool/common.rs index 495e7467a161..3e2f283eb75b 100644 --- a/crates/uv/src/commands/tool/common.rs +++ b/crates/uv/src/commands/tool/common.rs @@ -1,25 +1,32 @@ -use std::fmt::Write; -use std::{collections::BTreeSet, ffi::OsString}; - use anyhow::{bail, Context}; use itertools::Itertools; use owo_colors::OwoColorize; +use std::collections::Bound; +use std::fmt::Write; +use std::{collections::BTreeSet, ffi::OsString}; use tracing::{debug, warn}; - +use uv_cache::Cache; +use uv_client::BaseClientBuilder; use uv_distribution_types::{InstalledDist, Name}; #[cfg(unix)] use uv_fs::replace_symlink; use uv_fs::Simplified; use uv_installer::SitePackages; +use uv_pep440::{Version, VersionSpecifier, VersionSpecifiers}; use uv_pep508::PackageName; use uv_pypi_types::Requirement; -use uv_python::PythonEnvironment; -use uv_settings::ToolOptions; +use uv_python::{ + EnvironmentPreference, Interpreter, PythonDownloads, PythonEnvironment, PythonInstallation, + PythonPreference, PythonRequest, PythonVariant, VersionRequest, +}; +use uv_settings::{PythonInstallMirrors, ToolOptions}; use uv_shell::Shell; use uv_tool::{entrypoint_paths, tool_executable_dir, InstalledTools, Tool, ToolEntrypoint}; use uv_warnings::warn_user; -use crate::commands::ExitStatus; +use crate::commands::project::ProjectError; +use crate::commands::reporters::PythonDownloadReporter; +use crate::commands::{pip, ExitStatus}; use crate::printer::Printer; /// Return all packages which contain an executable with the given name. @@ -61,6 +68,95 @@ pub(crate) fn remove_entrypoints(tool: &Tool) { } } +/// Given a no-solution error and the [`Interpreter`] that was used during the solve, attempt to +/// discover an alternate [`Interpreter`] that satisfies the `requires-python` constraint. +pub(crate) async fn refine_interpreter( + interpreter: &Interpreter, + python_request: Option<&PythonRequest>, + err: &pip::operations::Error, + client_builder: &BaseClientBuilder<'_>, + reporter: &PythonDownloadReporter, + install_mirrors: &PythonInstallMirrors, + python_preference: PythonPreference, + python_downloads: PythonDownloads, + cache: &Cache, +) -> anyhow::Result, ProjectError> { + let pip::operations::Error::Resolve(uv_resolver::ResolveError::NoSolution(ref no_solution_err)) = + err + else { + return Ok(None); + }; + + // Infer the `requires-python` constraint from the error. + let requires_python = no_solution_err.find_requires_python(); + + // If the existing interpreter already satisfies the `requires-python` constraint, we don't need + // to refine it. We'd expect to fail again anyway. + if requires_python.contains(interpreter.python_version()) { + return Ok(None); + } + + // If the user passed a `--python` request, and the refined interpreter is incompatible, we + // can't use it. + if let Some(python_request) = python_request { + if !python_request.satisfied(interpreter, cache) { + return Ok(None); + } + } + + // We want an interpreter that's as close to the required version as possible. If we choose the + // "latest" Python, we risk choosing a version that lacks wheels for the tool's requirements + // (assuming those requirements don't publish source distributions). + // + // TODO(charlie): Solve for the Python version iteratively (or even, within the resolver + // itself). The current strategy can also fail if the tool's requirements have greater + // `requires-python` constraints, and we didn't see them in the initial solve. It can also fail + // if the tool's requirements don't publish wheels for this interpreter version, though that's + // rarer. + let lower_bound = match requires_python.as_ref() { + Bound::Included(version) => VersionSpecifier::greater_than_equal_version(version.clone()), + Bound::Excluded(version) => VersionSpecifier::greater_than_version(version.clone()), + Bound::Unbounded => unreachable!("`requires-python` should never be unbounded"), + }; + + let upper_bound = match requires_python.as_ref() { + Bound::Included(version) => { + let major = version.release().first().copied().unwrap_or(0); + let minor = version.release().get(1).copied().unwrap_or(0); + VersionSpecifier::less_than_version(Version::new([major, minor + 1])) + } + Bound::Excluded(version) => { + let major = version.release().first().copied().unwrap_or(0); + let minor = version.release().get(1).copied().unwrap_or(0); + VersionSpecifier::less_than_version(Version::new([major, minor + 1])) + } + Bound::Unbounded => unreachable!("`requires-python` should never be unbounded"), + }; + + let python_request = PythonRequest::Version(VersionRequest::Range( + VersionSpecifiers::from_iter([lower_bound, upper_bound]), + PythonVariant::default(), + )); + + debug!("Refining interpreter with: {python_request}"); + + let interpreter = PythonInstallation::find_or_download( + Some(&python_request), + EnvironmentPreference::OnlySystem, + python_preference, + python_downloads, + client_builder, + cache, + Some(reporter), + install_mirrors.python_install_mirror.as_deref(), + install_mirrors.pypy_install_mirror.as_deref(), + ) + .await? + .into_interpreter(); + + Ok(Some(interpreter)) +} + /// Installs tool executables for a given package and handles any conflicts. pub(crate) fn install_executables( environment: &PythonEnvironment, diff --git a/crates/uv/src/commands/tool/install.rs b/crates/uv/src/commands/tool/install.rs index f36d9a19bf5f..958dd30bcb50 100644 --- a/crates/uv/src/commands/tool/install.rs +++ b/crates/uv/src/commands/tool/install.rs @@ -29,12 +29,10 @@ use crate::commands::project::{ resolve_environment, resolve_names, sync_environment, update_environment, EnvironmentSpecification, ProjectError, }; -use crate::commands::tool::common::remove_entrypoints; +use crate::commands::tool::common::{install_executables, refine_interpreter, remove_entrypoints}; use crate::commands::tool::Target; use crate::commands::ExitStatus; -use crate::commands::{ - diagnostics, reporters::PythonDownloadReporter, tool::common::install_executables, -}; +use crate::commands::{diagnostics, reporters::PythonDownloadReporter}; use crate::printer::Printer; use crate::settings::ResolverInstallerSettings; @@ -448,10 +446,12 @@ pub(crate) async fn install( environment } else { + let spec = EnvironmentSpecification::from(spec); + // If we're creating a new environment, ensure that we can resolve the requirements prior // to removing any existing tools. - let resolution = match resolve_environment( - EnvironmentSpecification::from(spec), + let resolution = resolve_environment( + spec.clone(), &interpreter, settings.as_ref().into(), &state, @@ -464,15 +464,71 @@ pub(crate) async fn install( printer, preview, ) - .await - { + .await; + + // If the resolution failed, retry with the inferred `requires-python` constraint. + let resolution = match resolution { Ok(resolution) => resolution, - Err(ProjectError::Operation(err)) => { - return diagnostics::OperationDiagnostic::default() - .report(err) - .map_or(Ok(ExitStatus::Failure), |err| Err(err.into())) - } - Err(err) => return Err(err.into()), + Err(err) => match err { + ProjectError::Operation(err) => { + // If the resolution failed due to the discovered interpreter not satisfying the + // `requires-python` constraint, we can try to refine the interpreter. + // + // For example, if we discovered a Python 3.8 interpreter on the user's machine, + // but the tool requires Python 3.10 or later, we can try to download a + // Python 3.10 interpreter and re-resolve. + let Some(interpreter) = refine_interpreter( + &interpreter, + python_request.as_ref(), + &err, + &client_builder, + &reporter, + &install_mirrors, + python_preference, + python_downloads, + &cache, + ) + .await + .ok() + .flatten() else { + return diagnostics::OperationDiagnostic::default() + .report(err) + .map_or(Ok(ExitStatus::Failure), |err| Err(err.into())); + }; + + debug!( + "Re-resolving with Python {} (`{}`)", + interpreter.python_version(), + interpreter.sys_executable().display() + ); + + match resolve_environment( + spec, + &interpreter, + settings.as_ref().into(), + &state, + Box::new(DefaultResolveLogger), + connectivity, + concurrency, + native_tls, + allow_insecure_host, + &cache, + printer, + preview, + ) + .await + { + Ok(resolution) => resolution, + Err(ProjectError::Operation(err)) => { + return diagnostics::OperationDiagnostic::default() + .report(err) + .map_or(Ok(ExitStatus::Failure), |err| Err(err.into())); + } + Err(err) => return Err(err.into()), + } + } + err => return Err(err.into()), + }, }; let environment = installed_tools.create_environment(&from.name, interpreter)?; diff --git a/crates/uv/src/commands/tool/run.rs b/crates/uv/src/commands/tool/run.rs index 8d97bc10485d..bdeb735ece4f 100644 --- a/crates/uv/src/commands/tool/run.rs +++ b/crates/uv/src/commands/tool/run.rs @@ -37,11 +37,10 @@ use crate::commands::pip::loggers::{ }; use crate::commands::project::{resolve_names, EnvironmentSpecification, ProjectError}; use crate::commands::reporters::PythonDownloadReporter; +use crate::commands::tool::common::{matching_packages, refine_interpreter}; use crate::commands::tool::Target; use crate::commands::ExitStatus; -use crate::commands::{ - diagnostics, project::environment::CachedEnvironment, tool::common::matching_packages, -}; +use crate::commands::{diagnostics, project::environment::CachedEnvironment}; use crate::printer::Printer; use crate::settings::ResolverInstallerSettings; @@ -610,20 +609,20 @@ async fn get_or_create_environment( } // Create a `RequirementsSpecification` from the resolved requirements, to avoid re-resolving. - let spec = RequirementsSpecification { + let spec = EnvironmentSpecification::from(RequirementsSpecification { requirements: requirements .into_iter() .map(UnresolvedRequirementSpecification::from) .collect(), ..spec - }; + }); // TODO(zanieb): When implementing project-level tools, discover the project and check if it has the tool. // TODO(zanieb): Determine if we should layer on top of the project environment if it is present. - let environment = CachedEnvironment::get_or_create( - EnvironmentSpecification::from(spec), - interpreter, + let result = CachedEnvironment::get_or_create( + spec.clone(), + &interpreter, settings, &state, if show_resolution { @@ -645,7 +644,70 @@ async fn get_or_create_environment( printer, preview, ) - .await?; + .await; + + let environment = match result { + Ok(environment) => environment, + Err(err) => match err { + ProjectError::Operation(err) => { + // If the resolution failed due to the discovered interpreter not satisfying the + // `requires-python` constraint, we can try to refine the interpreter. + // + // For example, if we discovered a Python 3.8 interpreter on the user's machine, + // but the tool requires Python 3.10 or later, we can try to download a + // Python 3.10 interpreter and re-resolve. + let Some(interpreter) = refine_interpreter( + &interpreter, + python_request.as_ref(), + &err, + &client_builder, + &reporter, + &install_mirrors, + python_preference, + python_downloads, + cache, + ) + .await + .ok() + .flatten() else { + return Err(err.into()); + }; + + debug!( + "Re-resolving with Python {} (`{}`)", + interpreter.python_version(), + interpreter.sys_executable().display() + ); + + CachedEnvironment::get_or_create( + spec, + &interpreter, + settings, + &state, + if show_resolution { + Box::new(DefaultResolveLogger) + } else { + Box::new(SummaryResolveLogger) + }, + if show_resolution { + Box::new(DefaultInstallLogger) + } else { + Box::new(SummaryInstallLogger) + }, + installer_metadata, + connectivity, + concurrency, + native_tls, + allow_insecure_host, + cache, + printer, + preview, + ) + .await? + } + err => return Err(err), + }, + }; Ok((from, environment.into())) } diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index ca854b0adb49..2797093f2b4c 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -939,7 +939,7 @@ async fn run(mut cli: Cli) -> Result { ) .collect::>(); - commands::tool_run( + Box::pin(commands::tool_run( args.command, args.from, &requirements, @@ -959,7 +959,7 @@ async fn run(mut cli: Cli) -> Result { cache, printer, globals.preview, - ) + )) .await } Commands::Tool(ToolNamespace {