diff --git a/docs/docs/python/overview/lockfiles.mdx b/docs/docs/python/overview/lockfiles.mdx index 3fb76d6230f..7705b3f2153 100644 --- a/docs/docs/python/overview/lockfiles.mdx +++ b/docs/docs/python/overview/lockfiles.mdx @@ -61,6 +61,28 @@ $ pants generate-lockfiles The inputs used to generate a lockfile are third-party dependencies in your repo, expressed via [`python_requirement` targets](./third-party-dependencies.mdx) , or the `python_requirements` / `poetry_requirements` generator targets. In this case, since you haven't yet explicitly mapped your requirement targets to a resolve, they will all map to `python-default`, and so all serve as inputs to the default lockfile. +### Using uv for faster resolution (experimental) + +If `pants generate-lockfiles` is slow due to dependency resolution, you can opt in to using +[uv](https://github.com/astral-sh/uv) to pre-resolve pinned requirements, while still generating a +Pex lockfile. + +```toml title="pants.toml" +[python] +lockfile_resolver = "uv" +# Currently required: lock with a single Python major/minor version. +interpreter_constraints = ["CPython==3.11.*"] + +[uv] +# Example uv flag support. +args = ["--index-strategy", "unsafe-first-match"] +``` + +Limitations: + +- Only supported with `lock_style="strict"` or `lock_style="sources"` (not `universal`). +- Not supported with `complete_platforms`. + ### Multiple lockfiles It's generally simpler to have a single resolve for the whole repository, if you can get away with it. But sometimes you may need more than one resolve, if you genuinely have conflicting requirements in different parts of your repo. For example, you may have both Django 3 and Django 4 projects in your repo. diff --git a/docs/notes/2.32.x.md b/docs/notes/2.32.x.md index 9951f8545e7..ce80b80718a 100644 --- a/docs/notes/2.32.x.md +++ b/docs/notes/2.32.x.md @@ -113,6 +113,8 @@ The version of [Pex](https://github.com/pex-tool/pex) used by the Python backend - A new `--interpreter-selection-strategy` option to select the `"oldest"` or `"newest"` interpreter when multiple match constraints. - Linux PEX scies can now install themselves with a desktop entry. +Added an experimental `[python].lockfile_resolver` option to choose how Pants resolves third-party requirements when generating Pex lockfiles. When set to `uv`, Pants uses [`uv pip compile`](https://docs.astral.sh/uv/pip/compile/) to pre-resolve fully pinned requirements (with optional passthrough flags via `[uv].args`), and then materializes the final lockfile with `pex lock create --no-transitive`. This aims to unblock use-cases such as `uv`'s `--unsafe-first-match` resolver behavior (see https://github.com/pantsbuild/pants/issues/20679). Current limitations include requiring a single Python major/minor version (e.g. `CPython==3.11.*`) and not supporting `lock_style="universal"` / `complete_platforms` in `uv` mode yet. + #### Shell #### Javascript diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 495be2e6e9c..531d0397816 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -10,7 +10,8 @@ from operator import itemgetter from pants.backend.python.subsystems.python_tool_base import PythonToolBase -from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.subsystems.setup import LockfileResolver, PythonSetup +from pants.backend.python.subsystems.uv import Uv from pants.backend.python.target_types import ( PythonRequirementFindLinksField, PythonRequirementResolveField, @@ -45,6 +46,7 @@ WrappedGenerateLockfile, ) from pants.core.goals.resolves import ExportableTool +from pants.core.util_rules.external_tool import download_external_tool from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest from pants.engine.addresses import UnparsedAddressInputs from pants.engine.fs import ( @@ -64,15 +66,18 @@ merge_digests, path_globs_to_digest, ) -from pants.engine.process import ProcessCacheScope, execute_process_or_raise +from pants.engine.platform import Platform +from pants.engine.process import Process, ProcessCacheScope, execute_process_or_raise from pants.engine.rules import collect_rules, implicitly, rule from pants.engine.target import AllTargets from pants.engine.unions import UnionMembership, UnionRule +from pants.option.errors import OptionsError from pants.option.subsystem import _construct_subsystem from pants.util.docutil import bin_name from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet from pants.util.pip_requirement import PipRequirement +from pants.util.strutil import softwrap @dataclass(frozen=True) @@ -101,6 +106,22 @@ class _PipArgsAndConstraintsSetup: digest: Digest +def _strip_named_repo(value: str) -> str: + """Strip Pex-style named repo values like `name=https://...` down to the URL/path. + + Pants allows `[python-repos].indexes` / `[python-repos].find_links` entries to optionally be + named so they can be referenced by `--source` in Pex. `uv` does not understand this syntax. + """ + maybe_name, maybe_url = value.split("=", 1) if "=" in value else ("", value) + if ( + maybe_name + and "://" not in maybe_name + and ("://" in maybe_url or maybe_url.startswith("file:")) + ): + return maybe_url + return value + + async def _setup_pip_args_and_constraints_file(resolve_name: str) -> _PipArgsAndConstraintsSetup: resolve_config = await determine_resolve_pex_config( ResolvePexConfigRequest(resolve_name), **implicitly() @@ -123,6 +144,8 @@ async def generate_lockfile( generate_lockfiles_subsystem: GenerateLockfilesSubsystem, python_setup: PythonSetup, pex_subsystem: PexSubsystem, + uv: Uv, + platform: Platform, ) -> GenerateLockfileResult: if not req.requirements: raise ValueError( @@ -132,6 +155,82 @@ async def generate_lockfile( pip_args_setup = await _setup_pip_args_and_constraints_file(req.resolve_name) header_delimiter = "//" + use_uv = python_setup.lockfile_resolver == LockfileResolver.uv + uv_compile_output_digest: Digest | None = None + uv_compiled_requirements_path = "__uv_compiled_requirements.txt" + resolve_config = pip_args_setup.resolve_config + + if use_uv: + if req.lock_style == "universal": + raise OptionsError( + softwrap( + f""" + `[python].lockfile_resolver = "uv"` does not yet support `lock_style="universal"`. + + To use uv today, set `[python].resolves_to_lock_style` for `{req.resolve_name}` to + `"strict"` or `"sources"`, or set `[python].lockfile_resolver = "pip"`. + """ + ) + ) + if req.complete_platforms: + raise OptionsError( + softwrap( + f""" + `[python].lockfile_resolver = "uv"` does not yet support `complete_platforms`. + + Either remove `[python].resolves_to_complete_platforms` for `{req.resolve_name}`, + or set `[python].lockfile_resolver = "pip"`. + """ + ) + ) + + if resolve_config.overrides or resolve_config.sources or resolve_config.excludes: + raise OptionsError( + softwrap( + f""" + `[python].lockfile_resolver = "uv"` is not yet compatible with per-resolve + `overrides`, `sources`, or `excludes` for `{req.resolve_name}`. + + Either remove these settings for the resolve, or set `[python].lockfile_resolver = "pip"`. + """ + ) + ) + + # `uv pip compile` resolves for a single interpreter, so only enable it when + # interpreter constraints narrow to a single major/minor version. + if not req.interpreter_constraints: + raise OptionsError( + softwrap( + f""" + `[python].lockfile_resolver = "uv"` requires interpreter constraints to be set + (and to select a single Python major/minor version) for `{req.resolve_name}`. + + For example: + [python] + interpreter_constraints = [\"CPython==3.11.*\"] + """ + ) + ) + majors_minors = { + (maj, minor) + for maj, minor, _ in req.interpreter_constraints.enumerate_python_versions( + python_setup.interpreter_versions_universe + ) + } + if len(majors_minors) != 1: + raise OptionsError( + softwrap( + f""" + `[python].lockfile_resolver = "uv"` currently requires interpreter constraints + to select exactly one Python major/minor version for `{req.resolve_name}`, but + the constraints `{req.interpreter_constraints}` select: {sorted(majors_minors)}. + + Either narrow your constraints (e.g. `CPython==3.11.*`) or set + `[python].lockfile_resolver = "pip"`. + """ + ) + ) + python = await find_interpreter(req.interpreter_constraints, **implicitly()) # Resolve complete platform targets if specified @@ -185,6 +284,82 @@ async def generate_lockfile( existing_lockfile_digest = EMPTY_DIGEST output_flag = "--lock" if generate_lockfiles_subsystem.sync else "--output" + + if use_uv: + downloaded_uv = await download_external_tool(uv.get_request(platform)) + + requirements_in = "__uv_requirements.in" + uv_input_digest = await create_digest( + CreateDigest( + [ + FileContent( + requirements_in, + ("\n".join(req.requirements) + "\n").encode("utf-8"), + ) + ] + ) + ) + + uv_index_args: list[str] = [] + index_urls = [_strip_named_repo(i) for i in resolve_config.indexes] + if index_urls: + uv_index_args.extend(["--default-index", index_urls[0]]) + for extra_index in index_urls[1:]: + uv_index_args.extend(["--index", extra_index]) + else: + uv_index_args.append("--no-index") + + uv_find_links_args = list( + itertools.chain.from_iterable( + ["--find-links", _strip_named_repo(link)] + for link in (*resolve_config.find_links, *req.find_links) + ) + ) + + uv_constraints_args: list[str] = [] + if resolve_config.constraints_file: + uv_constraints_args.extend(["--constraints", resolve_config.constraints_file.path]) + + uv_build_args: list[str] = [] + if resolve_config.no_binary: + uv_build_args.extend(["--no-binary", ",".join(resolve_config.no_binary)]) + if resolve_config.only_binary: + uv_build_args.extend(["--only-binary", ",".join(resolve_config.only_binary)]) + + uv_process_input_digest = await merge_digests( + MergeDigests([downloaded_uv.digest, uv_input_digest, pip_args_setup.digest]) + ) + uv_result = await execute_process_or_raise( + **implicitly( + Process( + argv=( + downloaded_uv.exe, + "pip", + "compile", + requirements_in, + "--output-file", + uv_compiled_requirements_path, + "--format", + "requirements.txt", + "--no-header", + "--no-annotate", + "--python", + python.path, + *uv_index_args, + *uv_find_links_args, + *uv_constraints_args, + *uv_build_args, + *uv.args, + ), + input_digest=uv_process_input_digest, + output_files=(uv_compiled_requirements_path,), + description=f"Resolve requirements for {req.resolve_name} with uv", + cache_scope=ProcessCacheScope.PER_SESSION, + ) + ) + ) + uv_compile_output_digest = uv_result.output_digest + result = await execute_process_or_raise( **implicitly( PexCliProcess( @@ -219,11 +394,20 @@ async def generate_lockfile( f"--override={override}" for override in pip_args_setup.resolve_config.overrides ), - *req.requirements, + *( + ( + "--no-transitive", + "--requirement", + uv_compiled_requirements_path, + ) + if use_uv + else tuple(req.requirements) + ), ), additional_input_digest=await merge_digests( MergeDigests( [existing_lockfile_digest, pip_args_setup.digest] + + ([uv_compile_output_digest] if uv_compile_output_digest else []) + ([complete_platforms.digest] if complete_platforms else []) ) ), diff --git a/src/python/pants/backend/python/goals/lockfile_test.py b/src/python/pants/backend/python/goals/lockfile_test.py index 33291ea1099..c40de666c6f 100644 --- a/src/python/pants/backend/python/goals/lockfile_test.py +++ b/src/python/pants/backend/python/goals/lockfile_test.py @@ -434,6 +434,72 @@ def test_define_source_from_different_index(rule_runner: PythonRuleRunner) -> No assert sdist_cowsay in artifacts +def test_uv_lockfile_resolver_rejects_universal_lock_style(rule_runner: PythonRuleRunner) -> None: + rule_runner.set_options(["--python-lockfile-resolver=uv"], env_inherit=PYTHON_BOOTSTRAP_ENV) + with pytest.raises(ExecutionError) as excinfo: + rule_runner.request( + GenerateLockfileResult, + [ + GeneratePythonLockfile( + requirements=FrozenOrderedSet(["ansicolors==1.1.8"]), + find_links=FrozenOrderedSet([]), + interpreter_constraints=InterpreterConstraints(), + resolve_name="test", + lockfile_dest="test.lock", + diff=False, + lock_style="universal", + complete_platforms=(), + ) + ], + ) + assert 'does not yet support `lock_style="universal"`' in str(excinfo.value) + + +def test_uv_lockfile_resolver_requires_interpreter_constraints( + rule_runner: PythonRuleRunner, +) -> None: + rule_runner.set_options(["--python-lockfile-resolver=uv"], env_inherit=PYTHON_BOOTSTRAP_ENV) + with pytest.raises(ExecutionError) as excinfo: + rule_runner.request( + GenerateLockfileResult, + [ + GeneratePythonLockfile( + requirements=FrozenOrderedSet(["ansicolors==1.1.8"]), + find_links=FrozenOrderedSet([]), + interpreter_constraints=InterpreterConstraints(), + resolve_name="test", + lockfile_dest="test.lock", + diff=False, + lock_style="strict", + complete_platforms=(), + ) + ], + ) + assert "requires interpreter constraints to be set" in str(excinfo.value) + + +def test_uv_lockfile_resolver_requires_single_python_minor(rule_runner: PythonRuleRunner) -> None: + rule_runner.set_options(["--python-lockfile-resolver=uv"], env_inherit=PYTHON_BOOTSTRAP_ENV) + with pytest.raises(ExecutionError) as excinfo: + rule_runner.request( + GenerateLockfileResult, + [ + GeneratePythonLockfile( + requirements=FrozenOrderedSet(["ansicolors==1.1.8"]), + find_links=FrozenOrderedSet([]), + interpreter_constraints=InterpreterConstraints(["CPython>=3.10,<3.12"]), + resolve_name="test", + lockfile_dest="test.lock", + diff=False, + lock_style="strict", + complete_platforms=(), + ) + ], + ) + assert "requires interpreter constraints" in str(excinfo.value) + assert "select exactly one Python major/minor version" in str(excinfo.value) + + def test_override_version(rule_runner: PythonRuleRunner) -> None: args = [ "--python-resolves={'test': 'test.lock'}", diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 85c4886ed0b..b8e6de2ac00 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -32,7 +32,7 @@ from pants.backend.python.macros.python_artifact import PythonArtifact from pants.backend.python.macros.python_requirements import PythonRequirementsTargetGenerator from pants.backend.python.macros.uv_requirements import UvRequirementsTargetGenerator -from pants.backend.python.subsystems import debugpy +from pants.backend.python.subsystems import debugpy, uv from pants.backend.python.target_types import ( PexBinariesGeneratorTarget, PexBinary, @@ -70,6 +70,7 @@ def rules(): # Subsystems *coverage_py.rules(), *debugpy.rules(), + *uv.rules(), # Util rules *ancestor_files.rules(), *dependency_inference_rules.rules(), diff --git a/src/python/pants/backend/python/subsystems/setup.py b/src/python/pants/backend/python/subsystems/setup.py index f34dd9a47ed..2b59eda69df 100644 --- a/src/python/pants/backend/python/subsystems/setup.py +++ b/src/python/pants/backend/python/subsystems/setup.py @@ -42,6 +42,12 @@ class LockfileGenerator(enum.Enum): POETRY = "poetry" +@enum.unique +class LockfileResolver(enum.Enum): + pip = "pip" + uv = "uv" + + RESOLVE_OPTION_KEY__DEFAULT = "__default__" _T = TypeVar("_T") @@ -312,6 +318,24 @@ def default_to_resolve_interpreter_constraints(self) -> bool: ), advanced=True, ) + lockfile_resolver = EnumOption( + default=LockfileResolver.pip, + help=softwrap( + """ + Which resolver to use when generating Pex lockfiles with `pants generate-lockfiles`. + + - `pip` (default): Use `pex lock create` with pip's resolver. + - `uv` (experimental): Use `uv pip compile` to pre-resolve the full set of pinned + requirements, then run `pex lock create --no-transitive` to materialize a Pex lock. + + Limitations when using `uv`: + - Only supported for non-`universal` lock styles and without `complete_platforms`. + - Does not currently model all Pex-specific features (e.g. certain per-resolve + `--override` behavior) during the `uv` pre-resolution step. + """ + ), + advanced=True, + ) _resolves_to_interpreter_constraints = DictOption[list[str]]( help=softwrap( """ diff --git a/src/python/pants/backend/python/subsystems/uv.py b/src/python/pants/backend/python/subsystems/uv.py new file mode 100644 index 00000000000..f038842222d --- /dev/null +++ b/src/python/pants/backend/python/subsystems/uv.py @@ -0,0 +1,62 @@ +# Copyright 2025 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from pants.core.goals.resolves import ExportableTool +from pants.core.util_rules.external_tool import TemplatedExternalTool +from pants.engine.platform import Platform +from pants.engine.rules import collect_rules +from pants.engine.unions import UnionRule +from pants.option.option_types import ArgsListOption +from pants.util.strutil import softwrap + + +class Uv(TemplatedExternalTool): + options_scope = "uv" + name = "uv" + help = "The uv Python package manager (https://github.com/astral-sh/uv)." + + default_version = "0.9.5" + default_known_versions = [ + "0.9.5|macos_x86_64|58b1d4a25aa8ff99147c2550b33dcf730207fe7e0f9a0d5d36a1bbf36b845aca|19689319", + "0.9.5|macos_arm64|dc098ff224d78ed418e121fd374f655949d2c7031a70f6f6604eaf016a130433|18341942", + "0.9.5|linux_x86_64|3665ffb6c429c31ad6c778ac0489b7746e691acf025cf530b3510b2f9b1660ff|21566106", + "0.9.5|linux_arm64|42b9b83933a289fe9c0e48f4973dee49ce0dfb95e19ea0b525ca0dbca3bce71f|20079609", + ] + version_constraints = ">=0.0.0,<1" + + default_url_template = ( + "https://github.com/astral-sh/uv/releases/download/{version}/uv-{platform}.tar.gz" + ) + default_url_platform_mapping = { + # NB. Prefer musl over gnu, for increased compatibility. + "linux_arm64": "aarch64-unknown-linux-musl", + "linux_x86_64": "x86_64-unknown-linux-musl", + "macos_arm64": "aarch64-apple-darwin", + "macos_x86_64": "x86_64-apple-darwin", + } + + def generate_exe(self, plat: Platform) -> str: + platform = self.default_url_platform_mapping[plat.value] + return f"./uv-{platform}/uv" + + args = ArgsListOption( + passthrough=True, + tool_name="uv pip compile", + example="--index-strategy unsafe-first-match --resolution lowest-direct", + extra_help=softwrap( + """ + Only used when `[python].lockfile_resolver = "uv"`. + + For example, to prefer the first index that provides a given package, set: + + [uv] + args = ["--index-strategy", "unsafe-first-match"] + """ + ), + ) + + +def rules(): + return [*collect_rules(), UnionRule(ExportableTool, Uv)] diff --git a/src/rust/client/src/main.rs b/src/rust/client/src/main.rs index 559b4c86f07..c126e1570d7 100644 --- a/src/rust/client/src/main.rs +++ b/src/rust/client/src/main.rs @@ -1,7 +1,7 @@ // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). -use std::convert::{AsRef, Infallible}; +use std::convert::Infallible; use std::env; use std::ffi::{CString, OsString}; use std::os::unix::ffi::OsStringExt; diff --git a/src/rust/rule_graph/src/lib.rs b/src/rust/rule_graph/src/lib.rs index b706e1c3126..e24ef5ff630 100644 --- a/src/rust/rule_graph/src/lib.rs +++ b/src/rust/rule_graph/src/lib.rs @@ -496,7 +496,7 @@ impl RuleGraph { root_str, dep_entries .iter() - .map(|d| format!("\"{}\"", d.entry_str)) + .map(|d| format!("\"{}\"", &d.entry_str)) .collect::>() .join(" ") )) @@ -531,7 +531,7 @@ impl RuleGraph { k.fmt_for_graph(display_args), dep_entries .iter() - .map(|d| format!("\"{}\"", d.entry_str)) + .map(|d| format!("\"{}\"", &d.entry_str)) .collect::>() .join(" "), ))