Fix uv PEX builder to use pex3 lock export

docs/notes/2.32.x.md

@@ -121,6 +121,10 @@ with `--no-deps`, preserving reproducibility. This only applies to non-internal,
 explicit requirement strings and a local Python interpreter; other builds silently fall back to pip.
 See [#20679](https://github.com/pantsbuild/pants/issues/20679) for background.
 
+The experimental uv PEX builder (`[python].pex_builder = "uv"`) now correctly handles VCS
+(`git+https://...`) and direct URL requirements from PEX-native lockfiles by using
+`pex3 lock export` instead of parsing the internal lockfile format directly.
+
 The `runtime` field of [`aws_python_lambda_layer`](https://www.pantsbuild.org/2.32/reference/targets/python_aws_lambda_layer#runtime) or [`aws_python_lambda_function`](https://www.pantsbuild.org/2.32/reference/targets/python_aws_lambda_function#runtime) now has built-in complete platform configurations for x86-64 and arm64 Python 3.14. This provides stable support for Python 3.14 lambdas out of the box, allowing deleting manual `complete_platforms` configuration if any.
 
 The `grpc-python-plugin` tool now uses an updated `v1.73.1` plugin built from  <https://github.com/nhurden/protoc-gen-grpc-python-prebuilt]. This also brings `macos_arm64` support.

src/python/pants/backend/python/util_rules/pex.py

@@ -85,13 +85,13 @@
     add_prefix,
     create_digest,
     digest_to_snapshot,
-    get_digest_contents,
     merge_digests,
     remove_prefix,
 )
 from pants.engine.process import (
     Process,
     ProcessCacheScope,
+    ProcessExecutionFailure,
     ProcessResult,
     execute_process_or_raise,
     fallible_to_exec_result_or_raise,
@@ -560,10 +560,13 @@ async def _build_uv_venv(
         uv_request.description,
     )
 
-    # Try to extract the full resolved package list from the lockfile
-    # so we can pass pinned versions with --no-deps (reproducible).
+    # Try to export the lockfile via `pex3 lock export` so we can pass pinned
+    # versions with --no-deps (reproducible).  This uses Pex's stable CLI rather
+    # than parsing the internal lockfile JSON directly.
     # Fall back to letting uv resolve transitively if no lockfile.
-    all_resolved_reqs: tuple[str, ...] = ()
+    exported_reqs_digest: Digest | None = None
+    reqs_file = "pylock.toml"
+
     if isinstance(uv_request.requirements, PexRequirements) and isinstance(
         uv_request.requirements.from_superset, Resolve
     ):
@@ -573,42 +576,51 @@ async def _build_uv_venv(
         loaded_lockfile = await load_lockfile(LoadedLockfileRequest(lockfile), **implicitly())
         if loaded_lockfile.is_pex_native:
             try:
-                digest_contents = await get_digest_contents(loaded_lockfile.lockfile_digest)
-                lockfile_bytes = next(
-                    c.content for c in digest_contents if c.path == loaded_lockfile.lockfile_path
-                )
-                lockfile_data = json.loads(lockfile_bytes)
-                all_resolved_reqs = tuple(
-                    f"{req['project_name']}=={req['version']}"
-                    for resolve in lockfile_data.get("locked_resolves", ())
-                    for req in resolve.get("locked_requirements", ())
+                export_result = await fallible_to_exec_result_or_raise(
+                    **implicitly(
+                        PexCliProcess(
+                            subcommand=("lock", "export"),
+                            extra_args=(
+                                "--format",
+                                "pep-751",
+                                "-o",
+                                reqs_file,
+                                loaded_lockfile.lockfile_path,
+                            ),
+                            additional_input_digest=loaded_lockfile.lockfile_digest,
+                            description=f"Export lockfile for {uv_request.description}",
+                            output_files=(reqs_file,),
+                        )
+                    )
                 )
-            except (json.JSONDecodeError, KeyError, StopIteration) as e:
+                exported_reqs_digest = export_result.output_digest
+            except ProcessExecutionFailure as e:
                 logger.warning(
-                    "pex_builder=uv: failed to parse lockfile for %s: %s. "
+                    "pex_builder=uv: failed to export lockfile for %s: %s. "
                     "Falling back to transitive uv resolution.",
                     uv_request.description,
                     e,
                 )
-                all_resolved_reqs = ()
 
-    uv_reqs = all_resolved_reqs or uv_request.req_strings
+    use_exported_lockfile = exported_reqs_digest is not None
 
-    if all_resolved_reqs:
+    if use_exported_lockfile:
         logger.debug(
-            "pex_builder=uv: using %d pinned packages from lockfile with --no-deps for %s",
-            len(all_resolved_reqs),
+            "pex_builder=uv: using exported lockfile with --no-deps for %s",
             uv_request.description,
         )
+        assert exported_reqs_digest is not None
+        reqs_digest = exported_reqs_digest
     else:
         logger.debug(
             "pex_builder=uv: no lockfile available, using transitive uv resolution for %s",
             uv_request.description,
         )
-
-    reqs_file = "__uv_requirements.txt"
-    reqs_content = "\n".join(uv_reqs) + "\n"
-    reqs_digest = await create_digest(CreateDigest([FileContent(reqs_file, reqs_content.encode())]))
+        reqs_file = "__uv_requirements.txt"
+        reqs_content = "\n".join(uv_request.req_strings) + "\n"
+        reqs_digest = await create_digest(
+            CreateDigest([FileContent(reqs_file, reqs_content.encode())])
+        )
 
     complete_pex_env = pex_env.in_sandbox(working_directory=None)
     uv_cache_dir = ".cache/uv_cache"
@@ -661,7 +673,7 @@ async def _build_uv_venv(
         os.path.join(_UV_VENV_DIR, "bin", "python"),
         "-r",
         reqs_file,
-        *(("--no-deps",) if all_resolved_reqs else ()),
+        *(("--no-deps",) if use_exported_lockfile else ()),
         *downloaded_uv.args_for_uv_pip_install,
     )
 

src/python/pants/backend/python/util_rules/pex_test.py

@@ -10,6 +10,7 @@
 import textwrap
 import zipfile
 from pathlib import Path
+from types import SimpleNamespace
 
 import pytest
 import requests
@@ -20,6 +21,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
 from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.subsystems.uv import DownloadedUv
 from pants.backend.python.target_types import EntryPoint, PexCompletePlatformsField
 from pants.backend.python.util_rules import pex_test_utils
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
@@ -36,12 +38,15 @@
     VenvPex,
     VenvPexProcess,
     _build_pex_description,
+    _build_uv_venv,
     _BuildPexPythonSetup,
     _BuildPexRequirementsSetup,
     _determine_pex_python_and_platforms,
     _setup_pex_requirements,
+    _UvVenvRequest,
 )
 from pants.backend.python.util_rules.pex import rules as pex_rules
+from pants.backend.python.util_rules.pex_cli import PexCliProcess
 from pants.backend.python.util_rules.pex_environment import PythonExecutable
 from pants.backend.python.util_rules.pex_requirements import (
     EntireLockfile,
@@ -971,6 +976,167 @@ def test_digest_complete_platforms_codegen(rule_runner: RuleRunner) -> None:
     assert complete_platforms.digest != EMPTY_DIGEST
 
 
+def create_uv_venv_test_inputs(
+    rule_runner: RuleRunner,
+    *,
+    description: str,
+) -> tuple[_UvVenvRequest, Lockfile, LoadedLockfile]:
+    resolve = Resolve("python-default", False)
+    lockfile = Lockfile(
+        "3rdparty/python/default.lock", url_description_of_origin="test", resolve_name=resolve.name
+    )
+    loaded_lockfile = LoadedLockfile(
+        lockfile_digest=rule_runner.make_snapshot_of_empty_files([lockfile.url]).digest,
+        lockfile_path=lockfile.url,
+        metadata=None,
+        requirement_estimate=1,
+        is_pex_native=True,
+        as_constraints_strings=None,
+        original_lockfile=lockfile,
+    )
+    uv_request = _UvVenvRequest(
+        req_strings=("ansicolors==1.1.8",),
+        requirements=PexRequirements(("ansicolors==1.1.8",), from_superset=resolve),
+        python_path="/usr/bin/python3",
+        description=description,
+    )
+    return uv_request, lockfile, loaded_lockfile
+
+
+def run_uv_venv_with_mocks(
+    uv_request: _UvVenvRequest,
+    lockfile: Lockfile,
+    loaded_lockfile: LoadedLockfile,
+    *,
+    mock_fallible_to_exec_result_or_raise,
+    mock_create_digest=None,
+):
+    pex_env = SimpleNamespace(append_only_caches={})
+    pex_env.in_sandbox = lambda *, working_directory: pex_env
+    pex_env.environment_dict = lambda *, python_configured: {}
+
+    return run_rule_with_mocks(
+        _build_uv_venv,
+        rule_args=[uv_request, pex_env],
+        mock_calls={
+            "pants.backend.python.subsystems.uv.download_uv_binary": lambda: DownloadedUv(
+                digest=EMPTY_DIGEST,
+                exe="uv",
+                args_for_uv_pip_install=(),
+            ),
+            "pants.backend.python.util_rules.pex_requirements.get_lockfile_for_resolve": lambda _: lockfile,
+            "pants.backend.python.util_rules.pex_requirements.load_lockfile": lambda _: loaded_lockfile,
+            "pants.engine.process.fallible_to_exec_result_or_raise": mock_fallible_to_exec_result_or_raise,
+            "pants.engine.intrinsics.create_digest": mock_create_digest or (lambda _: EMPTY_DIGEST),
+            "pants.engine.intrinsics.merge_digests": lambda _: EMPTY_DIGEST,
+        },
+    )
+
+
+def test_build_uv_venv_uses_exported_lockfile_with_no_deps(rule_runner: RuleRunner) -> None:
+    uv_request, lockfile, loaded_lockfile = create_uv_venv_test_inputs(
+        rule_runner, description="test uv export path"
+    )
+    exported_digest = rule_runner.make_snapshot(
+        {"__uv_requirements.txt": "ansicolors==1.1.8\n"}
+    ).digest
+    install_argv: tuple[str, ...] | None = None
+
+    def mock_fallible_to_exec_result_or_raise(*args, **kwargs):
+        nonlocal install_argv
+        req = args[0]
+        if isinstance(req, PexCliProcess):
+            assert req.subcommand == ("lock", "export")
+            assert "--format" in req.extra_args
+            export_format = req.extra_args[req.extra_args.index("--format") + 1]
+            assert export_format in {"pip-no-hashes", "pep-751"}
+            return SimpleNamespace(output_digest=exported_digest)
+        assert isinstance(req, Process)
+        if req.argv[1:3] == ("pip", "install"):
+            install_argv = req.argv
+        return SimpleNamespace(output_digest=EMPTY_DIGEST)
+
+    def mock_create_digest(request: CreateDigest) -> Digest:
+        for entry in request:
+            assert not (isinstance(entry, FileContent) and entry.path == "__uv_requirements.txt"), (
+                "exported lockfile path should not synthesize requirements content"
+            )
+        return EMPTY_DIGEST
+
+    result = run_uv_venv_with_mocks(
+        uv_request,
+        lockfile,
+        loaded_lockfile,
+        mock_fallible_to_exec_result_or_raise=mock_fallible_to_exec_result_or_raise,
+        mock_create_digest=mock_create_digest,
+    )
+
+    assert result.venv_digest == EMPTY_DIGEST
+    assert install_argv is not None
+    assert "--no-deps" in install_argv
+
+
+def test_build_uv_venv_falls_back_when_lock_export_has_no_digest(rule_runner: RuleRunner) -> None:
+    uv_request, lockfile, loaded_lockfile = create_uv_venv_test_inputs(
+        rule_runner, description="test uv fallback path"
+    )
+    export_attempts = 0
+    install_argv: tuple[str, ...] | None = None
+    synthesized_reqs: bytes | None = None
+
+    def mock_fallible_to_exec_result_or_raise(*args, **kwargs):
+        nonlocal export_attempts, install_argv
+        req = args[0]
+        if isinstance(req, PexCliProcess):
+            export_attempts += 1
+            return SimpleNamespace(output_digest=None)
+        assert isinstance(req, Process)
+        if req.argv[1:3] == ("pip", "install"):
+            install_argv = req.argv
+        return SimpleNamespace(output_digest=EMPTY_DIGEST)
+
+    def mock_create_digest(request: CreateDigest) -> Digest:
+        nonlocal synthesized_reqs
+        for entry in request:
+            if isinstance(entry, FileContent) and entry.path == "__uv_requirements.txt":
+                synthesized_reqs = entry.content
+        return EMPTY_DIGEST
+
+    result = run_uv_venv_with_mocks(
+        uv_request,
+        lockfile,
+        loaded_lockfile,
+        mock_fallible_to_exec_result_or_raise=mock_fallible_to_exec_result_or_raise,
+        mock_create_digest=mock_create_digest,
+    )
+
+    assert result.venv_digest == EMPTY_DIGEST
+    assert export_attempts == 1
+    assert install_argv is not None
+    assert "--no-deps" not in install_argv
+    assert synthesized_reqs == b"ansicolors==1.1.8\n"
+
+
+def test_build_uv_venv_propagates_unexpected_export_errors(rule_runner: RuleRunner) -> None:
+    uv_request, lockfile, loaded_lockfile = create_uv_venv_test_inputs(
+        rule_runner, description="test unexpected error path"
+    )
+
+    def mock_fallible_to_exec_result_or_raise(*args, **kwargs):
+        req = args[0]
+        if isinstance(req, PexCliProcess):
+            raise ValueError("unexpected failure type")
+        return SimpleNamespace(output_digest=EMPTY_DIGEST)
+
+    with pytest.raises(ValueError, match="unexpected failure type"):
+        run_uv_venv_with_mocks(
+            uv_request,
+            lockfile,
+            loaded_lockfile,
+            mock_fallible_to_exec_result_or_raise=mock_fallible_to_exec_result_or_raise,
+        )
+
+
 def test_uv_pex_builder_resolves_dependencies(rule_runner: RuleRunner) -> None:
     """When pex_builder=uv, PEX should be built via uv venv + --venv-repository."""
     req_strings = ["six==1.12.0", "jsonschema==2.6.0"]