Fix uv PEX builder for VCS and direct URL requirements

docs/notes/2.32.x.md

@@ -119,6 +119,10 @@ with `--no-deps`, preserving reproducibility. This only applies to non-internal,
 explicit requirement strings and a local Python interpreter; other builds silently fall back to pip.
 See [#20679](https://github.com/pantsbuild/pants/issues/20679) for background.
 
+The experimental uv PEX builder (`[python].pex_builder = "uv"`) now correctly handles VCS
+(`git+https://...`), direct URL, and `file://` requirements from PEX-native lockfiles instead
+of incorrectly looking them up on PyPI.
+
 The `runtime` field of [`aws_python_lambda_layer`](https://www.pantsbuild.org/2.32/reference/targets/python_aws_lambda_layer#runtime) or [`aws_python_lambda_function`](https://www.pantsbuild.org/2.32/reference/targets/python_aws_lambda_function#runtime) now has built-in complete platform configurations for x86-64 and arm64 Python 3.14. This provides stable support for Python 3.14 lambdas out of the box, allowing deleting manual `complete_platforms` configuration if any.
 
 The `grpc-python-plugin` tool now uses an updated `v1.73.1` plugin built from  <https://github.com/nhurden/protoc-gen-grpc-python-prebuilt]. This also brings `macos_arm64` support.

src/python/pants/backend/python/util_rules/pex.py

@@ -18,6 +18,7 @@
 import packaging.specifiers
 import packaging.version
 from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name as canonicalize_project_name
 
 from pants.backend.python.subsystems import uv as uv_subsystem
 from pants.backend.python.subsystems.setup import PexBuilder, PythonSetup
@@ -547,6 +548,35 @@ def _check_uv_preconditions(
     return None
 
 
+def _parse_direct_ref_names(top_level_requirements: tuple[str, ...]) -> frozenset[str]:
+    """Extract canonicalized names from direct references in lockfile requirements.
+
+    Assumes PEX-serialized requirement strings normalize to ``name @ url``.
+    """
+    names: set[str] = set()
+    for req_str in top_level_requirements:
+        if not isinstance(req_str, str) or " @ " not in req_str:
+            continue
+        name_part = req_str.split(" @ ", 1)[0]
+        name = name_part.split("[", 1)[0].strip()
+        names.add(canonicalize_project_name(name))
+    return frozenset(names)
+
+
+def _format_lockfile_requirement(req: dict, direct_ref_names: frozenset[str] = frozenset()) -> str:
+    """Format a locked requirement for uv as ``name @ url`` or ``name==version``."""
+    name = req["project_name"]
+    normalized = canonicalize_project_name(name)
+    if normalized in direct_ref_names:
+        artifacts: Sequence = req.get("artifacts") or ()
+        first = artifacts[0] if len(artifacts) >= 1 else None
+        if isinstance(first, Mapping):
+            url = first.get("url")
+            if isinstance(url, str) and url:
+                return f"{name} @ {url}"
+    return f"{name}=={req['version']}"
+
+
 @rule
 async def _build_uv_venv(
     uv_request: _UvVenvRequest,
@@ -578,12 +608,21 @@ async def _build_uv_venv(
                     c.content for c in digest_contents if c.path == loaded_lockfile.lockfile_path
                 )
                 lockfile_data = json.loads(lockfile_bytes)
+                direct_ref_names = _parse_direct_ref_names(
+                    tuple(lockfile_data.get("requirements") or ())
+                )
                 all_resolved_reqs = tuple(
-                    f"{req['project_name']}=={req['version']}"
+                    _format_lockfile_requirement(req, direct_ref_names)
                     for resolve in lockfile_data.get("locked_resolves", ())
                     for req in resolve.get("locked_requirements", ())
                 )
-            except (json.JSONDecodeError, KeyError, StopIteration) as e:
+            except (
+                json.JSONDecodeError,
+                KeyError,
+                StopIteration,
+                TypeError,
+                AttributeError,
+            ) as e:
                 logger.warning(
                     "pex_builder=uv: failed to parse lockfile for %s: %s. "
                     "Falling back to transitive uv resolution.",

src/python/pants/backend/python/util_rules/pex_test.py

@@ -39,6 +39,8 @@
     _BuildPexPythonSetup,
     _BuildPexRequirementsSetup,
     _determine_pex_python_and_platforms,
+    _format_lockfile_requirement,
+    _parse_direct_ref_names,
     _setup_pex_requirements,
 )
 from pants.backend.python.util_rules.pex import rules as pex_rules
@@ -1084,3 +1086,202 @@ def test_uv_pex_builder_skipped_for_internal_only(rule_runner: RuleRunner) -> No
     assert set(parse_requirements(req_strings)).issubset(
         set(parse_requirements(pex_info["requirements"]))
     )
+
+
+@pytest.mark.parametrize(
+    ("top_level_requirements", "expected"),
+    [
+        ((), frozenset()),
+        (("requests==2.31.0", "numpy>=1.26", "six"), frozenset()),
+        (("my-pkg @ git+https://github.com/org/repo.git@abc123",), frozenset({"my-pkg"})),
+        (
+            ("custom-lib @ https://internal.example.com/custom_lib-2.0.whl",),
+            frozenset({"custom-lib"}),
+        ),
+        (("local-pkg @ file:///wheels/local_pkg-1.0.whl",), frozenset({"local-pkg"})),
+        (
+            (
+                "my-pkg[extra1,extra2] @ git+https://github.com/org/repo.git@v1 ; "
+                "python_version >= '3.8'",
+            ),
+            frozenset({"my-pkg"}),
+        ),
+        (
+            (
+                "requests==2.31.0",
+                "my-vcs-pkg @ git+https://github.com/org/repo.git@abc",
+                "numpy>=1.26",
+                "custom-lib @ https://internal.example.com/pkg.whl",
+                "local-pkg @ file:///wheels/pkg.whl",
+            ),
+            frozenset({"my-vcs-pkg", "custom-lib", "local-pkg"}),
+        ),
+        (
+            ("My_Package.Name @ git+https://github.com/org/repo.git@v1",),
+            frozenset({"my-package-name"}),
+        ),
+    ],
+)
+def test_parse_direct_ref_names(
+    top_level_requirements: tuple[str, ...], expected: frozenset[str]
+) -> None:
+    assert _parse_direct_ref_names(top_level_requirements) == expected
+
+
+def _lockfile_req(
+    project_name: str,
+    version: str,
+    artifacts: object = None,
+    *,
+    include_artifacts: bool = True,
+) -> dict[str, object]:
+    req: dict[str, object] = {"project_name": project_name, "version": version}
+    if include_artifacts:
+        req["artifacts"] = artifacts
+    return req
+
+
+@pytest.mark.parametrize(
+    ("req", "direct_ref_names", "expected"),
+    [
+        (
+            _lockfile_req(
+                "requests",
+                "2.31.0",
+                [
+                    {
+                        "url": "https://files.pythonhosted.org/packages/requests-2.31.0-py3-none-any.whl"
+                    },
+                    {"url": "https://files.pythonhosted.org/packages/requests-2.31.0.tar.gz"},
+                ],
+            ),
+            frozenset(),
+            "requests==2.31.0",
+        ),
+        (
+            _lockfile_req(
+                "hdrhistogram",
+                "0.10.3",
+                [{"url": "https://files.pythonhosted.org/packages/hdrhistogram-0.10.3.whl"}],
+            ),
+            frozenset(),
+            "hdrhistogram==0.10.3",
+        ),
+        (
+            _lockfile_req(
+                "my-pkg",
+                "0.1.0",
+                [{"url": "git+https://github.com/org/repo.git@abc123"}],
+            ),
+            frozenset({"my-pkg"}),
+            "my-pkg @ git+https://github.com/org/repo.git@abc123",
+        ),
+        (
+            _lockfile_req(
+                "my-pkg",
+                "0.1.0",
+                [{"url": "git+ssh://git@github.com/org/repo.git@abc123"}],
+            ),
+            frozenset({"my-pkg"}),
+            "my-pkg @ git+ssh://git@github.com/org/repo.git@abc123",
+        ),
+        (
+            _lockfile_req(
+                "my-pkg",
+                "1.0.0",
+                [{"url": "git+file:///local/repos/my-pkg@v1.0"}],
+            ),
+            frozenset({"my-pkg"}),
+            "my-pkg @ git+file:///local/repos/my-pkg@v1.0",
+        ),
+        (
+            _lockfile_req(
+                "hg-pkg",
+                "1.0.0",
+                [{"url": "hg+https://hg.example.com/repo@tip"}],
+            ),
+            frozenset({"hg-pkg"}),
+            "hg-pkg @ hg+https://hg.example.com/repo@tip",
+        ),
+        (
+            _lockfile_req(
+                "custom-lib",
+                "2.0.0",
+                [
+                    {
+                        "url": "https://internal.example.com/packages/custom_lib-2.0.0-py3-none-any.whl"
+                    }
+                ],
+            ),
+            frozenset({"custom-lib"}),
+            "custom-lib @ https://internal.example.com/packages/custom_lib-2.0.0-py3-none-any.whl",
+        ),
+        (
+            _lockfile_req(
+                "local-pkg",
+                "1.0.0",
+                [{"url": "file:///wheels/local_pkg-1.0.0-py3-none-any.whl"}],
+            ),
+            frozenset({"local-pkg"}),
+            "local-pkg @ file:///wheels/local_pkg-1.0.0-py3-none-any.whl",
+        ),
+        (
+            _lockfile_req(
+                "my_vcs_pkg",
+                "1.0.0",
+                [{"url": "git+https://github.com/org/repo.git@v1"}],
+            ),
+            frozenset({"my-vcs-pkg"}),
+            "my_vcs_pkg @ git+https://github.com/org/repo.git@v1",
+        ),
+        (
+            _lockfile_req(
+                "multi-pkg",
+                "1.0.0",
+                [
+                    {"url": "https://primary.example.com/pkg.whl"},
+                    {"url": "https://mirror.example.com/pkg.whl"},
+                ],
+            ),
+            frozenset({"multi-pkg"}),
+            "multi-pkg @ https://primary.example.com/pkg.whl",
+        ),
+        (
+            _lockfile_req(
+                "any-pkg",
+                "1.0.0",
+                [{"url": "git+https://example.com/repo.git@v1"}],
+            ),
+            frozenset(),
+            "any-pkg==1.0.0",
+        ),
+    ],
+)
+def test_format_lockfile_requirement(
+    req: dict, direct_ref_names: frozenset[str], expected: str
+) -> None:
+    assert _format_lockfile_requirement(req, direct_ref_names) == expected
+
+
+def test_format_lockfile_requirement_defaults_to_name_equals_version() -> None:
+    req = _lockfile_req(
+        "any-pkg",
+        "1.0.0",
+        [{"url": "git+https://example.com/repo.git@v1"}],
+    )
+    assert _format_lockfile_requirement(req) == "any-pkg==1.0.0"
+
+
+@pytest.mark.parametrize(
+    "req",
+    [
+        _lockfile_req("mystery", "1.0.0", []),
+        _lockfile_req("mystery", "1.0.0", include_artifacts=False),
+        _lockfile_req("mystery", "1.0.0", None),
+        _lockfile_req("mystery", "1.0.0", [None]),
+        _lockfile_req("mystery", "1.0.0", [{"hash": "aaa"}]),
+        _lockfile_req("mystery", "1.0.0", [{"url": ""}]),
+    ],
+)
+def test_format_lockfile_requirement_falls_back_for_invalid_or_missing_artifacts(req: dict) -> None:
+    assert _format_lockfile_requirement(req, frozenset({"mystery"})) == "mystery==1.0.0"