feat: add logfire.url_from_eval(report) method

logfire-api/logfire_api/__init__.py

@@ -201,6 +201,8 @@ def instrument_system_metrics(self, *args, **kwargs) -> None: ...
 
             def instrument_mcp(self, *args, **kwargs) -> None: ...
 
+            def url_from_eval(self, *args, **kwargs) -> None: ...
+
             def shutdown(self, *args, **kwargs) -> None: ...
 
         DEFAULT_LOGFIRE_INSTANCE = Logfire()
@@ -254,6 +256,7 @@ def shutdown(self, *args, **kwargs) -> None: ...
         instrument_mcp = DEFAULT_LOGFIRE_INSTANCE.instrument_mcp
         shutdown = DEFAULT_LOGFIRE_INSTANCE.shutdown
         suppress_scopes = DEFAULT_LOGFIRE_INSTANCE.suppress_scopes
+        url_from_eval = DEFAULT_LOGFIRE_INSTANCE.url_from_eval
 
         def loguru_handler() -> dict[str, Any]:
             return {}

logfire/__init__.py

@@ -63,6 +63,7 @@
 with_tags = DEFAULT_LOGFIRE_INSTANCE.with_tags
 # with_trace_sample_rate = DEFAULT_LOGFIRE_INSTANCE.with_trace_sample_rate
 with_settings = DEFAULT_LOGFIRE_INSTANCE.with_settings
+url_from_eval = DEFAULT_LOGFIRE_INSTANCE.url_from_eval
 
 # Logging
 log = DEFAULT_LOGFIRE_INSTANCE.log
@@ -176,4 +177,5 @@ def loguru_handler() -> Any:
     'set_baggage',
     'get_context',
     'attach_context',
+    'url_from_eval',
 )

logfire/_internal/config.py

@@ -696,6 +696,8 @@ def _load_configuration(
         self.advanced = advanced
 
         self.additional_span_processors = additional_span_processors
+        self.project_url: str | None = None
+        self._check_tokens_thread: Thread | None = None
 
         if metrics is None:
             metrics = MetricsOptions()
@@ -943,19 +945,22 @@ def add_span_processor(span_processor: SpanProcessor) -> None:
             if isinstance(self.metrics, MetricsOptions):
                 metric_readers = list(self.metrics.additional_readers)
 
+            # Try loading credentials from a file.
+            # We do this before checking send_to_logfire so that project_url
+            # is available for url_from_eval even when not sending data.
+            try:
+                credentials = LogfireCredentials.load_creds_file(self.data_dir)
+            except Exception:
+                # If we have tokens configured by other means, e.g. the env, no need to worry about the creds file.
+                if self.send_to_logfire and not self.token:
+                    raise
+                credentials = None
+            if credentials is not None:
+                self.project_url = self.project_url or credentials.project_url
+
             if self.send_to_logfire:
                 show_project_link: bool = self.console and self.console.show_project_link or False
 
-                # Try loading credentials from a file.
-                # If that works, we can use it to immediately print the project link.
-                try:
-                    credentials = LogfireCredentials.load_creds_file(self.data_dir)
-                except Exception:
-                    # If we have tokens configured by other means, e.g. the env, no need to worry about the creds file.
-                    if not self.token:
-                        raise
-                    credentials = None
-
                 if not self.token and self.send_to_logfire is True and credentials is None:
                     # If we don't have tokens or credentials from a file,
                     # try initializing a new project and writing a new creds file.
@@ -969,6 +974,7 @@ def add_span_processor(span_processor: SpanProcessor) -> None:
                     # This means that e.g. a token in an env var takes priority over a token in a creds file.
                     self.token = self.token or credentials.token
                     self.advanced.base_url = self.advanced.base_url or credentials.logfire_api_url
+                    self.project_url = self.project_url or credentials.project_url
 
                 if self.token:
                     # Convert to list for iteration (handles both str and list[str])
@@ -994,18 +1000,16 @@ def check_tokens():
                         with suppress_instrumentation():
                             for token in token_list:
                                 validated_credentials = self._initialize_credentials_from_token(token)
-                                if (
-                                    validated_credentials is not None
-                                    and show_project_link
-                                    and token not in printed_tokens
-                                ):
-                                    validated_credentials.print_token_summary()
+                                if validated_credentials is not None:
+                                    self.project_url = self.project_url or validated_credentials.project_url
+                                    if show_project_link and token not in printed_tokens:
+                                        validated_credentials.print_token_summary()
 
                     if emscripten:  # pragma: no cover
                         check_tokens()
                     else:
-                        thread = Thread(target=check_tokens, name='check_logfire_token')
-                        thread.start()
+                        self._check_tokens_thread = Thread(target=check_tokens, name='check_logfire_token')
+                        self._check_tokens_thread.start()
 
                     # Create exporters for each token
                     for token in token_list:
@@ -1227,6 +1231,15 @@ def warn_if_not_initialized(self, message: str):
                 category=LogfireNotConfiguredWarning,
             )
 
+    def wait_for_token_validation(self) -> None:
+        """Wait for the background token validation thread to complete.
+
+        This ensures that `project_url` is populated when the token is provided
+        via environment variable rather than a credentials file.
+        """
+        if self._check_tokens_thread is not None:
+            self._check_tokens_thread.join()
+
     def _initialize_credentials_from_token(self, token: str) -> LogfireCredentials | None:
         return LogfireCredentials.from_token(token, requests.Session(), self.advanced.generate_base_url(token))
 

logfire/_internal/main.py

@@ -82,6 +82,7 @@
     from flask.app import Flask
     from opentelemetry.instrumentation.asgi.types import ClientRequestHook, ClientResponseHook, ServerRequestHook
     from opentelemetry.metrics import _Gauge as Gauge
+    from pydantic_evals.reporting import EvaluationReport
     from pymongo.monitoring import CommandFailedEvent, CommandStartedEvent, CommandSucceededEvent
     from sqlalchemy import Engine
     from sqlalchemy.ext.asyncio import AsyncEngine
@@ -876,6 +877,25 @@ def force_flush(self, timeout_millis: int = 3_000) -> bool:  # pragma: no cover
         """
         return self._config.force_flush(timeout_millis)
 
+    def url_from_eval(self, report: EvaluationReport[Any, Any, Any]) -> str | None:
+        """Generate a Logfire URL to view an evaluation report.
+
+        Args:
+            report: An evaluation report from `pydantic_evals`.
+
+        Returns:
+            The URL string, or `None` if the project URL or trace/span IDs are not available.
+        """
+        # Wait for the background token validation thread to finish,
+        # since it may populate project_url when no credentials file exists.
+        self._config.wait_for_token_validation()
+        project_url = self._config.project_url
+        trace_id = report.trace_id
+        span_id = report.span_id
+        if not project_url or not trace_id or not span_id:
+            return None
+        return f'{project_url.rstrip("/")}/evals/compare?experiment={trace_id}-{span_id}'
+
     def log_slow_async_callbacks(self, slow_duration: float = 0.1) -> AbstractContextManager[None]:
         """Log a warning whenever a function running in the asyncio event loop blocks for too long.
 

pyproject.toml

@@ -180,6 +180,7 @@ dev = [
     "pytest-xdist>=3.6.1",
     "openai-agents[voice]>=0.0.7",
     "pydantic-ai-slim>=0.0.39",
+    "pydantic-evals>=0.0.39",
     "langchain>=0.0.27",
     "langchain-openai>=0.3.17",
     "langgraph >= 0",

tests/test_logfire_api.py

@@ -294,6 +294,10 @@ def func() -> None: ...
         pass
     logfire__all__.remove('attach_context')
 
+    assert hasattr(logfire_api, 'url_from_eval')
+    logfire_api.url_from_eval(MagicMock(trace_id='abc', span_id='def'))
+    logfire__all__.remove('url_from_eval')
+
     # If it's not empty, it means that some of the __all__ members are not tested.
     assert logfire__all__ == set(), logfire__all__
 

tests/test_url_from_eval.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import pytest
+import requests_mock
+
+try:
+    from pydantic_evals.reporting import EvaluationReport
+except Exception:
+    pytest.skip('pydantic_evals not importable (likely pydantic < 2.8)', allow_module_level=True)
+
+import logfire
+from logfire._internal.config import LogfireConfig
+
+
+def _make_report(trace_id: str | None = None, span_id: str | None = None) -> EvaluationReport:
+    return EvaluationReport(name='test', cases=[], trace_id=trace_id, span_id=span_id)
+
+
+def test_url_from_eval_with_project_url() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(trace_id='abc123', span_id='def456')
+    result = instance.url_from_eval(report)
+    assert result == 'https://logfire.pydantic.dev/my-org/my-project/evals/compare?experiment=abc123-def456'
+
+
+def test_url_from_eval_no_project_url() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(trace_id='abc123', span_id='def456')
+    result = instance.url_from_eval(report)
+    assert result is None
+
+
+def test_url_from_eval_no_trace_id() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(span_id='def456')
+    result = instance.url_from_eval(report)
+    assert result is None
+
+
+def test_url_from_eval_no_span_id() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(trace_id='abc123')
+    result = instance.url_from_eval(report)
+    assert result is None
+
+
+def test_url_from_eval_trailing_slash() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project/'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(trace_id='abc123', span_id='def456')
+    result = instance.url_from_eval(report)
+    assert result == 'https://logfire.pydantic.dev/my-org/my-project/evals/compare?experiment=abc123-def456'
+
+
+def test_url_from_eval_no_ids() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report()
+    result = instance.url_from_eval(report)
+    assert result is None
+
+
+def test_url_from_eval_waits_for_token_validation() -> None:
+    """Test that url_from_eval waits for the background token validation thread
+    to populate project_url when the token is provided directly (no creds file)."""
+    with requests_mock.Mocker() as mocker:
+        mocker.get(
+            'https://logfire-us.pydantic.dev/v1/info',
+            json={
+                'project_name': 'myproject',
+                'project_url': 'https://logfire-us.pydantic.dev/my-org/my-project',
+            },
+        )
+        logfire.configure(
+            send_to_logfire=True,
+            token='fake-token',
+            console=False,
+        )
+
+        report = _make_report(trace_id='abc123', span_id='def456')
+        # url_from_eval should wait for the background thread and return the URL
+        result = logfire.url_from_eval(report)
+        assert result == 'https://logfire-us.pydantic.dev/my-org/my-project/evals/compare?experiment=abc123-def456'