feat: add logfire.url_from_eval(report) method

logfire-api/logfire_api/__init__.py

@@ -201,6 +201,8 @@ def instrument_system_metrics(self, *args, **kwargs) -> None: ...
 
             def instrument_mcp(self, *args, **kwargs) -> None: ...
 
+            def url_from_eval(self, *args, **kwargs) -> None: ...
+
             def shutdown(self, *args, **kwargs) -> None: ...
 
         DEFAULT_LOGFIRE_INSTANCE = Logfire()
@@ -254,6 +256,7 @@ def shutdown(self, *args, **kwargs) -> None: ...
         instrument_mcp = DEFAULT_LOGFIRE_INSTANCE.instrument_mcp
         shutdown = DEFAULT_LOGFIRE_INSTANCE.shutdown
         suppress_scopes = DEFAULT_LOGFIRE_INSTANCE.suppress_scopes
+        url_from_eval = DEFAULT_LOGFIRE_INSTANCE.url_from_eval
 
         def loguru_handler() -> dict[str, Any]:
             return {}

logfire/__init__.py

@@ -63,6 +63,7 @@
 with_tags = DEFAULT_LOGFIRE_INSTANCE.with_tags
 # with_trace_sample_rate = DEFAULT_LOGFIRE_INSTANCE.with_trace_sample_rate
 with_settings = DEFAULT_LOGFIRE_INSTANCE.with_settings
+url_from_eval = DEFAULT_LOGFIRE_INSTANCE.url_from_eval
 
 # Logging
 log = DEFAULT_LOGFIRE_INSTANCE.log
@@ -176,4 +177,5 @@ def loguru_handler() -> Any:
     'set_baggage',
     'get_context',
     'attach_context',
+    'url_from_eval',
 )

logfire/_internal/config.py

@@ -696,6 +696,7 @@ def _load_configuration(
         self.advanced = advanced
 
         self.additional_span_processors = additional_span_processors
+        self.project_url: str | None = None
 
         if metrics is None:
             metrics = MetricsOptions()
@@ -969,6 +970,7 @@ def add_span_processor(span_processor: SpanProcessor) -> None:
                     # This means that e.g. a token in an env var takes priority over a token in a creds file.
                     self.token = self.token or credentials.token
                     self.advanced.base_url = self.advanced.base_url or credentials.logfire_api_url
+                    self.project_url = self.project_url or credentials.project_url
 
                 if self.token:
                     # Convert to list for iteration (handles both str and list[str])
@@ -994,12 +996,10 @@ def check_tokens():
                         with suppress_instrumentation():
                             for token in token_list:
                                 validated_credentials = self._initialize_credentials_from_token(token)
-                                if (
-                                    validated_credentials is not None
-                                    and show_project_link
-                                    and token not in printed_tokens
-                                ):
-                                    validated_credentials.print_token_summary()
+                                if validated_credentials is not None:
+                                    self.project_url = self.project_url or validated_credentials.project_url
+                                    if show_project_link and token not in printed_tokens:
+                                        validated_credentials.print_token_summary()
 
                     if emscripten:  # pragma: no cover
                         check_tokens()

logfire/_internal/main.py

@@ -82,6 +82,7 @@
     from flask.app import Flask
     from opentelemetry.instrumentation.asgi.types import ClientRequestHook, ClientResponseHook, ServerRequestHook
     from opentelemetry.metrics import _Gauge as Gauge
+    from pydantic_evals.reporting import EvaluationReport
     from pymongo.monitoring import CommandFailedEvent, CommandStartedEvent, CommandSucceededEvent
     from sqlalchemy import Engine
     from sqlalchemy.ext.asyncio import AsyncEngine
@@ -876,6 +877,22 @@ def force_flush(self, timeout_millis: int = 3_000) -> bool:  # pragma: no cover
         """
         return self._config.force_flush(timeout_millis)
 
+    def url_from_eval(self, report: EvaluationReport[Any, Any, Any]) -> str | None:
+        """Generate a Logfire URL to view an evaluation report.
+
+        Args:
+            report: An evaluation report from `pydantic_evals`.
+
+        Returns:
+            The URL string, or `None` if the project URL or trace/span IDs are not available.
+        """
+        project_url = self._config.project_url
+        trace_id = report.trace_id
+        span_id = report.span_id
+        if not project_url or not trace_id or not span_id:
+            return None
+        return f'{project_url}/evals/compare?experiment={trace_id}-{span_id}'
+
     def log_slow_async_callbacks(self, slow_duration: float = 0.1) -> AbstractContextManager[None]:
         """Log a warning whenever a function running in the asyncio event loop blocks for too long.
 

pyproject.toml

@@ -180,6 +180,7 @@ dev = [
     "pytest-xdist>=3.6.1",
     "openai-agents[voice]>=0.0.7",
     "pydantic-ai-slim>=0.0.39",
+    "pydantic-evals>=0.0.39",
     "langchain>=0.0.27",
     "langchain-openai>=0.3.17",
     "langgraph >= 0",

tests/test_logfire_api.py

@@ -289,6 +289,10 @@ def func() -> None: ...
         pass
     logfire__all__.remove('attach_context')
 
+    assert hasattr(logfire_api, 'url_from_eval')
+    logfire_api.url_from_eval(MagicMock(trace_id='abc', span_id='def'))
+    logfire__all__.remove('url_from_eval')
+
     # If it's not empty, it means that some of the __all__ members are not tested.
     assert logfire__all__ == set(), logfire__all__
 

tests/test_url_from_eval.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import pytest
+
+try:
+    from pydantic_evals.reporting import EvaluationReport
+except Exception:
+    pytest.skip('pydantic_evals not importable (likely pydantic < 2.8)', allow_module_level=True)
+
+import logfire
+from logfire._internal.config import LogfireConfig
+
+
+def _make_report(trace_id: str | None = None, span_id: str | None = None) -> EvaluationReport:
+    return EvaluationReport(name='test', cases=[], trace_id=trace_id, span_id=span_id)
+
+
+def test_url_from_eval_with_project_url() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(trace_id='abc123', span_id='def456')
+    result = instance.url_from_eval(report)
+    assert result == 'https://logfire.pydantic.dev/my-org/my-project/evals/compare?experiment=abc123-def456'
+
+
+def test_url_from_eval_no_project_url() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(trace_id='abc123', span_id='def456')
+    result = instance.url_from_eval(report)
+    assert result is None
+
+
+def test_url_from_eval_no_trace_id() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(span_id='def456')
+    result = instance.url_from_eval(report)
+    assert result is None
+
+
+def test_url_from_eval_no_span_id() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report(trace_id='abc123')
+    result = instance.url_from_eval(report)
+    assert result is None
+
+
+def test_url_from_eval_no_ids() -> None:
+    config = LogfireConfig(send_to_logfire=False, console=False)
+    config.project_url = 'https://logfire.pydantic.dev/my-org/my-project'
+    instance = logfire.Logfire(config=config)
+
+    report = _make_report()
+    result = instance.url_from_eval(report)
+    assert result is None