diff --git a/logfire-api/logfire_api/__init__.py b/logfire-api/logfire_api/__init__.py index 8bd6e56dd..1562be1a3 100644 --- a/logfire-api/logfire_api/__init__.py +++ b/logfire-api/logfire_api/__init__.py @@ -201,6 +201,8 @@ def instrument_system_metrics(self, *args, **kwargs) -> None: ... def instrument_mcp(self, *args, **kwargs) -> None: ... + def url_from_eval(self, *args, **kwargs) -> None: ... + def shutdown(self, *args, **kwargs) -> None: ... DEFAULT_LOGFIRE_INSTANCE = Logfire() @@ -254,6 +256,7 @@ def shutdown(self, *args, **kwargs) -> None: ... instrument_mcp = DEFAULT_LOGFIRE_INSTANCE.instrument_mcp shutdown = DEFAULT_LOGFIRE_INSTANCE.shutdown suppress_scopes = DEFAULT_LOGFIRE_INSTANCE.suppress_scopes + url_from_eval = DEFAULT_LOGFIRE_INSTANCE.url_from_eval def loguru_handler() -> dict[str, Any]: return {} diff --git a/logfire/__init__.py b/logfire/__init__.py index af0efb8ac..962b26c70 100644 --- a/logfire/__init__.py +++ b/logfire/__init__.py @@ -63,6 +63,7 @@ with_tags = DEFAULT_LOGFIRE_INSTANCE.with_tags # with_trace_sample_rate = DEFAULT_LOGFIRE_INSTANCE.with_trace_sample_rate with_settings = DEFAULT_LOGFIRE_INSTANCE.with_settings +url_from_eval = DEFAULT_LOGFIRE_INSTANCE.url_from_eval # Logging log = DEFAULT_LOGFIRE_INSTANCE.log @@ -176,4 +177,5 @@ def loguru_handler() -> Any: 'set_baggage', 'get_context', 'attach_context', + 'url_from_eval', ) diff --git a/logfire/_internal/config.py b/logfire/_internal/config.py index 79126af8d..d0644e7d6 100644 --- a/logfire/_internal/config.py +++ b/logfire/_internal/config.py @@ -696,6 +696,8 @@ def _load_configuration( self.advanced = advanced self.additional_span_processors = additional_span_processors + self.project_url: str | None = None + self._check_tokens_thread: Thread | None = None if metrics is None: metrics = MetricsOptions() @@ -943,19 +945,22 @@ def add_span_processor(span_processor: SpanProcessor) -> None: if isinstance(self.metrics, MetricsOptions): metric_readers = list(self.metrics.additional_readers) + # Try loading credentials from a file. + # We do this before checking send_to_logfire so that project_url + # is available for url_from_eval even when not sending data. + try: + credentials = LogfireCredentials.load_creds_file(self.data_dir) + except Exception: + # If we have tokens configured by other means, e.g. the env, no need to worry about the creds file. + if self.send_to_logfire and not self.token: + raise + credentials = None + if credentials is not None: + self.project_url = self.project_url or credentials.project_url + if self.send_to_logfire: show_project_link: bool = self.console and self.console.show_project_link or False - # Try loading credentials from a file. - # If that works, we can use it to immediately print the project link. - try: - credentials = LogfireCredentials.load_creds_file(self.data_dir) - except Exception: - # If we have tokens configured by other means, e.g. the env, no need to worry about the creds file. - if not self.token: - raise - credentials = None - if not self.token and self.send_to_logfire is True and credentials is None: # If we don't have tokens or credentials from a file, # try initializing a new project and writing a new creds file. @@ -969,6 +974,7 @@ def add_span_processor(span_processor: SpanProcessor) -> None: # This means that e.g. a token in an env var takes priority over a token in a creds file. self.token = self.token or credentials.token self.advanced.base_url = self.advanced.base_url or credentials.logfire_api_url + self.project_url = self.project_url or credentials.project_url if self.token: # Convert to list for iteration (handles both str and list[str]) @@ -994,18 +1000,16 @@ def check_tokens(): with suppress_instrumentation(): for token in token_list: validated_credentials = self._initialize_credentials_from_token(token) - if ( - validated_credentials is not None - and show_project_link - and token not in printed_tokens - ): - validated_credentials.print_token_summary() + if validated_credentials is not None: + self.project_url = self.project_url or validated_credentials.project_url + if show_project_link and token not in printed_tokens: + validated_credentials.print_token_summary() if emscripten: # pragma: no cover check_tokens() else: - thread = Thread(target=check_tokens, name='check_logfire_token') - thread.start() + self._check_tokens_thread = Thread(target=check_tokens, name='check_logfire_token') + self._check_tokens_thread.start() # Create exporters for each token for token in token_list: @@ -1227,6 +1231,15 @@ def warn_if_not_initialized(self, message: str): category=LogfireNotConfiguredWarning, ) + def wait_for_token_validation(self) -> None: + """Wait for the background token validation thread to complete. + + This ensures that `project_url` is populated when the token is provided + via environment variable rather than a credentials file. + """ + if self._check_tokens_thread is not None: + self._check_tokens_thread.join() + def _initialize_credentials_from_token(self, token: str) -> LogfireCredentials | None: return LogfireCredentials.from_token(token, requests.Session(), self.advanced.generate_base_url(token)) diff --git a/logfire/_internal/main.py b/logfire/_internal/main.py index 04c38d8a1..b1af802c2 100644 --- a/logfire/_internal/main.py +++ b/logfire/_internal/main.py @@ -82,6 +82,7 @@ from flask.app import Flask from opentelemetry.instrumentation.asgi.types import ClientRequestHook, ClientResponseHook, ServerRequestHook from opentelemetry.metrics import _Gauge as Gauge + from pydantic_evals.reporting import EvaluationReport from pymongo.monitoring import CommandFailedEvent, CommandStartedEvent, CommandSucceededEvent from sqlalchemy import Engine from sqlalchemy.ext.asyncio import AsyncEngine @@ -876,6 +877,25 @@ def force_flush(self, timeout_millis: int = 3_000) -> bool: # pragma: no cover """ return self._config.force_flush(timeout_millis) + def url_from_eval(self, report: EvaluationReport[Any, Any, Any]) -> str | None: + """Generate a Logfire URL to view an evaluation report. + + Args: + report: An evaluation report from `pydantic_evals`. + + Returns: + The URL string, or `None` if the project URL or trace/span IDs are not available. + """ + # Wait for the background token validation thread to finish, + # since it may populate project_url when no credentials file exists. + self._config.wait_for_token_validation() + project_url = self._config.project_url + trace_id = report.trace_id + span_id = report.span_id + if not project_url or not trace_id or not span_id: + return None + return f'{project_url.rstrip("/")}/evals/compare?experiment={trace_id}-{span_id}' + def log_slow_async_callbacks(self, slow_duration: float = 0.1) -> AbstractContextManager[None]: """Log a warning whenever a function running in the asyncio event loop blocks for too long. diff --git a/pyproject.toml b/pyproject.toml index b966a02b7..3c232ec6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,6 +180,7 @@ dev = [ "pytest-xdist>=3.6.1", "openai-agents[voice]>=0.0.7", "pydantic-ai-slim>=0.0.39", + "pydantic-evals>=0.0.39", "langchain>=0.0.27", "langchain-openai>=0.3.17", "langgraph >= 0", diff --git a/tests/test_logfire_api.py b/tests/test_logfire_api.py index 198454104..7fa9e04ba 100644 --- a/tests/test_logfire_api.py +++ b/tests/test_logfire_api.py @@ -294,6 +294,10 @@ def func() -> None: ... pass logfire__all__.remove('attach_context') + assert hasattr(logfire_api, 'url_from_eval') + logfire_api.url_from_eval(MagicMock(trace_id='abc', span_id='def')) + logfire__all__.remove('url_from_eval') + # If it's not empty, it means that some of the __all__ members are not tested. assert logfire__all__ == set(), logfire__all__ diff --git a/tests/test_url_from_eval.py b/tests/test_url_from_eval.py new file mode 100644 index 000000000..11336c04c --- /dev/null +++ b/tests/test_url_from_eval.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import pytest +import requests_mock + +try: + from pydantic_evals.reporting import EvaluationReport +except Exception: + pytest.skip('pydantic_evals not importable (likely pydantic < 2.8)', allow_module_level=True) + +import logfire +from logfire._internal.config import LogfireConfig + + +def _make_report(trace_id: str | None = None, span_id: str | None = None) -> EvaluationReport: + return EvaluationReport(name='test', cases=[], trace_id=trace_id, span_id=span_id) + + +def test_url_from_eval_with_project_url() -> None: + config = LogfireConfig(send_to_logfire=False, console=False) + config.project_url = 'https://logfire.pydantic.dev/my-org/my-project' + instance = logfire.Logfire(config=config) + + report = _make_report(trace_id='abc123', span_id='def456') + result = instance.url_from_eval(report) + assert result == 'https://logfire.pydantic.dev/my-org/my-project/evals/compare?experiment=abc123-def456' + + +def test_url_from_eval_no_project_url() -> None: + config = LogfireConfig(send_to_logfire=False, console=False) + instance = logfire.Logfire(config=config) + + report = _make_report(trace_id='abc123', span_id='def456') + result = instance.url_from_eval(report) + assert result is None + + +def test_url_from_eval_no_trace_id() -> None: + config = LogfireConfig(send_to_logfire=False, console=False) + config.project_url = 'https://logfire.pydantic.dev/my-org/my-project' + instance = logfire.Logfire(config=config) + + report = _make_report(span_id='def456') + result = instance.url_from_eval(report) + assert result is None + + +def test_url_from_eval_no_span_id() -> None: + config = LogfireConfig(send_to_logfire=False, console=False) + config.project_url = 'https://logfire.pydantic.dev/my-org/my-project' + instance = logfire.Logfire(config=config) + + report = _make_report(trace_id='abc123') + result = instance.url_from_eval(report) + assert result is None + + +def test_url_from_eval_trailing_slash() -> None: + config = LogfireConfig(send_to_logfire=False, console=False) + config.project_url = 'https://logfire.pydantic.dev/my-org/my-project/' + instance = logfire.Logfire(config=config) + + report = _make_report(trace_id='abc123', span_id='def456') + result = instance.url_from_eval(report) + assert result == 'https://logfire.pydantic.dev/my-org/my-project/evals/compare?experiment=abc123-def456' + + +def test_url_from_eval_no_ids() -> None: + config = LogfireConfig(send_to_logfire=False, console=False) + config.project_url = 'https://logfire.pydantic.dev/my-org/my-project' + instance = logfire.Logfire(config=config) + + report = _make_report() + result = instance.url_from_eval(report) + assert result is None + + +def test_url_from_eval_waits_for_token_validation() -> None: + """Test that url_from_eval waits for the background token validation thread + to populate project_url when the token is provided directly (no creds file).""" + with requests_mock.Mocker() as mocker: + mocker.get( + 'https://logfire-us.pydantic.dev/v1/info', + json={ + 'project_name': 'myproject', + 'project_url': 'https://logfire-us.pydantic.dev/my-org/my-project', + }, + ) + logfire.configure( + send_to_logfire=True, + token='fake-token', + console=False, + ) + + report = _make_report(trace_id='abc123', span_id='def456') + # url_from_eval should wait for the background thread and return the URL + result = logfire.url_from_eval(report) + assert result == 'https://logfire-us.pydantic.dev/my-org/my-project/evals/compare?experiment=abc123-def456' diff --git a/uv.lock b/uv.lock index 9a5ab986a..538b4646b 100644 --- a/uv.lock +++ b/uv.lock @@ -659,7 +659,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and implementation_name != 'PyPy' and platform_python_implementation != 'PyPy'" }, { name = "pycparser", version = "3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } @@ -1507,7 +1507,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -3333,6 +3333,8 @@ dev = [ { name = "pydantic" }, { name = "pydantic-ai-slim", version = "0.8.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pydantic-ai-slim", version = "1.56.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pydantic-evals", version = "0.8.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pydantic-evals", version = "1.56.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pymongo" }, { name = "pymysql" }, { name = "pyright" }, @@ -3485,6 +3487,7 @@ dev = [ { name = "pyarrow", marker = "python_full_version >= '3.13'", specifier = ">=18.1.0" }, { name = "pydantic", specifier = ">=2.11.0" }, { name = "pydantic-ai-slim", specifier = ">=0.0.39" }, + { name = "pydantic-evals", specifier = ">=0.0.39" }, { name = "pymongo", specifier = ">=4.10.1" }, { name = "pymysql", specifier = ">=1.1.1" }, { name = "pyright", specifier = "!=1.1.407" }, @@ -6687,6 +6690,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] +[[package]] +name = "pydantic-evals" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10' and platform_python_implementation == 'PyPy'", + "python_full_version < '3.10' and platform_python_implementation != 'PyPy'", +] +dependencies = [ + { name = "anyio", marker = "python_full_version < '3.10'" }, + { name = "eval-type-backport", marker = "python_full_version < '3.10'" }, + { name = "logfire-api", marker = "python_full_version < '3.10'" }, + { name = "pydantic", marker = "python_full_version < '3.10'" }, + { name = "pydantic-ai-slim", version = "0.8.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pyyaml", marker = "python_full_version < '3.10'" }, + { name = "rich", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/9d/460a1f2c9f5f263e9d8e9661acbd654ccc81ad3373ea43048d914091a817/pydantic_evals-0.8.1.tar.gz", hash = "sha256:c398a623c31c19ce70e346ad75654fcb1517c3f6a821461f64fe5cbbe0813023", size = 43933, upload-time = "2025-08-29T14:46:28.903Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/f9/1d21c4687167c4fa76fd3b1ed47f9bc2d38fd94cbacd9aa3f19e82e59830/pydantic_evals-0.8.1-py3-none-any.whl", hash = "sha256:6c76333b1d79632f619eb58a24ac656e9f402c47c75ad750ba0230d7f5514344", size = 52602, upload-time = "2025-08-29T14:46:16.602Z" }, +] + +[[package]] +name = "pydantic-evals" +version = "1.56.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "anyio", marker = "python_full_version >= '3.10'" }, + { name = "logfire-api", marker = "python_full_version >= '3.10'" }, + { name = "pydantic", marker = "python_full_version >= '3.10'" }, + { name = "pydantic-ai-slim", version = "1.56.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "rich", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/f2/8c59284a2978af3fbda45ae3217218eaf8b071207a9290b54b7613983e5d/pydantic_evals-1.56.0.tar.gz", hash = "sha256:206635107127af6a3ee4b1fc8f77af6afb14683615a2d6b3609f79467c1c0d28", size = 47210, upload-time = "2026-02-06T01:13:25.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/51/9875d19ff6d584aaeb574aba76b49d931b822546fc60b29c4fc0da98170d/pydantic_evals-1.56.0-py3-none-any.whl", hash = "sha256:d1efb410c97135aabd2a22453b10c981b2b9851985e9354713af67ae0973b7a9", size = 56407, upload-time = "2026-02-06T01:13:17.098Z" }, +] + [[package]] name = "pydantic-graph" version = "0.8.1"