diff --git a/logfire/_internal/integrations/llm_providers/anthropic.py b/logfire/_internal/integrations/llm_providers/anthropic.py index 35c03df7e..01e22be99 100644 --- a/logfire/_internal/integrations/llm_providers/anthropic.py +++ b/logfire/_internal/integrations/llm_providers/anthropic.py @@ -1,12 +1,23 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +import json +from typing import TYPE_CHECKING, Any, cast import anthropic from anthropic.types import Message, TextBlock, TextDelta from logfire._internal.utils import handle_internal_errors +from .semconv import ( + OPERATION_NAME, + PROVIDER_NAME, + REQUEST_MAX_TOKENS, + REQUEST_STOP_SEQUENCES, + REQUEST_TEMPERATURE, + REQUEST_TOP_K, + REQUEST_TOP_P, + TOOL_DEFINITIONS, +) from .types import EndpointConfig, StreamState if TYPE_CHECKING: @@ -22,24 +33,58 @@ ) +def _extract_request_parameters(json_data: dict[str, Any], span_data: dict[str, Any]) -> None: + """Extract request parameters from json_data and add to span_data.""" + if (max_tokens := json_data.get('max_tokens')) is not None: + span_data[REQUEST_MAX_TOKENS] = max_tokens + + if (temperature := json_data.get('temperature')) is not None: + span_data[REQUEST_TEMPERATURE] = temperature + + if (top_p := json_data.get('top_p')) is not None: + span_data[REQUEST_TOP_P] = top_p + + if (top_k := json_data.get('top_k')) is not None: + span_data[REQUEST_TOP_K] = top_k + + if (stop_sequences := json_data.get('stop_sequences')) is not None: + span_data[REQUEST_STOP_SEQUENCES] = json.dumps(stop_sequences) + + if (tools := json_data.get('tools')) is not None: + span_data[TOOL_DEFINITIONS] = json.dumps(tools) + + def get_endpoint_config(options: FinalRequestOptions) -> EndpointConfig: """Returns the endpoint config for Anthropic or Bedrock depending on the url.""" url = options.url - json_data = options.json_data - if not isinstance(json_data, dict): # pragma: no cover + raw_json_data = options.json_data + if not isinstance(raw_json_data, dict): # pragma: no cover # Ensure that `{request_data[model]!r}` doesn't raise an error, just a warning about `model` missing. - json_data = {} + raw_json_data = {} + json_data = cast('dict[str, Any]', raw_json_data) if url == '/v1/messages': + span_data: dict[str, Any] = { + 'request_data': json_data, + PROVIDER_NAME: 'anthropic', + OPERATION_NAME: 'chat', + } + _extract_request_parameters(json_data, span_data) + return EndpointConfig( message_template='Message with {request_data[model]!r}', - span_data={'request_data': json_data}, + span_data=span_data, stream_state_cls=AnthropicMessageStreamState, ) else: + span_data = { + 'request_data': json_data, + 'url': url, + PROVIDER_NAME: 'anthropic', + } return EndpointConfig( message_template='Anthropic API call to {url!r}', - span_data={'request_data': json_data, 'url': url}, + span_data=span_data, ) diff --git a/logfire/_internal/integrations/llm_providers/llm_provider.py b/logfire/_internal/integrations/llm_providers/llm_provider.py index 1514ca2bb..5f56784e2 100644 --- a/logfire/_internal/integrations/llm_providers/llm_provider.py +++ b/logfire/_internal/integrations/llm_providers/llm_provider.py @@ -4,6 +4,8 @@ from contextlib import AbstractContextManager, ExitStack, contextmanager, nullcontext from typing import TYPE_CHECKING, Any, Callable, cast +from opentelemetry.trace import SpanKind + from logfire import attach_context, get_context from logfire.propagate import ContextCarrier @@ -136,7 +138,7 @@ def instrumented_llm_request_sync(*args: Any, **kwargs: Any) -> Any: message_template, span_data, kwargs = _instrumentation_setup(*args, **kwargs) if message_template is None: return original_request_method(*args, **kwargs) - with logfire_llm.span(message_template, **span_data) as span: + with logfire_llm.span(message_template, _span_kind=SpanKind.CLIENT, **span_data) as span: with maybe_suppress_instrumentation(suppress_otel): if kwargs.get('stream'): return original_request_method(*args, **kwargs) @@ -148,7 +150,7 @@ async def instrumented_llm_request_async(*args: Any, **kwargs: Any) -> Any: message_template, span_data, kwargs = _instrumentation_setup(*args, **kwargs) if message_template is None: return await original_request_method(*args, **kwargs) - with logfire_llm.span(message_template, **span_data) as span: + with logfire_llm.span(message_template, _span_kind=SpanKind.CLIENT, **span_data) as span: with maybe_suppress_instrumentation(suppress_otel): if kwargs.get('stream'): return await original_request_method(*args, **kwargs) diff --git a/logfire/_internal/integrations/llm_providers/openai.py b/logfire/_internal/integrations/llm_providers/openai.py index 40c53d8bd..260617abb 100644 --- a/logfire/_internal/integrations/llm_providers/openai.py +++ b/logfire/_internal/integrations/llm_providers/openai.py @@ -18,6 +18,19 @@ from logfire import LogfireSpan from ...utils import handle_internal_errors, log_internal_error +from .semconv import ( + OPERATION_NAME, + PROVIDER_NAME, + REQUEST_FREQUENCY_PENALTY, + REQUEST_MAX_TOKENS, + REQUEST_MODEL, + REQUEST_PRESENCE_PENALTY, + REQUEST_SEED, + REQUEST_STOP_SEQUENCES, + REQUEST_TEMPERATURE, + REQUEST_TOP_P, + TOOL_DEFINITIONS, +) from .types import EndpointConfig, StreamState if TYPE_CHECKING: @@ -33,37 +46,81 @@ ) +def _extract_request_parameters(json_data: dict[str, Any], span_data: dict[str, Any]) -> None: + """Extract request parameters from json_data and add to span_data.""" + if (max_tokens := json_data.get('max_tokens')) is not None: + span_data[REQUEST_MAX_TOKENS] = max_tokens + elif (max_output_tokens := json_data.get('max_output_tokens')) is not None: + span_data[REQUEST_MAX_TOKENS] = max_output_tokens + + if (temperature := json_data.get('temperature')) is not None: + span_data[REQUEST_TEMPERATURE] = temperature + + if (top_p := json_data.get('top_p')) is not None: + span_data[REQUEST_TOP_P] = top_p + + if (stop := json_data.get('stop')) is not None: + if isinstance(stop, str): + span_data[REQUEST_STOP_SEQUENCES] = json.dumps([stop]) + else: + span_data[REQUEST_STOP_SEQUENCES] = json.dumps(stop) + + if (seed := json_data.get('seed')) is not None: + span_data[REQUEST_SEED] = seed + + if (frequency_penalty := json_data.get('frequency_penalty')) is not None: + span_data[REQUEST_FREQUENCY_PENALTY] = frequency_penalty + + if (presence_penalty := json_data.get('presence_penalty')) is not None: + span_data[REQUEST_PRESENCE_PENALTY] = presence_penalty + + if (tools := json_data.get('tools')) is not None: + span_data[TOOL_DEFINITIONS] = json.dumps(tools) + + def get_endpoint_config(options: FinalRequestOptions) -> EndpointConfig: """Returns the endpoint config for OpenAI depending on the url.""" url = options.url - json_data = options.json_data - if not isinstance(json_data, dict): # pragma: no cover + raw_json_data = options.json_data + if not isinstance(raw_json_data, dict): # pragma: no cover # Ensure that `{request_data[model]!r}` doesn't raise an error, just a warning about `model` missing. - json_data = {} + raw_json_data = {} + json_data = cast('dict[str, Any]', raw_json_data) if url == '/chat/completions': if is_current_agent_span('Chat completion with {gen_ai.request.model!r}'): return EndpointConfig(message_template='', span_data={}) + span_data: dict[str, Any] = { + 'request_data': json_data, + 'gen_ai.request.model': json_data.get('model'), + PROVIDER_NAME: 'openai', + OPERATION_NAME: 'chat', + } + _extract_request_parameters(json_data, span_data) + return EndpointConfig( message_template='Chat Completion with {request_data[model]!r}', - span_data={'request_data': json_data, 'gen_ai.request.model': json_data['model']}, + span_data=span_data, stream_state_cls=OpenaiChatCompletionStreamState, ) elif url == '/responses': if is_current_agent_span('Responses API', 'Responses API with {gen_ai.request.model!r}'): return EndpointConfig(message_template='', span_data={}) - stream = json_data.get('stream', False) # type: ignore - span_data: dict[str, Any] = { - 'gen_ai.request.model': json_data['model'], - 'request_data': {'model': json_data['model'], 'stream': stream}, + stream = json_data.get('stream', False) + span_data = { + 'gen_ai.request.model': json_data.get('model'), + 'request_data': {'model': json_data.get('model'), 'stream': stream}, 'events': inputs_to_events( - json_data['input'], # type: ignore - json_data.get('instructions'), # type: ignore + json_data.get('input'), + json_data.get('instructions'), ), + PROVIDER_NAME: 'openai', + OPERATION_NAME: 'chat', } + _extract_request_parameters(json_data, span_data) return EndpointConfig( message_template='Responses API with {gen_ai.request.model!r}', @@ -71,25 +128,51 @@ def get_endpoint_config(options: FinalRequestOptions) -> EndpointConfig: stream_state_cls=OpenaiResponsesStreamState, ) elif url == '/completions': + span_data = { + 'request_data': json_data, + 'gen_ai.request.model': json_data.get('model'), + PROVIDER_NAME: 'openai', + OPERATION_NAME: 'text_completion', + } + _extract_request_parameters(json_data, span_data) return EndpointConfig( message_template='Completion with {request_data[model]!r}', - span_data={'request_data': json_data, 'gen_ai.request.model': json_data['model']}, + span_data=span_data, stream_state_cls=OpenaiCompletionStreamState, ) elif url == '/embeddings': + span_data = { + 'request_data': json_data, + 'gen_ai.request.model': json_data.get('model'), + PROVIDER_NAME: 'openai', + OPERATION_NAME: 'embeddings', + } + _extract_request_parameters(json_data, span_data) return EndpointConfig( message_template='Embedding Creation with {request_data[model]!r}', - span_data={'request_data': json_data, 'gen_ai.request.model': json_data['model']}, + span_data=span_data, ) elif url == '/images/generations': + span_data = { + 'request_data': json_data, + 'gen_ai.request.model': json_data.get('model'), + PROVIDER_NAME: 'openai', + OPERATION_NAME: 'image_generation', + } + _extract_request_parameters(json_data, span_data) return EndpointConfig( message_template='Image Generation with {request_data[model]!r}', - span_data={'request_data': json_data, 'gen_ai.request.model': json_data['model']}, + span_data=span_data, ) else: - span_data = {'request_data': json_data, 'url': url} + span_data = { + 'request_data': json_data, + 'url': url, + PROVIDER_NAME: 'openai', + } if 'model' in json_data: - span_data['gen_ai.request.model'] = json_data['model'] + span_data[REQUEST_MODEL] = json_data['model'] + _extract_request_parameters(json_data, span_data) return EndpointConfig( message_template='OpenAI API call to {url!r}', span_data=span_data, diff --git a/logfire/_internal/integrations/llm_providers/semconv.py b/logfire/_internal/integrations/llm_providers/semconv.py new file mode 100644 index 000000000..1f89909f3 --- /dev/null +++ b/logfire/_internal/integrations/llm_providers/semconv.py @@ -0,0 +1,44 @@ +"""Gen AI Semantic Convention attribute names. + +These constants follow the OpenTelemetry Gen AI Semantic Conventions. +See: https://opentelemetry.io/docs/specs/semconv/gen-ai/ +""" + +from __future__ import annotations + +# Provider and operation +PROVIDER_NAME = 'gen_ai.provider.name' +OPERATION_NAME = 'gen_ai.operation.name' + +# Model information +REQUEST_MODEL = 'gen_ai.request.model' +RESPONSE_MODEL = 'gen_ai.response.model' + +# Request parameters +REQUEST_MAX_TOKENS = 'gen_ai.request.max_tokens' +REQUEST_TEMPERATURE = 'gen_ai.request.temperature' +REQUEST_TOP_P = 'gen_ai.request.top_p' +REQUEST_TOP_K = 'gen_ai.request.top_k' +REQUEST_STOP_SEQUENCES = 'gen_ai.request.stop_sequences' +REQUEST_SEED = 'gen_ai.request.seed' +REQUEST_FREQUENCY_PENALTY = 'gen_ai.request.frequency_penalty' +REQUEST_PRESENCE_PENALTY = 'gen_ai.request.presence_penalty' + +# Response metadata +RESPONSE_ID = 'gen_ai.response.id' +RESPONSE_FINISH_REASONS = 'gen_ai.response.finish_reasons' + +# Token usage +INPUT_TOKENS = 'gen_ai.usage.input_tokens' +OUTPUT_TOKENS = 'gen_ai.usage.output_tokens' + +# Message content +INPUT_MESSAGES = 'gen_ai.input.messages' +OUTPUT_MESSAGES = 'gen_ai.output.messages' +SYSTEM_INSTRUCTIONS = 'gen_ai.system_instructions' + +# Tool definitions +TOOL_DEFINITIONS = 'gen_ai.tool.definitions' + +# Conversation tracking +CONVERSATION_ID = 'gen_ai.conversation.id' diff --git a/logfire/_internal/main.py b/logfire/_internal/main.py index 8ecd21b3e..9997f9c0e 100644 --- a/logfire/_internal/main.py +++ b/logfire/_internal/main.py @@ -26,7 +26,7 @@ from opentelemetry.context import Context from opentelemetry.metrics import CallbackT, Counter, Histogram, UpDownCounter from opentelemetry.sdk.trace import ReadableSpan, Span -from opentelemetry.trace import SpanContext +from opentelemetry.trace import SpanContext, SpanKind from opentelemetry.util import types as otel_types from typing_extensions import LiteralString, ParamSpec @@ -188,6 +188,7 @@ def _span( _span_name: str | None = None, _level: LevelName | int | None = None, _links: Sequence[tuple[SpanContext, otel_types.Attributes]] = (), + _span_kind: SpanKind = SpanKind.INTERNAL, ) -> LogfireSpan: try: if _level is not None: @@ -243,6 +244,7 @@ def _span( self._spans_tracer, json_schema_properties, links=_links, + span_kind=_span_kind, ) except Exception: log_internal_error() @@ -540,6 +542,7 @@ def span( _span_name: str | None = None, _level: LevelName | None = None, _links: Sequence[tuple[SpanContext, otel_types.Attributes]] = (), + _span_kind: SpanKind = SpanKind.INTERNAL, **attributes: Any, ) -> LogfireSpan: """Context manager for creating a span. @@ -559,6 +562,10 @@ def span( _tags: An optional sequence of tags to include in the span. _level: An optional log level name. _links: An optional sequence of links to other spans. Each link is a tuple of a span context and attributes. + _span_kind: The [OpenTelemetry span kind](https://opentelemetry.io/docs/concepts/signals/traces/#span-kind). + If not provided, defaults to `INTERNAL`. + Users don't typically need to set this. + Not related to the `kind` column of the `records` table in Logfire. attributes: The arguments to include in the span and format the message template with. Attributes starting with an underscore are not allowed. """ @@ -571,6 +578,7 @@ def span( _span_name=_span_name, _level=_level, _links=_links, + _span_kind=_span_kind, ) @overload @@ -2386,12 +2394,14 @@ def __init__( tracer: _ProxyTracer, json_schema_properties: JsonSchemaProperties, links: Sequence[tuple[SpanContext, otel_types.Attributes]], + span_kind: SpanKind = SpanKind.INTERNAL, ) -> None: self._span_name = span_name self._otlp_attributes = otlp_attributes self._tracer = tracer self._json_schema_properties = json_schema_properties self._links = list(trace_api.Link(context=context, attributes=attributes) for context, attributes in links) + self._span_kind = span_kind self._added_attributes = False self._token: None | Token[Context] = None @@ -2410,6 +2420,7 @@ def _start(self): name=self._span_name, attributes=self._otlp_attributes, links=self._links, + kind=self._span_kind, ) @handle_internal_errors diff --git a/tests/otel_integrations/test_anthropic.py b/tests/otel_integrations/test_anthropic.py index 24cbaca09..a5b67a256 100644 --- a/tests/otel_integrations/test_anthropic.py +++ b/tests/otel_integrations/test_anthropic.py @@ -2,7 +2,7 @@ import json from collections.abc import AsyncIterator, Iterator -from typing import Any +from typing import Any, cast import anthropic import httpx @@ -160,6 +160,9 @@ def test_sync_messages(instrumented_client: anthropic.Anthropic, exporter: TestE } ) ), + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'async': False, 'logfire.msg_template': 'Message with {request_data[model]!r}', 'logfire.msg': "Message with 'claude-3-haiku-20240307'", @@ -192,6 +195,9 @@ def test_sync_messages(instrumented_client: anthropic.Anthropic, exporter: TestE 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.request.max_tokens': {}, 'async': {}, 'response_data': { 'type': 'object', @@ -242,6 +248,9 @@ async def test_async_messages(instrumented_async_client: anthropic.AsyncAnthropi 'model': 'claude-3-haiku-20240307', } ), + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'async': True, 'logfire.msg_template': 'Message with {request_data[model]!r}', 'logfire.msg': "Message with 'claude-3-haiku-20240307'", @@ -273,6 +282,9 @@ async def test_async_messages(instrumented_async_client: anthropic.AsyncAnthropi 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.request.max_tokens': {}, 'async': {}, 'response_data': { 'type': 'object', @@ -316,10 +328,13 @@ def test_sync_message_empty_response_chunk(instrumented_client: anthropic.Anthro 'code.function': 'test_sync_message_empty_response_chunk', 'code.lineno': 123, 'request_data': '{"max_tokens":1000,"messages":[],"model":"claude-3-haiku-20240307","stream":true,"system":"empty response chunk"}', + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'async': False, 'logfire.msg_template': 'Message with {request_data[model]!r}', 'logfire.msg': "Message with 'claude-3-haiku-20240307'", - 'logfire.json_schema': '{"type":"object","properties":{"request_data":{"type":"object"},"async":{}}}', + 'logfire.json_schema': '{"type":"object","properties":{"request_data":{"type":"object"},"gen_ai.provider.name":{},"gen_ai.operation.name":{},"gen_ai.request.max_tokens":{},"async":{}}}', 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), }, @@ -340,10 +355,13 @@ def test_sync_message_empty_response_chunk(instrumented_client: anthropic.Anthro 'code.lineno': 123, 'logfire.msg': "streaming response from 'claude-3-haiku-20240307' took 1.00s", 'logfire.span_type': 'log', + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': '{"combined_chunk_content":"","chunk_count":0}', - 'logfire.json_schema': '{"type":"object","properties":{"duration":{},"request_data":{"type":"object"},"async":{},"response_data":{"type":"object"}}}', + 'logfire.json_schema': '{"type":"object","properties":{"duration":{},"request_data":{"type":"object"},"gen_ai.provider.name":{},"gen_ai.operation.name":{},"gen_ai.request.max_tokens":{},"async":{},"response_data":{"type":"object"}}}', }, }, ] @@ -378,10 +396,13 @@ def test_sync_messages_stream(instrumented_client: anthropic.Anthropic, exporter 'code.function': 'test_sync_messages_stream', 'code.lineno': 123, 'request_data': '{"max_tokens":1000,"messages":[{"role":"user","content":"What is four plus five?"}],"model":"claude-3-haiku-20240307","stream":true,"system":"You are a helpful assistant."}', + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'async': False, 'logfire.msg_template': 'Message with {request_data[model]!r}', 'logfire.msg': "Message with 'claude-3-haiku-20240307'", - 'logfire.json_schema': '{"type":"object","properties":{"request_data":{"type":"object"},"async":{}}}', + 'logfire.json_schema': '{"type":"object","properties":{"request_data":{"type":"object"},"gen_ai.provider.name":{},"gen_ai.operation.name":{},"gen_ai.request.max_tokens":{},"async":{}}}', 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), }, @@ -402,10 +423,13 @@ def test_sync_messages_stream(instrumented_client: anthropic.Anthropic, exporter 'code.lineno': 123, 'logfire.msg': "streaming response from 'claude-3-haiku-20240307' took 1.00s", 'logfire.span_type': 'log', + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': '{"combined_chunk_content":"The answer is secret","chunk_count":2}', - 'logfire.json_schema': '{"type":"object","properties":{"duration":{},"request_data":{"type":"object"},"async":{},"response_data":{"type":"object"}}}', + 'logfire.json_schema': '{"type":"object","properties":{"duration":{},"request_data":{"type":"object"},"gen_ai.provider.name":{},"gen_ai.operation.name":{},"gen_ai.request.max_tokens":{},"async":{},"response_data":{"type":"object"}}}', }, }, ] @@ -443,10 +467,13 @@ async def test_async_messages_stream( 'code.function': 'test_async_messages_stream', 'code.lineno': 123, 'request_data': '{"max_tokens":1000,"messages":[{"role":"user","content":"What is four plus five?"}],"model":"claude-3-haiku-20240307","stream":true,"system":"You are a helpful assistant."}', + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'async': True, 'logfire.msg_template': 'Message with {request_data[model]!r}', 'logfire.msg': "Message with 'claude-3-haiku-20240307'", - 'logfire.json_schema': '{"type":"object","properties":{"request_data":{"type":"object"},"async":{}}}', + 'logfire.json_schema': '{"type":"object","properties":{"request_data":{"type":"object"},"gen_ai.provider.name":{},"gen_ai.operation.name":{},"gen_ai.request.max_tokens":{},"async":{}}}', 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), }, @@ -467,10 +494,13 @@ async def test_async_messages_stream( 'code.lineno': 123, 'logfire.msg': "streaming response from 'claude-3-haiku-20240307' took 1.00s", 'logfire.span_type': 'log', + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': '{"combined_chunk_content":"The answer is secret","chunk_count":2}', - 'logfire.json_schema': '{"type":"object","properties":{"duration":{},"request_data":{"type":"object"},"async":{},"response_data":{"type":"object"}}}', + 'logfire.json_schema': '{"type":"object","properties":{"duration":{},"request_data":{"type":"object"},"gen_ai.provider.name":{},"gen_ai.operation.name":{},"gen_ai.request.max_tokens":{},"async":{},"response_data":{"type":"object"}}}', }, }, ] @@ -504,6 +534,9 @@ def test_tool_messages(instrumented_client: anthropic.Anthropic, exporter: TestE 'model': 'claude-3-haiku-20240307', 'system': 'tool response', }, + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'async': False, 'logfire.msg_template': 'Message with {request_data[model]!r}', 'logfire.msg': "Message with 'claude-3-haiku-20240307'", @@ -530,6 +563,9 @@ def test_tool_messages(instrumented_client: anthropic.Anthropic, exporter: TestE 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.request.max_tokens': {}, 'async': {}, 'response_data': { 'type': 'object', @@ -562,6 +598,7 @@ def test_unknown_method(instrumented_client: anthropic.Anthropic, exporter: Test 'request_data': '{"max_tokens_to_sample":1000,"model":"claude-2.1","prompt":"prompt"}', 'url': '/v1/complete', 'async': False, + 'gen_ai.provider.name': 'anthropic', 'logfire.msg_template': 'Anthropic API call to {url!r}', 'logfire.msg': "Anthropic API call to '/v1/complete'", 'code.filepath': 'test_anthropic.py', @@ -572,3 +609,58 @@ def test_unknown_method(instrumented_client: anthropic.Anthropic, exporter: Test } ] ) + + +def test_request_parameters(instrumented_client: anthropic.Anthropic, exporter: TestExporter) -> None: + """Test that all request parameters are extracted and added to span attributes.""" + tools: list[Any] = [ + { + 'name': 'get_weather', + 'description': 'Get the current weather', + 'input_schema': { + 'type': 'object', + 'properties': {'location': {'type': 'string'}}, + 'required': ['location'], + }, + } + ] + response = instrumented_client.messages.create( + max_tokens=1000, + model='claude-3-haiku-20240307', + system='You are a helpful assistant.', + messages=[{'role': 'user', 'content': 'What is four plus five?'}], + temperature=0.7, + top_p=0.9, + top_k=40, + stop_sequences=['END', 'STOP'], + tools=cast(Any, tools), + ) + assert isinstance(response.content[0], TextBlock) + assert response.content[0].text == 'Nine' + + spans = exporter.exported_spans_as_dict() + assert len(spans) == 1 + attributes = spans[0]['attributes'] + + # Verify all request parameters are present + assert attributes['gen_ai.request.max_tokens'] == 1000 + assert attributes['gen_ai.request.temperature'] == 0.7 + assert attributes['gen_ai.request.top_p'] == 0.9 + assert attributes['gen_ai.request.top_k'] == 40 + assert attributes['gen_ai.request.stop_sequences'] == '["END", "STOP"]' + assert json.loads(attributes['gen_ai.tool.definitions']) == tools + + +def test_extract_request_parameters_without_max_tokens() -> None: + """Test _extract_request_parameters when max_tokens is not in json_data (covers branch 37->40).""" + from logfire._internal.integrations.llm_providers.anthropic import ( + _extract_request_parameters, # pyright: ignore[reportPrivateUsage] + ) + + # Test with no max_tokens - covers the branch where max_tokens is None + json_data: dict[str, Any] = {'temperature': 0.5} + span_data: dict[str, Any] = {} + _extract_request_parameters(json_data, span_data) + + assert span_data.get('gen_ai.request.temperature') == 0.5 + assert 'gen_ai.request.max_tokens' not in span_data diff --git a/tests/otel_integrations/test_anthropic_bedrock.py b/tests/otel_integrations/test_anthropic_bedrock.py index ab7884f6c..f625fa19f 100644 --- a/tests/otel_integrations/test_anthropic_bedrock.py +++ b/tests/otel_integrations/test_anthropic_bedrock.py @@ -89,6 +89,9 @@ def test_sync_messages(mock_client: AnthropicBedrock, exporter: TestExporter): 'model': model_id, } ), + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.max_tokens': 1000, 'async': False, 'logfire.msg_template': 'Message with {request_data[model]!r}', 'logfire.msg': f"Message with '{model_id}'", @@ -120,6 +123,9 @@ def test_sync_messages(mock_client: AnthropicBedrock, exporter: TestExporter): 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.request.max_tokens': {}, 'async': {}, 'response_data': { 'type': 'object', diff --git a/tests/otel_integrations/test_openai.py b/tests/otel_integrations/test_openai.py index 7b36ad7ac..acb46008b 100644 --- a/tests/otel_integrations/test_openai.py +++ b/tests/otel_integrations/test_openai.py @@ -412,82 +412,139 @@ def test_sync_chat_completions(instrumented_client: openai.Client, exporter: Tes 'code.filepath': 'test_openai.py', 'code.function': 'test_sync_chat_completions', 'code.lineno': 123, - 'request_data': ( - { - 'messages': [ - {'role': 'system', 'content': 'You are a helpful assistant.'}, - {'role': 'user', 'content': 'What is four plus five?'}, - ], - 'model': 'gpt-4', - } - ), + 'request_data': { + 'messages': [ + {'role': 'system', 'content': 'You are a helpful assistant.'}, + {'role': 'user', 'content': 'What is four plus five?'}, + ], + 'model': 'gpt-4', + }, + 'gen_ai.provider.name': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', 'async': False, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', + 'gen_ai.system': 'openai', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', 'gen_ai.response.model': 'gpt-4', + 'operation.cost': 0.00012, 'gen_ai.usage.input_tokens': 2, 'gen_ai.usage.output_tokens': 1, - 'operation.cost': 0.00012, - 'response_data': ( - { - 'message': { - 'content': 'Nine', - 'refusal': None, - 'audio': None, - 'annotations': None, - 'role': 'assistant', - 'function_call': None, - 'tool_calls': None, - }, - 'usage': { - 'completion_tokens': 1, - 'prompt_tokens': 2, - 'total_tokens': 3, - 'completion_tokens_details': None, - 'prompt_tokens_details': None, - }, - } - ), - 'logfire.json_schema': ( - { - 'type': 'object', - 'properties': { - 'request_data': {'type': 'object'}, - 'async': {}, - 'gen_ai.system': {}, - 'gen_ai.request.model': {}, - 'gen_ai.response.model': {}, - 'gen_ai.usage.input_tokens': {}, - 'gen_ai.usage.output_tokens': {}, - 'operation.cost': {}, - 'response_data': { - 'type': 'object', - 'properties': { - 'message': { - 'type': 'object', - 'title': 'ChatCompletionMessage', - 'x-python-datatype': 'PydanticModel', - }, - 'usage': { - 'type': 'object', - 'title': 'CompletionUsage', - 'x-python-datatype': 'PydanticModel', - }, + 'response_data': { + 'message': { + 'content': 'Nine', + 'refusal': None, + 'audio': None, + 'annotations': None, + 'role': 'assistant', + 'function_call': None, + 'tool_calls': None, + }, + 'usage': { + 'completion_tokens': 1, + 'prompt_tokens': 2, + 'total_tokens': 3, + 'completion_tokens_details': None, + 'prompt_tokens_details': None, + }, + }, + 'logfire.json_schema': { + 'type': 'object', + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.system': {}, + 'async': {}, + 'gen_ai.response.model': {}, + 'operation.cost': {}, + 'gen_ai.usage.input_tokens': {}, + 'gen_ai.usage.output_tokens': {}, + 'response_data': { + 'type': 'object', + 'properties': { + 'message': { + 'type': 'object', + 'title': 'ChatCompletionMessage', + 'x-python-datatype': 'PydanticModel', + }, + 'usage': { + 'type': 'object', + 'title': 'CompletionUsage', + 'x-python-datatype': 'PydanticModel', }, }, }, - } - ), + }, + }, }, } ] ) +def test_sync_chat_completions_with_all_request_params( + instrumented_client: openai.Client, exporter: TestExporter +) -> None: + """Test that all optional request parameters are extracted to span attributes.""" + response = instrumented_client.chat.completions.create( + model='gpt-4', + messages=[ + {'role': 'user', 'content': 'What is four plus five?'}, + ], + max_tokens=100, + temperature=0.7, + top_p=0.9, + stop=['END', 'STOP'], + seed=42, + frequency_penalty=0.5, + presence_penalty=0.3, + ) + assert response.choices[0].message.content == 'Nine' + spans = exporter.exported_spans_as_dict() + attrs = spans[0]['attributes'] + assert attrs['gen_ai.request.max_tokens'] == 100 + assert attrs['gen_ai.request.temperature'] == 0.7 + assert attrs['gen_ai.request.top_p'] == 0.9 + assert attrs['gen_ai.request.stop_sequences'] == '["END", "STOP"]' + assert attrs['gen_ai.request.seed'] == 42 + assert attrs['gen_ai.request.frequency_penalty'] == 0.5 + assert attrs['gen_ai.request.presence_penalty'] == 0.3 + + +def test_sync_chat_completions_with_stop_string(instrumented_client: openai.Client, exporter: TestExporter) -> None: + """Test that stop as a string is properly converted to JSON array.""" + response = instrumented_client.chat.completions.create( + model='gpt-4', + messages=[ + {'role': 'user', 'content': 'What is four plus five?'}, + ], + stop='END', + ) + assert response.choices[0].message.content == 'Nine' + spans = exporter.exported_spans_as_dict() + attrs = spans[0]['attributes'] + assert attrs['gen_ai.request.stop_sequences'] == '["END"]' + + +def test_extract_request_parameters_max_output_tokens() -> None: + """Test that max_output_tokens is extracted when max_tokens is absent. + + The Responses API uses max_output_tokens instead of max_tokens. + """ + from logfire._internal.integrations.llm_providers.openai import ( + _extract_request_parameters, # pyright: ignore[reportPrivateUsage] + ) + + json_data: dict[str, Any] = {'max_output_tokens': 200} + span_data: dict[str, Any] = {} + _extract_request_parameters(json_data, span_data) + assert span_data['gen_ai.request.max_tokens'] == 200 + + async def test_async_chat_completions(instrumented_async_client: openai.AsyncClient, exporter: TestExporter) -> None: response = await instrumented_async_client.chat.completions.create( model='gpt-4', @@ -509,76 +566,74 @@ async def test_async_chat_completions(instrumented_async_client: openai.AsyncCli 'code.filepath': 'test_openai.py', 'code.function': 'test_async_chat_completions', 'code.lineno': 123, - 'request_data': ( - { - 'messages': [ - {'role': 'system', 'content': 'You are a helpful assistant.'}, - {'role': 'user', 'content': 'What is four plus five?'}, - ], - 'model': 'gpt-4', - } - ), + 'request_data': { + 'messages': [ + {'role': 'system', 'content': 'You are a helpful assistant.'}, + {'role': 'user', 'content': 'What is four plus five?'}, + ], + 'model': 'gpt-4', + }, + 'gen_ai.provider.name': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', 'async': True, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', + 'gen_ai.system': 'openai', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', 'gen_ai.response.model': 'gpt-4', + 'operation.cost': 0.00012, 'gen_ai.usage.input_tokens': 2, 'gen_ai.usage.output_tokens': 1, - 'operation.cost': 0.00012, - 'response_data': ( - { - 'message': { - 'content': 'Nine', - 'refusal': None, - 'audio': None, - 'annotations': None, - 'role': 'assistant', - 'function_call': None, - 'tool_calls': None, - }, - 'usage': { - 'completion_tokens': 1, - 'prompt_tokens': 2, - 'total_tokens': 3, - 'completion_tokens_details': None, - 'prompt_tokens_details': None, - }, - } - ), - 'logfire.json_schema': ( - { - 'type': 'object', - 'properties': { - 'request_data': {'type': 'object'}, - 'async': {}, - 'gen_ai.system': {}, - 'gen_ai.request.model': {}, - 'gen_ai.response.model': {}, - 'gen_ai.usage.input_tokens': {}, - 'gen_ai.usage.output_tokens': {}, - 'operation.cost': {}, - 'response_data': { - 'type': 'object', - 'properties': { - 'message': { - 'type': 'object', - 'title': 'ChatCompletionMessage', - 'x-python-datatype': 'PydanticModel', - }, - 'usage': { - 'type': 'object', - 'title': 'CompletionUsage', - 'x-python-datatype': 'PydanticModel', - }, + 'response_data': { + 'message': { + 'content': 'Nine', + 'refusal': None, + 'audio': None, + 'annotations': None, + 'role': 'assistant', + 'function_call': None, + 'tool_calls': None, + }, + 'usage': { + 'completion_tokens': 1, + 'prompt_tokens': 2, + 'total_tokens': 3, + 'completion_tokens_details': None, + 'prompt_tokens_details': None, + }, + }, + 'logfire.json_schema': { + 'type': 'object', + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.system': {}, + 'async': {}, + 'gen_ai.response.model': {}, + 'operation.cost': {}, + 'gen_ai.usage.input_tokens': {}, + 'gen_ai.usage.output_tokens': {}, + 'response_data': { + 'type': 'object', + 'properties': { + 'message': { + 'type': 'object', + 'title': 'ChatCompletionMessage', + 'x-python-datatype': 'PydanticModel', + }, + 'usage': { + 'type': 'object', + 'title': 'CompletionUsage', + 'x-python-datatype': 'PydanticModel', }, }, }, - } - ), + }, + }, }, } ] @@ -610,13 +665,21 @@ def test_sync_chat_empty_response_chunk(instrumented_client: openai.Client, expo 'model': 'gpt-4', 'stream': True, }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', 'async': False, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'async': {}, + }, }, 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), @@ -643,7 +706,9 @@ def test_sync_chat_empty_response_chunk(instrumented_client: openai.Client, expo 'code.lineno': 123, 'logfire.msg': "streaming response from 'gpt-4' took 1.00s", 'gen_ai.request.model': 'gpt-4', + 'gen_ai.provider.name': 'openai', 'logfire.span_type': 'log', + 'gen_ai.operation.name': 'chat', 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': {'combined_chunk_content': '', 'chunk_count': 0}, @@ -652,7 +717,9 @@ def test_sync_chat_empty_response_chunk(instrumented_client: openai.Client, expo 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'duration': {}, 'response_data': {'type': 'object'}, }, @@ -690,13 +757,21 @@ def test_sync_chat_empty_response_choices(instrumented_client: openai.Client, ex 'model': 'gpt-4', 'stream': True, }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', 'async': False, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'async': {}, + }, }, 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), @@ -723,7 +798,9 @@ def test_sync_chat_empty_response_choices(instrumented_client: openai.Client, ex 'code.lineno': 123, 'logfire.msg': "streaming response from 'gpt-4' took 1.00s", 'gen_ai.request.model': 'gpt-4', + 'gen_ai.provider.name': 'openai', 'logfire.span_type': 'log', + 'gen_ai.operation.name': 'chat', 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': {'message': None, 'usage': None}, @@ -732,7 +809,9 @@ def test_sync_chat_empty_response_choices(instrumented_client: openai.Client, ex 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'duration': {}, 'response_data': {'type': 'object'}, }, @@ -820,13 +899,42 @@ def test_sync_chat_tool_call_stream(instrumented_client: openai.Client, exporter } ], }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', + 'gen_ai.tool.definitions': [ + { + 'type': 'function', + 'function': { + 'name': 'get_current_weather', + 'description': 'Get the current weather in a given location', + 'parameters': { + 'type': 'object', + 'properties': { + 'location': { + 'type': 'string', + 'description': 'The city and state, e.g. San Francisco, CA', + }, + 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}, + }, + 'required': ['location'], + }, + }, + } + ], 'async': False, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.tool.definitions': {}, + 'async': {}, + }, }, 'logfire.tags': ('LLM',), 'logfire.span_type': 'span', @@ -875,7 +983,29 @@ def test_sync_chat_tool_call_stream(instrumented_client: openai.Client, exporter ], }, 'gen_ai.request.model': 'gpt-4', + 'gen_ai.provider.name': 'openai', 'async': False, + 'gen_ai.operation.name': 'chat', + 'gen_ai.tool.definitions': [ + { + 'type': 'function', + 'function': { + 'name': 'get_current_weather', + 'description': 'Get the current weather in a given location', + 'parameters': { + 'type': 'object', + 'properties': { + 'location': { + 'type': 'string', + 'description': 'The city and state, e.g. San Francisco, CA', + }, + 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}, + }, + 'required': ['location'], + }, + }, + } + ], 'duration': 1.0, 'response_data': { 'message': { @@ -912,7 +1042,10 @@ def test_sync_chat_tool_call_stream(instrumented_client: openai.Client, exporter 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.tool.definitions': {}, 'duration': {}, 'response_data': { 'type': 'object', @@ -1036,13 +1169,42 @@ async def test_async_chat_tool_call_stream( } ], }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', + 'gen_ai.tool.definitions': [ + { + 'type': 'function', + 'function': { + 'name': 'get_current_weather', + 'description': 'Get the current weather in a given location', + 'parameters': { + 'type': 'object', + 'properties': { + 'location': { + 'type': 'string', + 'description': 'The city and state, e.g. San Francisco, CA', + }, + 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}, + }, + 'required': ['location'], + }, + }, + } + ], 'async': True, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.tool.definitions': {}, + 'async': {}, + }, }, 'logfire.tags': ('LLM',), 'logfire.span_type': 'span', @@ -1091,7 +1253,29 @@ async def test_async_chat_tool_call_stream( ], }, 'gen_ai.request.model': 'gpt-4', + 'gen_ai.provider.name': 'openai', 'async': True, + 'gen_ai.operation.name': 'chat', + 'gen_ai.tool.definitions': [ + { + 'type': 'function', + 'function': { + 'name': 'get_current_weather', + 'description': 'Get the current weather in a given location', + 'parameters': { + 'type': 'object', + 'properties': { + 'location': { + 'type': 'string', + 'description': 'The city and state, e.g. San Francisco, CA', + }, + 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}, + }, + 'required': ['location'], + }, + }, + } + ], 'duration': 1.0, 'response_data': { 'message': { @@ -1128,7 +1312,10 @@ async def test_async_chat_tool_call_stream( 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.tool.definitions': {}, 'duration': {}, 'response_data': { 'type': 'object', @@ -1203,13 +1390,21 @@ def test_sync_chat_completions_stream(instrumented_client: openai.Client, export 'model': 'gpt-4', 'stream': True, }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', 'async': False, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'async': {}, + }, }, 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), @@ -1239,7 +1434,9 @@ def test_sync_chat_completions_stream(instrumented_client: openai.Client, export 'code.lineno': 123, 'logfire.msg': "streaming response from 'gpt-4' took 1.00s", 'gen_ai.request.model': 'gpt-4', + 'gen_ai.provider.name': 'openai', 'logfire.span_type': 'log', + 'gen_ai.operation.name': 'chat', 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': { @@ -1260,7 +1457,9 @@ def test_sync_chat_completions_stream(instrumented_client: openai.Client, export 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'duration': {}, 'response_data': { 'type': 'object', @@ -1315,13 +1514,21 @@ async def test_async_chat_completions_stream( 'model': 'gpt-4', 'stream': True, }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'gpt-4', + 'gen_ai.operation.name': 'chat', 'async': True, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', 'logfire.msg': "Chat Completion with 'gpt-4'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'async': {}, + }, }, 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), @@ -1351,7 +1558,9 @@ async def test_async_chat_completions_stream( 'code.lineno': 123, 'logfire.msg': "streaming response from 'gpt-4' took 1.00s", 'gen_ai.request.model': 'gpt-4', + 'gen_ai.provider.name': 'openai', 'logfire.span_type': 'log', + 'gen_ai.operation.name': 'chat', 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': { @@ -1372,7 +1581,9 @@ async def test_async_chat_completions_stream( 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'duration': {}, 'response_data': { 'type': 'object', @@ -1412,7 +1623,9 @@ def test_completions(instrumented_client: openai.Client, exporter: TestExporter) 'code.function': 'test_completions', 'code.lineno': 123, 'request_data': {'model': 'gpt-3.5-turbo-instruct', 'prompt': 'What is four plus five?'}, + 'gen_ai.provider.name': 'openai', 'async': False, + 'gen_ai.operation.name': 'text_completion', 'logfire.msg_template': 'Completion with {request_data[model]!r}', 'logfire.msg': "Completion with 'gpt-3.5-turbo-instruct'", 'logfire.span_type': 'span', @@ -1438,7 +1651,9 @@ def test_completions(instrumented_client: openai.Client, exporter: TestExporter) 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'gen_ai.system': {}, 'gen_ai.request.model': {}, 'gen_ai.response.model': {}, @@ -1489,20 +1704,24 @@ def test_responses_stream(exporter: TestExporter) -> None: 'code.filepath': 'test_openai.py', 'code.function': 'test_responses_stream', 'code.lineno': 123, - 'request_data': {'model': 'gpt-4.1', 'stream': True}, - 'gen_ai.request.model': 'gpt-4.1', + 'gen_ai.provider.name': 'openai', 'events': [ {'event.name': 'gen_ai.user.message', 'content': 'What is four plus five?', 'role': 'user'} ], + 'request_data': {'model': 'gpt-4.1', 'stream': True}, + 'gen_ai.request.model': 'gpt-4.1', + 'gen_ai.operation.name': 'chat', 'async': False, 'logfire.msg_template': 'Responses API with {gen_ai.request.model!r}', 'logfire.msg': "Responses API with 'gpt-4.1'", 'logfire.json_schema': { 'type': 'object', 'properties': { + 'gen_ai.provider.name': {}, + 'events': {'type': 'array'}, 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, - 'events': {'type': 'array'}, + 'gen_ai.operation.name': {}, 'async': {}, }, }, @@ -1526,28 +1745,28 @@ def test_responses_stream(exporter: TestExporter) -> None: 'code.function': 'test_responses_stream', 'code.lineno': 123, 'request_data': {'model': 'gpt-4.1', 'stream': True}, - 'gen_ai.request.model': 'gpt-4.1', - 'async': False, - 'duration': 1.0, + 'gen_ai.provider.name': 'openai', 'events': [ - { - 'event.name': 'gen_ai.user.message', - 'content': 'What is four plus five?', - 'role': 'user', - }, + {'event.name': 'gen_ai.user.message', 'content': 'What is four plus five?', 'role': 'user'}, { 'event.name': 'gen_ai.assistant.message', 'content': 'Four plus five equals **nine**.', 'role': 'assistant', }, ], + 'gen_ai.request.model': 'gpt-4.1', + 'async': False, + 'gen_ai.operation.name': 'chat', + 'duration': 1.0, 'logfire.json_schema': { 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'events': {'type': 'array'}, 'gen_ai.request.model': {}, 'async': {}, - 'events': {'type': 'array'}, + 'gen_ai.operation.name': {}, 'duration': {}, }, }, @@ -1584,13 +1803,21 @@ def test_completions_stream(instrumented_client: openai.Client, exporter: TestEx 'prompt': 'What is four plus five?', 'stream': True, }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'gpt-3.5-turbo-instruct', + 'gen_ai.operation.name': 'text_completion', 'async': False, 'logfire.msg_template': 'Completion with {request_data[model]!r}', 'logfire.msg': "Completion with 'gpt-3.5-turbo-instruct'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'async': {}, + }, }, 'logfire.span_type': 'span', 'logfire.tags': ('LLM',), @@ -1617,7 +1844,9 @@ def test_completions_stream(instrumented_client: openai.Client, exporter: TestEx 'code.lineno': 123, 'logfire.msg': "streaming response from 'gpt-3.5-turbo-instruct' took 1.00s", 'gen_ai.request.model': 'gpt-3.5-turbo-instruct', + 'gen_ai.provider.name': 'openai', 'logfire.span_type': 'log', + 'gen_ai.operation.name': 'text_completion', 'logfire.tags': ('LLM',), 'duration': 1.0, 'response_data': {'combined_chunk_content': 'The answer is Nine', 'chunk_count': 2}, @@ -1626,7 +1855,9 @@ def test_completions_stream(instrumented_client: openai.Client, exporter: TestEx 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'duration': {}, 'response_data': {'type': 'object'}, }, @@ -1661,7 +1892,9 @@ def test_embeddings(instrumented_client: openai.Client, exporter: TestExporter) 'model': 'text-embedding-3-small', 'encoding_format': 'base64', }, + 'gen_ai.provider.name': 'openai', 'async': False, + 'gen_ai.operation.name': 'embeddings', 'logfire.msg_template': 'Embedding Creation with {request_data[model]!r}', 'logfire.msg': "Embedding Creation with 'text-embedding-3-small'", 'logfire.span_type': 'span', @@ -1675,7 +1908,9 @@ def test_embeddings(instrumented_client: openai.Client, exporter: TestExporter) 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'gen_ai.system': {}, 'gen_ai.request.model': {}, 'gen_ai.response.model': {}, @@ -1715,13 +1950,15 @@ def test_images(instrumented_client: openai.Client, exporter: TestExporter) -> N 'code.function': 'test_images', 'code.lineno': 123, 'request_data': {'prompt': 'A picture of a cat.', 'model': 'dall-e-3'}, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'dall-e-3', + 'gen_ai.operation.name': 'image_generation', 'async': False, 'logfire.msg_template': 'Image Generation with {request_data[model]!r}', 'logfire.msg': "Image Generation with 'dall-e-3'", 'logfire.span_type': 'span', - 'logfire.tags': ('LLM',), 'gen_ai.system': 'openai', + 'logfire.tags': ('LLM',), 'response_data': { 'images': [ { @@ -1735,9 +1972,11 @@ def test_images(instrumented_client: openai.Client, exporter: TestExporter) -> N 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, 'gen_ai.request.model': {}, - 'async': {}, + 'gen_ai.operation.name': {}, 'gen_ai.system': {}, + 'async': {}, 'response_data': { 'type': 'object', 'properties': { @@ -1840,7 +2079,9 @@ def test_dont_suppress_httpx(exporter: TestExporter) -> None: 'code.function': 'test_dont_suppress_httpx', 'code.lineno': 123, 'request_data': {'model': 'gpt-3.5-turbo-instruct', 'prompt': 'xxx'}, + 'gen_ai.provider.name': 'openai', 'async': False, + 'gen_ai.operation.name': 'text_completion', 'logfire.msg_template': 'Completion with {request_data[model]!r}', 'logfire.msg': "Completion with 'gpt-3.5-turbo-instruct'", 'logfire.span_type': 'span', @@ -1866,7 +2107,9 @@ def test_dont_suppress_httpx(exporter: TestExporter) -> None: 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'gen_ai.system': {}, 'gen_ai.request.model': {}, 'gen_ai.response.model': {}, @@ -1947,7 +2190,9 @@ def test_suppress_httpx(exporter: TestExporter) -> None: 'code.function': 'test_suppress_httpx', 'code.lineno': 123, 'request_data': {'model': 'gpt-3.5-turbo-instruct', 'prompt': 'xxx'}, + 'gen_ai.provider.name': 'openai', 'async': False, + 'gen_ai.operation.name': 'text_completion', 'logfire.msg_template': 'Completion with {request_data[model]!r}', 'logfire.msg': "Completion with 'gpt-3.5-turbo-instruct'", 'logfire.span_type': 'span', @@ -1973,7 +2218,9 @@ def test_suppress_httpx(exporter: TestExporter) -> None: 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'gen_ai.system': {}, 'gen_ai.request.model': {}, 'gen_ai.response.model': {}, @@ -2039,6 +2286,7 @@ def test_create_files(instrumented_client: openai.Client, exporter: TestExporter 'request_data': {'purpose': 'fine-tune'}, 'url': '/files', 'async': False, + 'gen_ai.provider.name': 'openai', 'logfire.msg_template': 'OpenAI API call to {url!r}', 'logfire.msg': "OpenAI API call to '/files'", 'code.filepath': 'test_openai.py', @@ -2047,7 +2295,13 @@ def test_create_files(instrumented_client: openai.Client, exporter: TestExporter 'gen_ai.system': 'openai', 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'url': {}, 'async': {}, 'gen_ai.system': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'url': {}, + 'gen_ai.provider.name': {}, + 'async': {}, + 'gen_ai.system': {}, + }, }, }, } @@ -2072,6 +2326,7 @@ async def test_create_files_async(instrumented_async_client: openai.AsyncClient, 'request_data': {'purpose': 'fine-tune'}, 'url': '/files', 'async': True, + 'gen_ai.provider.name': 'openai', 'logfire.msg_template': 'OpenAI API call to {url!r}', 'logfire.msg': "OpenAI API call to '/files'", 'code.filepath': 'test_openai.py', @@ -2080,7 +2335,13 @@ async def test_create_files_async(instrumented_async_client: openai.AsyncClient, 'gen_ai.system': 'openai', 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'url': {}, 'async': {}, 'gen_ai.system': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'url': {}, + 'gen_ai.provider.name': {}, + 'async': {}, + 'gen_ai.system': {}, + }, }, }, } @@ -2117,22 +2378,26 @@ def test_create_assistant(instrumented_client: openai.Client, exporter: TestExpo ), 'url': '/assistants', 'async': False, + 'gen_ai.provider.name': 'openai', 'logfire.msg_template': 'OpenAI API call to {url!r}', + 'gen_ai.tool.definitions': [{'type': 'code_interpreter'}], 'logfire.msg': "OpenAI API call to '/assistants'", 'code.filepath': 'test_openai.py', 'code.function': 'test_create_assistant', 'code.lineno': 123, - 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4o', + 'gen_ai.system': 'openai', 'gen_ai.response.model': 'gpt-4-turbo', 'logfire.json_schema': { 'type': 'object', 'properties': { 'request_data': {'type': 'object'}, 'url': {}, + 'gen_ai.provider.name': {}, 'async': {}, - 'gen_ai.system': {}, + 'gen_ai.tool.definitions': {}, 'gen_ai.request.model': {}, + 'gen_ai.system': {}, 'gen_ai.response.model': {}, }, }, @@ -2160,6 +2425,7 @@ def test_create_thread(instrumented_client: openai.Client, exporter: TestExporte 'request_data': {}, 'url': '/threads', 'async': False, + 'gen_ai.provider.name': 'openai', 'logfire.msg_template': 'OpenAI API call to {url!r}', 'logfire.msg': "OpenAI API call to '/threads'", 'code.filepath': 'test_openai.py', @@ -2168,7 +2434,13 @@ def test_create_thread(instrumented_client: openai.Client, exporter: TestExporte 'gen_ai.system': 'openai', 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'url': {}, 'async': {}, 'gen_ai.system': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'url': {}, + 'gen_ai.provider.name': {}, + 'async': {}, + 'gen_ai.system': {}, + }, }, }, } @@ -2217,18 +2489,35 @@ def test_responses_api(exporter: TestExporter) -> None: 'code.filepath': 'test_openai.py', 'code.function': 'test_responses_api', 'code.lineno': 123, + 'gen_ai.provider.name': 'openai', 'async': False, 'request_data': {'model': 'gpt-4.1', 'stream': False}, + 'gen_ai.operation.name': 'chat', + 'gen_ai.tool.definitions': [ + { + 'type': 'function', + 'name': 'get_weather', + 'description': 'Get current temperature for a given location.', + 'parameters': { + 'type': 'object', + 'properties': { + 'location': { + 'type': 'string', + 'description': 'City and country e.g. Bogotá, Colombia', + } + }, + 'required': ['location'], + 'additionalProperties': False, + }, + } + ], 'logfire.msg_template': 'Responses API with {gen_ai.request.model!r}', 'logfire.msg': "Responses API with 'gpt-4.1'", + 'gen_ai.system': 'openai', 'logfire.tags': ('LLM',), 'logfire.span_type': 'span', - 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4.1', 'gen_ai.response.model': 'gpt-4.1-2025-04-14', - 'gen_ai.usage.input_tokens': 65, - 'gen_ai.usage.output_tokens': 17, - 'operation.cost': 0.000266, 'events': [ {'event.name': 'gen_ai.system.message', 'content': 'Be nice', 'role': 'system'}, { @@ -2248,14 +2537,20 @@ def test_responses_api(exporter: TestExporter) -> None: ], }, ], + 'gen_ai.usage.input_tokens': 65, + 'gen_ai.usage.output_tokens': 17, + 'operation.cost': 0.000266, 'logfire.json_schema': { 'type': 'object', 'properties': { + 'gen_ai.provider.name': {}, + 'events': {'type': 'array'}, 'gen_ai.request.model': {}, 'request_data': {'type': 'object'}, - 'events': {'type': 'array'}, - 'async': {}, + 'gen_ai.operation.name': {}, + 'gen_ai.tool.definitions': {}, 'gen_ai.system': {}, + 'async': {}, 'gen_ai.response.model': {}, 'gen_ai.usage.input_tokens': {}, 'gen_ai.usage.output_tokens': {}, @@ -2274,18 +2569,18 @@ def test_responses_api(exporter: TestExporter) -> None: 'code.filepath': 'test_openai.py', 'code.function': 'test_responses_api', 'code.lineno': 123, + 'gen_ai.provider.name': 'openai', 'async': False, 'request_data': {'model': 'gpt-4.1', 'stream': False}, + 'gen_ai.operation.name': 'chat', 'logfire.msg_template': 'Responses API with {gen_ai.request.model!r}', 'logfire.msg': "Responses API with 'gpt-4.1'", 'logfire.tags': ('LLM',), - 'logfire.span_type': 'span', 'gen_ai.system': 'openai', + 'logfire.span_type': 'span', 'gen_ai.request.model': 'gpt-4.1', 'gen_ai.response.model': 'gpt-4.1-2025-04-14', 'gen_ai.usage.input_tokens': 43, - 'gen_ai.usage.output_tokens': 21, - 'operation.cost': 0.000254, 'events': [ { 'event.name': 'gen_ai.user.message', @@ -2316,14 +2611,18 @@ def test_responses_api(exporter: TestExporter) -> None: 'role': 'assistant', }, ], + 'gen_ai.usage.output_tokens': 21, + 'operation.cost': 0.000254, 'logfire.json_schema': { 'type': 'object', 'properties': { + 'gen_ai.provider.name': {}, + 'events': {'type': 'array'}, 'gen_ai.request.model': {}, 'request_data': {'type': 'object'}, - 'events': {'type': 'array'}, - 'async': {}, + 'gen_ai.operation.name': {}, 'gen_ai.system': {}, + 'async': {}, 'gen_ai.response.model': {}, 'gen_ai.usage.input_tokens': {}, 'gen_ai.usage.output_tokens': {}, @@ -2380,13 +2679,21 @@ def test_openrouter_streaming_reasoning(exporter: TestExporter) -> None: 'model': 'google/gemini-2.5-flash', 'stream': True, }, + 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'google/gemini-2.5-flash', + 'gen_ai.operation.name': 'chat', 'async': False, 'logfire.msg_template': 'Chat Completion with {request_data[model]!r}', 'logfire.msg': "Chat Completion with 'google/gemini-2.5-flash'", 'logfire.json_schema': { 'type': 'object', - 'properties': {'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, 'async': {}}, + 'properties': { + 'request_data': {'type': 'object'}, + 'gen_ai.provider.name': {}, + 'gen_ai.request.model': {}, + 'gen_ai.operation.name': {}, + 'async': {}, + }, }, 'logfire.tags': ('LLM',), 'logfire.span_type': 'span', @@ -2413,7 +2720,9 @@ def test_openrouter_streaming_reasoning(exporter: TestExporter) -> None: 'stream': True, }, 'gen_ai.request.model': 'google/gemini-2.5-flash', + 'gen_ai.provider.name': 'openai', 'async': False, + 'gen_ai.operation.name': 'chat', 'duration': 1.0, 'response_data': { 'message': { @@ -2467,7 +2776,9 @@ def test_openrouter_streaming_reasoning(exporter: TestExporter) -> None: 'properties': { 'request_data': {'type': 'object'}, 'gen_ai.request.model': {}, + 'gen_ai.provider.name': {}, 'async': {}, + 'gen_ai.operation.name': {}, 'duration': {}, 'response_data': { 'type': 'object',