diff --git a/settings.gradle.kts b/settings.gradle.kts index ed340cc0..fcd4dece 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("org.gradle.toolchains.foojay-resolver-convention") version "0.8.0" + id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0" } rootProject.name = "coral-server" diff --git a/src/main/kotlin/org/coralprotocol/coralserver/Main.kt b/src/main/kotlin/org/coralprotocol/coralserver/Main.kt index 43e606be..14193836 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/Main.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/Main.kt @@ -25,6 +25,7 @@ fun main(args: Array) { blockchainModule, networkModule, agentModule, + llmProxyModule, sessionModule, module { single { diff --git a/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/AgentLlmConfig.kt b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/AgentLlmConfig.kt new file mode 100644 index 00000000..fbcde4f2 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/AgentLlmConfig.kt @@ -0,0 +1,15 @@ +package org.coralprotocol.coralserver.agent.registry + +import kotlinx.serialization.Serializable + +@Serializable +data class AgentLlmConfig( + val proxies: List = emptyList() +) + +@Serializable +data class AgentLlmProxy( + val name: String, + val format: String, + val model: String? = null +) diff --git a/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgent.kt b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgent.kt index 1439da6b..7891e0bc 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgent.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgent.kt @@ -26,6 +26,7 @@ data class RegistryAgent( val edition: Int = MAXIMUM_SUPPORTED_AGENT_VERSION, val runtimes: LocalAgentRuntimes, val options: Map = mapOf(), + val llm: AgentLlmConfig? = null, val marketplace: RegistryAgentMarketplaceSettings? = null, @Transient diff --git a/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgentValidation.kt b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgentValidation.kt index 1e694425..ba7e45a3 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgentValidation.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/RegistryAgentValidation.kt @@ -14,6 +14,7 @@ import org.bitcoinj.core.Base58 import org.coralprotocol.coralserver.agent.registry.option.AgentOption import org.coralprotocol.coralserver.agent.runtime.PrototypeRuntime import org.coralprotocol.coralserver.agent.runtime.prototype.* +import org.coralprotocol.coralserver.llmproxy.LlmProviderProfile import java.net.URI import java.net.URISyntaxException @@ -72,6 +73,12 @@ val AGENT_MARKETPLACE_PRICING_DESCRIPTION_LENGTH = 1..256 const val AGENT_MARKETPLACE_PRICING_MIN_MIN = 0.00 const val AGENT_MARKETPLACE_PRICING_MIN_MAX = 20.00 +// [llm.proxies] +const val AGENT_LLM_PROXIES_MAX_ENTRIES = 16 +val AGENT_LLM_PROXY_NAME_LENGTH = 1..32 +val AGENT_LLM_PROXY_NAME_PATTERN = "^[A-Z_0-9]+$".toRegex() +val AGENT_LLM_PROXY_MODEL_LENGTH = 1..128 + // [marketplace.identities.erc8004] const val AGENT_MARKETPLACE_ERC8004_ENDPOINTS_MAX_ENTRIES = 32 val AGENT_MARKETPLACE_ERC8004_ENDPOINTS_NAME_LENGTH = 1..32 @@ -583,11 +590,36 @@ private fun RegistryAgent.validateMarketplace() { * * @throws RegistryException if this registry agent contains any number of invalid values */ +private fun RegistryAgent.validateLlm() { + val llm = llm ?: return + + if (llm.proxies.size > AGENT_LLM_PROXIES_MAX_ENTRIES) + throw RegistryException("llm proxy count cannot exceed $AGENT_LLM_PROXIES_MAX_ENTRIES, was ${llm.proxies.size}") + + val names = mutableSetOf() + for ((index, proxy) in llm.proxies.withIndex()) { + validateStringLength("llm.proxies[$index].name", proxy.name, AGENT_LLM_PROXY_NAME_LENGTH) + + if (!proxy.name.matches(AGENT_LLM_PROXY_NAME_PATTERN)) + throw RegistryException("llm.proxies[$index].name (\"${proxy.name}\") must only contain uppercase alphanumeric or underscore characters") + + if (!names.add(proxy.name)) + throw RegistryException("llm.proxies[$index].name (\"${proxy.name}\") is not unique") + + if (LlmProviderProfile.fromId(proxy.format) == null) + throw RegistryException("llm.proxies[$index].format (\"${proxy.format}\") is not a known format. Valid formats: ${LlmProviderProfile.entries.joinToString { it.providerId }}") + + if (proxy.model != null) + validateStringLength("llm.proxies[$index].model", proxy.model, AGENT_LLM_PROXY_MODEL_LENGTH) + } +} + fun RegistryAgent.validate() { validateName() validateVersion() validateOptionalAgentInfo() validateRuntimes() validateOptions() + validateLlm() validateMarketplace() } \ No newline at end of file diff --git a/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/UnresolvedRegistryAgent.kt b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/UnresolvedRegistryAgent.kt index 4b5f416a..abaeb0e3 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/UnresolvedRegistryAgent.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/agent/registry/UnresolvedRegistryAgent.kt @@ -43,6 +43,10 @@ data class UnresolvedRegistryAgent( @Optional val options: Map = mapOf(), + @Description("LLM proxy configuration declaring which proxy endpoints this agent needs") + @Optional + val llm: AgentLlmConfig? = null, + @Description("Information for this agent relevant to it's potential listing on the marketplace") @Optional val marketplace: RegistryAgentMarketplaceSettings? = null @@ -114,6 +118,7 @@ data class UnresolvedRegistryAgent( info = agentInfo.resolve(context.registrySourceIdentifier), runtimes = runtimes, options = options, + llm = llm, path = context.path, marketplace = marketplace ) diff --git a/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/ApplicationRuntimeContext.kt b/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/ApplicationRuntimeContext.kt index 890746d9..213bb028 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/ApplicationRuntimeContext.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/ApplicationRuntimeContext.kt @@ -66,6 +66,12 @@ class ApplicationRuntimeContext( return builder.build() } + fun getLlmProxyUrl(executionContext: SessionAgentExecutionContext, addressConsumer: AddressConsumer): Url { + val builder = URLBuilder(getApiUrl(addressConsumer)) + builder.appendPathSegments("llm-proxy", executionContext.agent.secret) + return builder.build() + } + fun getMcpUrl( transport: McpTransportType, executionContext: SessionAgentExecutionContext, diff --git a/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeApiUrl.kt b/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeApiUrl.kt index bad8c106..fdb95a37 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeApiUrl.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeApiUrl.kt @@ -17,9 +17,9 @@ sealed interface PrototypeApiUrl { @Serializable @SerialName("proxy") - object Proxy : PrototypeApiUrl { + data object Proxy : PrototypeApiUrl { override fun resolve(executionContext: SessionAgentExecutionContext): String { - TODO("Not yet implemented") + TODO("format changing soon") } } diff --git a/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeModelProvider.kt b/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeModelProvider.kt index d89bf0a4..bb0c0965 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeModelProvider.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/agent/runtime/prototype/PrototypeModelProvider.kt @@ -11,7 +11,7 @@ import ai.koog.prompt.executor.clients.openai.OpenAIModels import ai.koog.prompt.executor.clients.openrouter.OpenRouterClientSettings import ai.koog.prompt.executor.clients.openrouter.OpenRouterLLMClient import ai.koog.prompt.executor.clients.openrouter.OpenRouterModels -import ai.koog.prompt.executor.llms.SingleLLMPromptExecutor +import ai.koog.prompt.executor.llms.MultiLLMPromptExecutor import ai.koog.prompt.executor.model.PromptExecutor import ai.koog.prompt.llm.LLModel import dev.eav.tomlkt.TomlClassDiscriminator @@ -60,8 +60,8 @@ sealed class PrototypeModelProvider { override val name: PrototypeString, override val url: PrototypeApiUrl? = null, ) : PrototypeModelProvider() { - override fun getExecutor(executionContext: SessionAgentExecutionContext): PromptExecutor = - SingleLLMPromptExecutor( + override fun getExecutor(executionContext: SessionAgentExecutionContext): PromptExecutor { + return MultiLLMPromptExecutor( OpenAILLMClient( apiKey = key.resolve(executionContext), settings = if (url == null) OpenAIClientSettings() else OpenAIClientSettings( @@ -69,6 +69,7 @@ sealed class PrototypeModelProvider { ) ) ) + } override val modelClass: Any get() = OpenAIModels.Chat @@ -81,8 +82,8 @@ sealed class PrototypeModelProvider { override val name: PrototypeString, override val url: PrototypeApiUrl? = null, ) : PrototypeModelProvider() { - override fun getExecutor(executionContext: SessionAgentExecutionContext): PromptExecutor = - SingleLLMPromptExecutor( + override fun getExecutor(executionContext: SessionAgentExecutionContext): PromptExecutor { + return MultiLLMPromptExecutor( AnthropicLLMClient( apiKey = key.resolve(executionContext), settings = if (url == null) AnthropicClientSettings() else AnthropicClientSettings( @@ -90,6 +91,7 @@ sealed class PrototypeModelProvider { ) ) ) + } override val modelClass: Any get() = AnthropicModels @@ -102,8 +104,8 @@ sealed class PrototypeModelProvider { override val name: PrototypeString, override val url: PrototypeApiUrl? = null, ) : PrototypeModelProvider() { - override fun getExecutor(executionContext: SessionAgentExecutionContext): PromptExecutor = - SingleLLMPromptExecutor( + override fun getExecutor(executionContext: SessionAgentExecutionContext): PromptExecutor { + return MultiLLMPromptExecutor( OpenRouterLLMClient( apiKey = key.resolve(executionContext), settings = if (url == null) OpenRouterClientSettings() else OpenRouterClientSettings( @@ -111,6 +113,7 @@ sealed class PrototypeModelProvider { ) ) ) + } override val modelClass: Any get() = OpenRouterModels diff --git a/src/main/kotlin/org/coralprotocol/coralserver/config/LlmProxyConfig.kt b/src/main/kotlin/org/coralprotocol/coralserver/config/LlmProxyConfig.kt new file mode 100644 index 00000000..ee303f50 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/config/LlmProxyConfig.kt @@ -0,0 +1,16 @@ +package org.coralprotocol.coralserver.config + +data class LlmProxyConfig( + val enabled: Boolean = true, + val requestTimeoutSeconds: Long = 300, + val retryMaxAttempts: Int = 0, + val retryInitialDelayMs: Long = 1000, + val retryMaxDelayMs: Long = 10000, + val providers: Map = emptyMap() +) + +data class LlmProxyProviderConfig( + val apiKey: String? = null, + val baseUrl: String? = null, + val timeoutSeconds: Long? = null +) diff --git a/src/main/kotlin/org/coralprotocol/coralserver/config/RootConfig.kt b/src/main/kotlin/org/coralprotocol/coralserver/config/RootConfig.kt index eb60b15e..45f43f5c 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/config/RootConfig.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/config/RootConfig.kt @@ -38,7 +38,10 @@ data class RootConfig( val loggingConfig: LoggingConfig = LoggingConfig(), @param:ConfigAlias("console") - val consoleConfig: ConsoleConfig = ConsoleConfig() + val consoleConfig: ConsoleConfig = ConsoleConfig(), + + @param:ConfigAlias("llm-proxy") + val llmProxyConfig: LlmProxyConfig = LlmProxyConfig() ) { /** * Calculates the address required to access the server for a given consumer. diff --git a/src/main/kotlin/org/coralprotocol/coralserver/events/SessionEvent.kt b/src/main/kotlin/org/coralprotocol/coralserver/events/SessionEvent.kt index 0e2185be..a769def8 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/events/SessionEvent.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/events/SessionEvent.kt @@ -7,6 +7,7 @@ import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonClassDiscriminator import org.coralprotocol.coralserver.agent.graph.UniqueAgentName +import org.coralprotocol.coralserver.llmproxy.LlmErrorKind import org.coralprotocol.coralserver.session.* import org.coralprotocol.coralserver.util.InstantSerializer import org.coralprotocol.coralserver.util.utcTimeNow @@ -77,4 +78,18 @@ sealed class SessionEvent { @Serializable @SerialName("docker_container_removed") data class DockerContainerRemoved(val containerId: String) : SessionEvent() + + @Serializable + @SerialName("llm_proxy_call") + data class LlmProxyCall( + val agentName: UniqueAgentName, + val provider: String, + val model: String?, + val inputTokens: Long?, + val outputTokens: Long?, + val durationMs: Long, + val streaming: Boolean, + val success: Boolean, + val errorKind: LlmErrorKind? = null + ) : SessionEvent() } diff --git a/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmCallResult.kt b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmCallResult.kt new file mode 100644 index 00000000..0219d176 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmCallResult.kt @@ -0,0 +1,33 @@ +package org.coralprotocol.coralserver.llmproxy + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +enum class LlmErrorKind { + @SerialName("rate_limited") RATE_LIMITED, + @SerialName("credentials") CREDENTIALS, + @SerialName("upstream_health") UPSTREAM_HEALTH, + @SerialName("request_error") REQUEST_ERROR, + @SerialName("connectivity") CONNECTIVITY, + @SerialName("response_too_large") RESPONSE_TOO_LARGE, + @SerialName("unknown") UNKNOWN +} + +data class LlmCallResult( + val provider: String, + val model: String?, + val inputTokens: Long? = null, + val outputTokens: Long? = null, + val durationMs: Long, + val streaming: Boolean, + val success: Boolean, + val errorKind: LlmErrorKind? = null, + val statusCode: Int? = null, + val chunkCount: Int? = null, +) { + fun formatTokenInfo(): String { + if (inputTokens == null && outputTokens == null) return "" + return " tokens=${inputTokens ?: "?"}→${outputTokens ?: "?"}" + } +} diff --git a/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderProfile.kt b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderProfile.kt new file mode 100644 index 00000000..32b2da42 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderProfile.kt @@ -0,0 +1,41 @@ +package org.coralprotocol.coralserver.llmproxy + +enum class LlmProviderProfile( + val providerId: String, + val defaultBaseUrl: String, + val authStyle: AuthStyle, + val defaultHeaders: Map, + val strategy: LlmProviderStrategy, + val sdkBaseUrlEnvVar: String? = null, + val sdkPathSuffix: String = "" +) { + OPENAI( + "openai", "https://api.openai.com", AuthStyle.Bearer, emptyMap(), OpenAIStrategy, + sdkBaseUrlEnvVar = "OPENAI_BASE_URL", sdkPathSuffix = "v1" + ), + + ANTHROPIC( + "anthropic", + "https://api.anthropic.com", + AuthStyle.Custom("x-api-key"), + mapOf("anthropic-version" to "2023-06-01"), + AnthropicStrategy, + sdkBaseUrlEnvVar = "ANTHROPIC_BASE_URL" + ), + + OPENROUTER( + "openrouter", "https://openrouter.ai", AuthStyle.Bearer, emptyMap(), OpenAIStrategy, + sdkBaseUrlEnvVar = "OPENROUTER_BASE_URL" + ); + + companion object { + private val byId = entries.associateBy { it.providerId } + + fun fromId(id: String): LlmProviderProfile? = byId[id.lowercase()] + } +} + +sealed class AuthStyle { + data object Bearer : AuthStyle() + data class Custom(val headerName: String) : AuthStyle() +} diff --git a/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderStrategy.kt b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderStrategy.kt new file mode 100644 index 00000000..c3fea739 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderStrategy.kt @@ -0,0 +1,140 @@ +@file:OptIn(ExperimentalSerializationApi::class) + +package org.coralprotocol.coralserver.llmproxy + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.* +import org.coralprotocol.coralserver.logging.LoggingInterface + +@Serializable +@JsonIgnoreUnknownKeys +data class LlmUsage( + @JsonNames("prompt_tokens", "input_tokens") + val inputTokens: Long? = null, + + @JsonNames("completion_tokens", "output_tokens") + val outputTokens: Long? = null, +) + +@Serializable +@JsonIgnoreUnknownKeys +private data class LlmUsageWrapper(val usage: LlmUsage? = null) + +interface LlmProviderStrategy { + fun prepareStreamingRequest(requestBody: String, json: Json, logger: LoggingInterface): String = requestBody + fun extractBufferedTokens(responseBody: String, json: Json): LlmUsage? + fun createStreamParser(json: Json): StreamTokenParser +} + +/** + * Stateful parser for a single SSE stream. Processes raw SSE lines and extracts token usage. + * Each streaming request should create a fresh instance via [LlmProviderStrategy.createStreamParser]. + */ +interface StreamTokenParser { + fun processLine(line: String) + val inputTokens: Long? + val outputTokens: Long? + val chunkCount: Int +} + +object OpenAIStrategy : LlmProviderStrategy { + override fun prepareStreamingRequest(requestBody: String, json: Json, logger: LoggingInterface): String { + return try { + val obj = json.decodeFromString(requestBody) + if (obj.containsKey("stream_options")) return requestBody + val modified = buildJsonObject { + obj.forEach { (key, value) -> put(key, value) } + putJsonObject("stream_options") { put("include_usage", true) } + } + json.encodeToString(JsonObject.serializer(), modified) + } catch (e: Exception) { + logger.error(e) { "Failed to inject stream_options into request body" } + requestBody + } + } + + override fun extractBufferedTokens(responseBody: String, json: Json) = extractLlmUsage(responseBody, json) + override fun createStreamParser(json: Json): StreamTokenParser = OpenAIStreamParser(json) +} + +object AnthropicStrategy : LlmProviderStrategy { + override fun extractBufferedTokens(responseBody: String, json: Json) = extractLlmUsage(responseBody, json) + override fun createStreamParser(json: Json): StreamTokenParser = AnthropicStreamParser(json) +} + +/** + * OpenAI SSE format: `data: {json}` lines, `data: [DONE]` terminator. + * Usage appears in the final chunk when `stream_options.include_usage=true`. + */ +private class OpenAIStreamParser(private val json: Json) : StreamTokenParser { + override var inputTokens: Long? = null; private set + override var outputTokens: Long? = null; private set + override var chunkCount: Int = 0; private set + + override fun processLine(line: String) { + if (!line.startsWith("data: ") || line.startsWith("data: [DONE]")) return + chunkCount++ + try { + val usageWrapper = json.decodeFromString(line.removePrefix("data: ")) + + inputTokens = usageWrapper.usage?.inputTokens ?: inputTokens + outputTokens = usageWrapper.usage?.outputTokens ?: outputTokens + } catch (_: SerializationException) { + // ignored, not containing usage information is not an error + } + } +} + +/** + * Anthropic SSE format: `event: {type}` + `data: {json}` pairs. + * Input tokens in `message_start` event, output tokens in `message_delta` event. + */ +private class AnthropicStreamParser(private val json: Json) : StreamTokenParser { + override var inputTokens: Long? = null; private set + override var outputTokens: Long? = null; private set + override var chunkCount: Int = 0; private set + private var lastEventType: String? = null + + override fun processLine(line: String) { + if (line.startsWith("event: ")) { + lastEventType = line.removePrefix("event: ").trim() + return + } + + if (!line.startsWith("data: ")) return + chunkCount++ + + try { + val obj = json.decodeFromString(line.removePrefix("data: ")) + when (lastEventType) { + "message_start" -> { + val usage = obj["message"]?.jsonObject?.let { extractLlmUsage(it, json) } + inputTokens = usage?.inputTokens ?: inputTokens + } + + "message_delta" -> { + val usage = extractLlmUsage(obj, json) + outputTokens = usage?.outputTokens ?: outputTokens + } + } + } catch (_: SerializationException) { + // ignored, not containing usage information is not an error + } + } +} + +fun extractLlmUsage(body: String, json: Json) = + try { + json.decodeFromString(body).usage + } catch (_: SerializationException) { + null + } + +fun extractLlmUsage(body: JsonObject, json: Json) = + try { + json.decodeFromJsonElement(body).usage + } catch (_: SerializationException) { + null + } \ No newline at end of file diff --git a/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProxyService.kt b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProxyService.kt new file mode 100644 index 00000000..0d85b169 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProxyService.kt @@ -0,0 +1,357 @@ +package org.coralprotocol.coralserver.llmproxy + +import io.ktor.client.* +import io.ktor.client.network.sockets.* +import io.ktor.client.plugins.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.server.application.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.utils.io.* +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.jsonPrimitive +import org.coralprotocol.coralserver.config.LlmProxyConfig +import org.coralprotocol.coralserver.events.SessionEvent +import org.coralprotocol.coralserver.routes.RouteException +import org.coralprotocol.coralserver.session.SessionAgent +import kotlin.coroutines.cancellation.CancellationException + +private val ALLOWED_METHODS = setOf(HttpMethod.Get, HttpMethod.Post) +private val METHODS_WITH_BODY = setOf(HttpMethod.Post) + +private const val MAX_REQUEST_BODY_BYTES = 20 * 1024 * 1024 // 20 MB +private const val MAX_RESPONSE_BODY_BYTES = 80 * 1024 * 1024L // 80 MB +private const val MAX_STREAM_CHARS = 80 * 1024 * 1024L // 80 M chars (~bytes for ASCII SSE) + +class LlmProxyException(message: String) : Exception(message) + +class LlmProxyService( + private val config: LlmProxyConfig, + private val httpClient: HttpClient, + private val json: Json +) { + + + /** + * Methods: GET, POST + * POST body: JSON only (application/json and application/+json) + * Responses: JSON or SSE + * Forwarded: path, query params, provider auth, most normal headers + * Not supported: multipart, binary uploads, file/audio/image upload endpoints + * Current scope: inference-style endpoints like chat/messages/responses/embeddings/models + * Security behavior: Authorization/provider auth is normalized by the proxy, Cookie is dropped + */ + suspend fun proxyRequest( + agent: SessionAgent, + providerName: String, + pathParts: List, + call: ApplicationCall + ) { + val profile = LlmProviderProfile.fromId(providerName) ?: throw RouteException( + HttpStatusCode.BadRequest, + "Unknown provider: $providerName" + ) + + val providerConfig = config.providers[providerName] + val serverKey = providerConfig?.apiKey + val agentKey = ProxyHeaders.extractAgentKey(call, profile) + val apiKey = serverKey ?: agentKey ?: throw RouteException( + HttpStatusCode.Unauthorized, + "No API key available for provider: $providerName (neither server-configured nor provided by agent)" + ) + + validateRequestShape(call) + + val baseUrl = providerConfig?.baseUrl ?: profile.defaultBaseUrl + val upstreamUrl = URLBuilder(baseUrl).apply { + appendEncodedPathSegments(pathParts) + call.request.queryParameters.entries().forEach { (name, values) -> + values.forEach { value -> parameters.append(name, value) } + } + }.buildString() + val timeoutMs = ((providerConfig?.timeoutSeconds ?: config.requestTimeoutSeconds) * 1000) + val hasBody = call.request.httpMethod in METHODS_WITH_BODY + val requestBody = readRequestBody(hasBody, call) + + val requestJson = if (hasBody) tryParseJson(requestBody) else null + val isStreaming = requestJson?.get("stream")?.jsonPrimitive?.booleanOrNull == true + val model = requestJson?.get("model")?.jsonPrimitive?.content + + val finalBody = + if (isStreaming) profile.strategy.prepareStreamingRequest(requestBody, json, agent.logger) else requestBody + + val req = ProxyRequest( + agent, + profile, + apiKey, + upstreamUrl, + timeoutMs, + finalBody, + model, + hasBody, + System.currentTimeMillis() + ) + + agent.logger.debug { + "LLM Proxy → $providerName/${ + URLBuilder().appendPathSegments(pathParts).buildString() + } model=$model streaming=$isStreaming " + + "auth=${if (serverKey != null) "server" else "agent"}" + } + + try { + if (isStreaming) proxyStreaming(req, call) else proxyBuffered(req, call) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + val durationMs = System.currentTimeMillis() - req.startTime + emitTelemetry( + agent, + LlmCallResult( + providerName, + model, + durationMs = durationMs, + streaming = isStreaming, + success = false, + errorKind = classifyError(e) + ) + ) + if (!call.response.isCommitted) { + agent.logger.warn { "LLM proxy request failed: ${e::class.simpleName}: ${e.message}" } + throw RouteException(HttpStatusCode.BadGateway, "LLM proxy request failed") + } + } + } + + private suspend fun proxyBuffered(req: ProxyRequest, call: ApplicationCall) { + val response = httpClient.request(req.upstreamUrl) { + configureProxy(req, call) + } + + val responseBody = readBoundedBody(response) + val durationMs = System.currentTimeMillis() - req.startTime + + ProxyHeaders.forwardResponseHeaders(response, call) + val upstreamContentType = response.contentType() ?: ContentType.Application.Json + call.respondText(responseBody, upstreamContentType, response.status) + + val usage = req.profile.strategy.extractBufferedTokens(responseBody, json) + emitTelemetry( + req.agent, + LlmCallResult( + req.profile.providerId, req.model, usage?.inputTokens, usage?.outputTokens, durationMs, + streaming = false, + success = response.status.isSuccess(), + errorKind = if (response.status.isSuccess()) null else classifyHttpError(response.status), + statusCode = response.status.value + ) + ) + } + + private suspend fun proxyStreaming(req: ProxyRequest, call: ApplicationCall) { + httpClient.prepareRequest(req.upstreamUrl) { + configureProxy(req, call) + timeout { socketTimeoutMillis = req.timeoutMs } + }.execute { response -> + if (!response.status.isSuccess()) { + val errorBody = response.bodyAsText() + val durationMs = System.currentTimeMillis() - req.startTime + val upstreamContentType = response.contentType() ?: ContentType.Application.Json + call.respondText(errorBody, upstreamContentType, response.status) + emitTelemetry( + req.agent, + LlmCallResult( + req.profile.providerId, req.model, durationMs = durationMs, + streaming = true, success = false, + errorKind = classifyHttpError(response.status), + statusCode = response.status.value + ) + ) + return@execute + } + + call.response.header(HttpHeaders.ContentType, ContentType.Text.EventStream.toString()) + call.response.header(HttpHeaders.CacheControl, "no-store") + call.response.header("X-Accel-Buffering", "no") + + call.respondTextWriter { + val channel = response.bodyAsChannel() + val parser = req.profile.strategy.createStreamParser(json) + var totalChars = 0L + + try { + while (!channel.isClosedForRead) { + val line = channel.readUTF8Line() ?: break + totalChars += line.length + 1 + if (totalChars > MAX_STREAM_CHARS) { + val durationMs = System.currentTimeMillis() - req.startTime + emitTelemetry( + req.agent, + LlmCallResult( + req.profile.providerId, req.model, durationMs = durationMs, + streaming = true, success = false, + errorKind = LlmErrorKind.RESPONSE_TOO_LARGE, + statusCode = response.status.value + ) + ) + break + } + parser.processLine(line) + write(line) + write("\n") + flush() + } + + if (totalChars <= MAX_STREAM_CHARS) { + val durationMs = System.currentTimeMillis() - req.startTime + emitTelemetry( + req.agent, + LlmCallResult( + req.profile.providerId, req.model, + parser.inputTokens, parser.outputTokens, durationMs, + streaming = true, success = true, + statusCode = response.status.value, chunkCount = parser.chunkCount + ) + ) + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + val durationMs = System.currentTimeMillis() - req.startTime + emitTelemetry( + req.agent, + LlmCallResult( + req.profile.providerId, req.model, + parser.inputTokens, parser.outputTokens, durationMs, + streaming = true, success = false, + errorKind = classifyError(e), + statusCode = response.status.value + ) + ) + } + } + } + } + + private fun HttpRequestBuilder.configureProxy(req: ProxyRequest, call: ApplicationCall) { + method = call.request.httpMethod + timeout { requestTimeoutMillis = req.timeoutMs } + ProxyHeaders.applyUpstream(this, call, req.profile, req.apiKey) + if (req.hasBody) { + contentType(call.request.contentType()) + setBody(req.requestBody) + } + } + + private fun validateRequestShape(call: ApplicationCall) { + val method = call.request.httpMethod + if (method !in ALLOWED_METHODS) { + throw RouteException(HttpStatusCode.MethodNotAllowed, "Unsupported proxy method: $method") + } + + if (method in METHODS_WITH_BODY && !isSupportedJsonContentType(call.request.contentType())) { + throw RouteException( + HttpStatusCode.UnsupportedMediaType, + "LLM proxy only supports JSON request bodies" + ) + } + } + + private fun isSupportedJsonContentType(contentType: ContentType): Boolean { + val normalized = contentType.withoutParameters() + return normalized.match(ContentType.Application.Json) || + (normalized.contentType == "application" && normalized.contentSubtype.endsWith("+json")) + } + + private suspend fun readRequestBody(hasBody: Boolean, call: ApplicationCall): String { + if (!hasBody) return "" + val declaredLength = call.request.headers[HttpHeaders.ContentLength]?.toLongOrNull() + if (declaredLength != null && declaredLength > MAX_REQUEST_BODY_BYTES) { + throw RouteException( + HttpStatusCode.PayloadTooLarge, + "Request body exceeds ${MAX_REQUEST_BODY_BYTES / 1024 / 1024} MB limit" + ) + } + val body = call.receiveText() + if (body.encodeToByteArray().size > MAX_REQUEST_BODY_BYTES) { + throw RouteException( + HttpStatusCode.PayloadTooLarge, + "Request body exceeds ${MAX_REQUEST_BODY_BYTES / 1024 / 1024} MB limit" + ) + } + return body + } + + private fun tryParseJson(body: String): JsonObject? { + return try { + json.decodeFromString(body) + } catch (_: Exception) { + null + } + } + + private suspend fun readBoundedBody(response: HttpResponse): String { + val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull() + if (declaredLength != null && declaredLength > MAX_RESPONSE_BODY_BYTES) { + throw LlmProxyException("Upstream response exceeds ${MAX_RESPONSE_BODY_BYTES / 1024 / 1024} MB limit") + } + val body = response.bodyAsText() + if (body.encodeToByteArray().size > MAX_RESPONSE_BODY_BYTES) { + throw LlmProxyException("Upstream response exceeds ${MAX_RESPONSE_BODY_BYTES / 1024 / 1024} MB limit") + } + return body + } + + private fun classifyHttpError(status: HttpStatusCode): LlmErrorKind = when (status.value) { + 429 -> LlmErrorKind.RATE_LIMITED + in 401..403 -> LlmErrorKind.CREDENTIALS + in 500..599 -> LlmErrorKind.UPSTREAM_HEALTH + in 400..499 -> LlmErrorKind.REQUEST_ERROR + else -> LlmErrorKind.UNKNOWN + } + + private fun classifyError(e: Exception): LlmErrorKind = when (e) { + is ConnectTimeoutException -> LlmErrorKind.CONNECTIVITY + is HttpRequestTimeoutException -> LlmErrorKind.CONNECTIVITY + is LlmProxyException -> LlmErrorKind.RESPONSE_TOO_LARGE + else -> if (e.message?.contains( + "timeout", + ignoreCase = true + ) == true + ) LlmErrorKind.CONNECTIVITY else LlmErrorKind.UNKNOWN + } + + private suspend fun emitTelemetry(agent: SessionAgent, result: LlmCallResult) { + try { + if (result.success) { + val chunks = if (result.chunkCount != null) " ${result.chunkCount} chunks" else "" + val mode = if (result.streaming) "stream complete" else "${result.statusCode ?: "ok"}" + agent.logger.debug { "LLM Proxy ← $mode ${result.durationMs}ms$chunks${result.formatTokenInfo()}" } + } else { + val mode = if (result.streaming) " (stream)" else "" + agent.logger.warn { "LLM Proxy ← ${result.statusCode ?: "err"} ${result.durationMs}ms error=${result.errorKind}$mode" } + } + + agent.accumulateTokens(result.provider, result.model, result.inputTokens, result.outputTokens) + agent.session.events.emit( + SessionEvent.LlmProxyCall( + agentName = agent.name, + provider = result.provider, + model = result.model, + inputTokens = result.inputTokens, + outputTokens = result.outputTokens, + durationMs = result.durationMs, + streaming = result.streaming, + success = result.success, + errorKind = result.errorKind + ) + ) + } catch (e: Exception) { + agent.logger.error(e) { "Failed to emit LLM proxy telemetry event" } + } + } +} diff --git a/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/ProxyHeaders.kt b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/ProxyHeaders.kt new file mode 100644 index 00000000..b8b7277d --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/ProxyHeaders.kt @@ -0,0 +1,74 @@ +package org.coralprotocol.coralserver.llmproxy + +import io.ktor.client.request.HttpRequestBuilder +import io.ktor.client.request.header +import io.ktor.client.statement.HttpResponse +import io.ktor.http.HttpHeaders +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.header + +object ProxyHeaders { + private val HOP_BY_HOP = setOf( + HttpHeaders.Connection, + HttpHeaders.TransferEncoding, + HttpHeaders.Upgrade, + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + ) + + private val STRIP_REQUEST = (HOP_BY_HOP + setOf( + HttpHeaders.Authorization, + HttpHeaders.Host, + HttpHeaders.ContentLength, + HttpHeaders.ContentType, + HttpHeaders.AcceptEncoding, + HttpHeaders.Cookie, + "x-api-key", + )).map { it.lowercase() }.toSet() + + private val STRIP_RESPONSE = (HOP_BY_HOP + setOf( + HttpHeaders.ContentLength, + HttpHeaders.ContentEncoding, + HttpHeaders.SetCookie, + )).map { it.lowercase() }.toSet() + + fun applyUpstream(builder: HttpRequestBuilder, call: ApplicationCall, profile: LlmProviderProfile, apiKey: String) { + when (profile.authStyle) { + is AuthStyle.Bearer -> builder.header(HttpHeaders.Authorization, "Bearer $apiKey") + is AuthStyle.Custom -> builder.header(profile.authStyle.headerName, apiKey) + } + + profile.defaultHeaders.forEach { (name, value) -> builder.header(name, value) } + + val defaultLower = profile.defaultHeaders.keys.map { it.lowercase() }.toSet() + for ((name, values) in call.request.headers.entries()) { + val lower = name.lowercase() + if (lower in STRIP_REQUEST || lower in defaultLower) continue + values.forEach { builder.header(name, it) } + } + } + + fun forwardResponseHeaders(from: HttpResponse, call: ApplicationCall) { + for ((name, values) in from.headers.entries()) { + if (name.lowercase() in STRIP_RESPONSE) continue + values.forEach { call.response.header(name, it) } + } + } + + fun extractAgentKey(call: ApplicationCall, profile: LlmProviderProfile): String? { + return when (profile.authStyle) { + is AuthStyle.Bearer -> { + val authHeader = call.request.headers[HttpHeaders.Authorization] ?: return null + if (authHeader.startsWith("Bearer ", ignoreCase = true)) { + authHeader.substring(7).trim().ifEmpty { null } + } else null + } + is AuthStyle.Custom -> { + call.request.headers[profile.authStyle.headerName]?.trim()?.ifEmpty { null } + } + } + } +} diff --git a/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/ProxyRequest.kt b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/ProxyRequest.kt new file mode 100644 index 00000000..f82feb06 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/ProxyRequest.kt @@ -0,0 +1,15 @@ +package org.coralprotocol.coralserver.llmproxy + +import org.coralprotocol.coralserver.session.SessionAgent + +data class ProxyRequest( + val agent: SessionAgent, + val profile: LlmProviderProfile, + val apiKey: String, + val upstreamUrl: String, + val timeoutMs: Long, + val requestBody: String, + val model: String?, + val hasBody: Boolean, + val startTime: Long +) diff --git a/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/TokenUsage.kt b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/TokenUsage.kt new file mode 100644 index 00000000..221017a7 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/llmproxy/TokenUsage.kt @@ -0,0 +1,22 @@ +package org.coralprotocol.coralserver.llmproxy + +import kotlinx.serialization.Serializable +import java.util.concurrent.atomic.AtomicLong + +@Serializable +data class TokenUsage( + val inputTokens: Long = 0, + val outputTokens: Long = 0 +) + +class AtomicTokenUsage { + private val input = AtomicLong(0) + private val output = AtomicLong(0) + + fun add(inputTokens: Long?, outputTokens: Long?) { + if (inputTokens != null) input.addAndGet(inputTokens) + if (outputTokens != null) output.addAndGet(outputTokens) + } + + fun snapshot(): TokenUsage = TokenUsage(input.get(), output.get()) +} diff --git a/src/main/kotlin/org/coralprotocol/coralserver/modules/ConfigModule.kt b/src/main/kotlin/org/coralprotocol/coralserver/modules/ConfigModule.kt index 3f304164..c980d8c4 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/modules/ConfigModule.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/modules/ConfigModule.kt @@ -33,4 +33,5 @@ val configModuleParts = module { single(createdAtStart = true) { get().sessionConfig } single(createdAtStart = true) { get().loggingConfig } single(createdAtStart = true) { get().consoleConfig } + single(createdAtStart = true) { get().llmProxyConfig } } \ No newline at end of file diff --git a/src/main/kotlin/org/coralprotocol/coralserver/modules/LlmProxyModule.kt b/src/main/kotlin/org/coralprotocol/coralserver/modules/LlmProxyModule.kt new file mode 100644 index 00000000..bf77a4f2 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/modules/LlmProxyModule.kt @@ -0,0 +1,41 @@ +package org.coralprotocol.coralserver.modules + +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.HttpRequestRetry +import io.ktor.http.HttpStatusCode +import org.coralprotocol.coralserver.config.LlmProxyConfig +import org.coralprotocol.coralserver.llmproxy.LlmProxyService +import org.koin.core.qualifier.named +import org.koin.dsl.module + +const val LOGGER_LLM_PROXY = "llm-proxy" +const val LLM_PROXY_HTTP_CLIENT = "llmProxyHttpClient" + +val llmProxyModule = module { + single(named(LLM_PROXY_HTTP_CLIENT)) { + val config = get() + HttpClient(CIO) { + if (config.retryMaxAttempts > 0) { + install(HttpRequestRetry) { + maxRetries = config.retryMaxAttempts + retryIf { _, response -> + response.status == HttpStatusCode.Conflict || response.status.value in 500..599 + } + exponentialDelay( + base = 2.0, + baseDelayMs = config.retryInitialDelayMs, + maxDelayMs = config.retryMaxDelayMs + ) + } + } + } + } + single { + LlmProxyService( + config = get(), + httpClient = get(named(LLM_PROXY_HTTP_CLIENT)), + json = get() + ) + } +} diff --git a/src/main/kotlin/org/coralprotocol/coralserver/modules/LoggingModule.kt b/src/main/kotlin/org/coralprotocol/coralserver/modules/LoggingModule.kt index dec4c053..239f36da 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/modules/LoggingModule.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/modules/LoggingModule.kt @@ -125,4 +125,5 @@ val namedLoggingModule = module { single(named(LOGGER_CONFIG)) { get() } single(named(LOGGER_LOG_API)) { get() } single(named(LOGGER_LOCAL_SESSION)) { get() } + single(named(LOGGER_LLM_PROXY)) { get() } } \ No newline at end of file diff --git a/src/main/kotlin/org/coralprotocol/coralserver/modules/ktor/CoralServerModule.kt b/src/main/kotlin/org/coralprotocol/coralserver/modules/ktor/CoralServerModule.kt index 2d888b63..0e44b80c 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/modules/ktor/CoralServerModule.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/modules/ktor/CoralServerModule.kt @@ -242,6 +242,7 @@ fun Application.coralServerModule(isTest: Boolean = false) { // url / custom auth authApi() mcpRoutes() + llmProxyRoutes() eventRoutes() logRoutes() diff --git a/src/main/kotlin/org/coralprotocol/coralserver/routes/api/v1/LlmProxyApi.kt b/src/main/kotlin/org/coralprotocol/coralserver/routes/api/v1/LlmProxyApi.kt new file mode 100644 index 00000000..39f06c91 --- /dev/null +++ b/src/main/kotlin/org/coralprotocol/coralserver/routes/api/v1/LlmProxyApi.kt @@ -0,0 +1,37 @@ +package org.coralprotocol.coralserver.routes.api.v1 + +import io.ktor.http.* +import io.ktor.server.routing.* +import org.coralprotocol.coralserver.config.LlmProxyConfig +import org.coralprotocol.coralserver.llmproxy.LlmProxyService +import org.coralprotocol.coralserver.routes.RouteException +import org.coralprotocol.coralserver.session.LocalSessionManager +import org.coralprotocol.coralserver.session.SessionException +import org.koin.ktor.ext.inject + +fun Route.llmProxyRoutes() { + val localSessionManager by inject() + val llmProxyService by inject() + val llmProxyConfig by inject() + + route("/llm-proxy/{agentSecret}/{provider}/{path...}") { + handle { + if (!llmProxyConfig.enabled) { + throw RouteException(HttpStatusCode.ServiceUnavailable, "LLM proxy is disabled") + } + + val agentSecret = call.parameters["agentSecret"] + ?: throw RouteException(HttpStatusCode.BadRequest, "Missing agent secret") + val provider = call.parameters["provider"] + ?: throw RouteException(HttpStatusCode.BadRequest, "Missing provider") + + val agent = try { + localSessionManager.locateAgent(agentSecret).agent + } catch (_: SessionException.InvalidAgentSecret) { + throw RouteException(HttpStatusCode.Unauthorized, "Invalid agent secret") + } + + llmProxyService.proxyRequest(agent, provider, call.parameters.getAll("path") ?: emptyList(), call) + } + } +} diff --git a/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgent.kt b/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgent.kt index 34ac099a..32529645 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgent.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgent.kt @@ -21,6 +21,8 @@ import org.coralprotocol.coralserver.agent.graph.GraphAgent import org.coralprotocol.coralserver.agent.graph.UniqueAgentName import org.coralprotocol.coralserver.config.SessionConfig import org.coralprotocol.coralserver.events.SessionEvent +import org.coralprotocol.coralserver.llmproxy.AtomicTokenUsage +import org.coralprotocol.coralserver.llmproxy.TokenUsage import org.coralprotocol.coralserver.logging.LoggingTag import org.coralprotocol.coralserver.mcp.McpInstructionSnippet import org.coralprotocol.coralserver.mcp.McpResourceName @@ -32,6 +34,7 @@ import org.koin.core.component.KoinComponent import org.koin.core.component.get import org.koin.core.component.inject import java.util.concurrent.ConcurrentHashMap + import kotlin.time.Clock import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Instant @@ -96,6 +99,20 @@ class SessionAgent( */ val links = mutableSetOf() + /** + * Cumulative token usage from LLM proxy calls, broken down by provider/model. + */ + private val _tokensByModel = ConcurrentHashMap() + val tokensByModel: Map + get() = _tokensByModel.mapValues { it.value.snapshot() } + + fun accumulateTokens(provider: String, model: String?, input: Long?, output: Long?) { + if (input != null || output != null) { + val key = "$provider/${model ?: "unknown"}" + _tokensByModel.computeIfAbsent(key) { AtomicTokenUsage() }.add(input, output) + } + } + /** * A list of all ongoing waits this agent is performing */ @@ -563,7 +580,8 @@ class SessionAgent( status = status.value, description = description, links = links.map { it.name }.toSet(), - annotations = graphAgent.annotations + annotations = graphAgent.annotations, + tokensByModel = tokensByModel ) /** diff --git a/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgentExecutionContext.kt b/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgentExecutionContext.kt index 415f892f..4d07e0ba 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgentExecutionContext.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/session/SessionAgentExecutionContext.kt @@ -2,6 +2,7 @@ package org.coralprotocol.coralserver.session +import io.ktor.http.* import io.ktor.utils.io.* import kotlinx.coroutines.flow.update import org.coralprotocol.coralserver.agent.graph.GraphAgentProvider @@ -12,7 +13,9 @@ import org.coralprotocol.coralserver.agent.runtime.RuntimeId import org.coralprotocol.coralserver.config.AddressConsumer import org.coralprotocol.coralserver.config.DebugConfig import org.coralprotocol.coralserver.config.DockerConfig +import org.coralprotocol.coralserver.config.LlmProxyConfig import org.coralprotocol.coralserver.events.SessionEvent +import org.coralprotocol.coralserver.llmproxy.LlmProviderProfile import org.coralprotocol.coralserver.mcp.McpTransportType import org.coralprotocol.coralserver.session.reporting.SessionAgentUsageReport import org.coralprotocol.coralserver.util.utcTimeNow @@ -40,6 +43,7 @@ class SessionAgentExecutionContext( val debugConfig by inject() val dockerConfig by inject() + val llmProxyConfig by inject() val disposableResources = mutableListOf() @@ -125,6 +129,22 @@ class SessionAgentExecutionContext( if (agent.graphAgent.provider is GraphAgentProvider.Remote) this["CORAL_REMOTE_AGENT"] = "1" + + val llmProxies = registryAgent.llm?.proxies + if (llmProxyConfig.enabled && !llmProxies.isNullOrEmpty()) { + for (proxy in llmProxies) { + val proxyBaseUrl = URLBuilder( + applicationRuntimeContext.getLlmProxyUrl( + this@SessionAgentExecutionContext, + addressConsumer + ) + ) + + val profile = LlmProviderProfile.fromId(proxy.format) ?: continue + this["CORAL_PROXY_URL_${proxy.name}"] = + proxyBaseUrl.appendPathSegments(profile.providerId, profile.sdkPathSuffix).buildString() + } + } } } diff --git a/src/main/kotlin/org/coralprotocol/coralserver/session/state/SessionAgentState.kt b/src/main/kotlin/org/coralprotocol/coralserver/session/state/SessionAgentState.kt index c8005cb4..29fcd1fb 100644 --- a/src/main/kotlin/org/coralprotocol/coralserver/session/state/SessionAgentState.kt +++ b/src/main/kotlin/org/coralprotocol/coralserver/session/state/SessionAgentState.kt @@ -4,6 +4,7 @@ import io.github.smiley4.schemakenerator.core.annotations.Description import kotlinx.serialization.Serializable import org.coralprotocol.coralserver.agent.graph.UniqueAgentName import org.coralprotocol.coralserver.agent.registry.RegistryAgentIdentifier +import org.coralprotocol.coralserver.llmproxy.TokenUsage import org.coralprotocol.coralserver.session.SessionAgentStatus import org.coralprotocol.coralserver.session.SessionResource @@ -25,5 +26,8 @@ data class SessionAgentState( @Description("A list of agents that this agent is aware of, constructed from agent groups in the AgentGraph") val links: Set, - override val annotations: Map + override val annotations: Map, + + @Description("Token usage broken down by provider/model (e.g. 'openai/gpt-4.1')") + val tokensByModel: Map = emptyMap(), ) : SessionResource \ No newline at end of file diff --git a/src/test/kotlin/org/coralprotocol/coralserver/CoralTest.kt b/src/test/kotlin/org/coralprotocol/coralserver/CoralTest.kt index 1f298fad..64d88921 100644 --- a/src/test/kotlin/org/coralprotocol/coralserver/CoralTest.kt +++ b/src/test/kotlin/org/coralprotocol/coralserver/CoralTest.kt @@ -161,7 +161,9 @@ abstract class CoralTest(body: CoralTest.() -> Unit) : KoinTest, FunSpec(body as single(named(LOGGER_LOG_API)) { testLogger } single(named(LOGGER_TEST)) { testLogger } + single(named(LOGGER_LLM_PROXY)) { prodLogger } }, + llmProxyModule, module { single { Json { @@ -186,6 +188,9 @@ abstract class CoralTest(body: CoralTest.() -> Unit) : KoinTest, FunSpec(body as } } } + single(named(LLM_PROXY_HTTP_CLIENT)) { + createClient { } + } }, blockchainModule, agentModule, diff --git a/src/test/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderStrategyTest.kt b/src/test/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderStrategyTest.kt new file mode 100644 index 00000000..7682696f --- /dev/null +++ b/src/test/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProviderStrategyTest.kt @@ -0,0 +1,78 @@ +package org.coralprotocol.coralserver.llmproxy + +import io.kotest.matchers.nulls.shouldBeNull +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import kotlinx.serialization.json.Json +import org.coralprotocol.coralserver.CoralTest +import org.coralprotocol.coralserver.logging.Logger +import org.coralprotocol.coralserver.modules.LOGGER_LLM_PROXY +import org.koin.core.component.get +import org.koin.core.qualifier.named + +class LlmProviderStrategyTest : CoralTest({ + + val json = Json { ignoreUnknownKeys = true } + + test("extractsPromptTokensAndCompletionTokens") { + val promptTokens = 100L + val completionTokens = 25L + val totalTokens = 125L + + val body = + """{"usage":{"prompt_tokens":$promptTokens,"completion_tokens":$completionTokens,"total_tokens":$totalTokens}}""" + val usage = OpenAIStrategy.extractBufferedTokens(body, json).shouldNotBeNull() + usage.inputTokens.shouldBe(promptTokens) + usage.outputTokens.shouldBe(completionTokens) + } + + test("returnsNullsForMissingOrMalformedInput") { + OpenAIStrategy.extractBufferedTokens("""{"id":"test"}""", json).shouldBeNull() + OpenAIStrategy.extractBufferedTokens("not json", json).shouldBeNull() + } + + test("injectsStreamOptionsWhenAbsentPreservesWhenPresent") { + val logger = get(named(LOGGER_LLM_PROXY)) + + val without = """{"model":"gpt-4","stream":true,"messages":[]}""" + OpenAIStrategy.prepareStreamingRequest(without, json, logger).shouldContain("include_usage") + + val with = """{"model":"gpt-4","stream_options":{"include_usage":false}}""" + OpenAIStrategy.prepareStreamingRequest(with, json, logger).shouldBe(with) + } + + test("openaiStreamParserExtractsTokensFromFinalChunk") { + val promptTokens = 10L + val completionTokens = 2L + + val parser = OpenAIStrategy.createStreamParser(json) + parser.processLine("""data: {"choices":[{"delta":{"content":"Hello"}}]}""") + parser.processLine("""data: {"choices":[{"delta":{"content":" world"}}],"usage":{"prompt_tokens":$promptTokens,"completion_tokens":$completionTokens}}""") + parser.processLine("data: [DONE]") + + parser.inputTokens.shouldBe(promptTokens) + parser.outputTokens.shouldBe(completionTokens) + parser.chunkCount.shouldBe(2) + } + + test("anthropicStreamParserExtractsTokensFromMessageStartAndDelta") { + val parser = AnthropicStrategy.createStreamParser(json) + + val inputTokens = 42L + val outputTokens = 2L + + parser.processLine("event: message_start") + parser.processLine("""data: {"type":"message_start","message":{"usage":{"input_tokens":$inputTokens}}}""") + + parser.processLine("event: content_block_delta") + parser.processLine("""data: {"type":"content_block_delta","delta":{"text":"Hello"}}""") + + parser.processLine("event: message_delta") + parser.processLine("""data: {"type":"message_delta","usage":{"output_tokens":$outputTokens}}""") + + parser.inputTokens.shouldBe(inputTokens) + parser.outputTokens.shouldBe(outputTokens) + parser.chunkCount.shouldBe(3) + } +}) diff --git a/src/test/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProxyTest.kt b/src/test/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProxyTest.kt new file mode 100644 index 00000000..eb17cd81 --- /dev/null +++ b/src/test/kotlin/org/coralprotocol/coralserver/llmproxy/LlmProxyTest.kt @@ -0,0 +1,329 @@ +package org.coralprotocol.coralserver.llmproxy + +import io.kotest.assertions.ktor.client.shouldBeOK +import io.kotest.matchers.booleans.shouldBeFalse +import io.kotest.matchers.booleans.shouldBeTrue +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.equals.shouldBeEqual +import io.kotest.matchers.nulls.shouldBeNull +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.string.shouldContain +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.server.application.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withTimeout +import org.coralprotocol.coralserver.CoralTest +import org.coralprotocol.coralserver.agent.debug.PuppetDebugAgent +import org.coralprotocol.coralserver.agent.graph.GraphAgentProvider +import org.coralprotocol.coralserver.agent.runtime.RuntimeId +import org.coralprotocol.coralserver.config.LlmProxyConfig +import org.coralprotocol.coralserver.config.LlmProxyProviderConfig +import org.coralprotocol.coralserver.events.SessionEvent +import org.coralprotocol.coralserver.routes.api.v1.LocalSessions +import org.coralprotocol.coralserver.session.LocalSession +import org.coralprotocol.coralserver.session.LocalSessionManager +import org.coralprotocol.coralserver.session.SessionIdentifier +import org.coralprotocol.coralserver.utils.dsl.sessionRequest +import org.koin.core.context.loadKoinModules +import org.koin.dsl.module +import org.koin.test.inject +import java.util.* +import kotlin.time.Duration.Companion.seconds + +private const val totalTokens = 20L +private const val outputTokens = 5L +private const val inputTokens = 15L; + + +private const val MOCK_OPENAI_RESPONSE = """{ + "id":"chatcmpl-test","object":"chat.completion","model":"gpt-test", + "choices":[{"index":0,"message":{"role":"assistant","content":"Hello from upstream"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":$inputTokens,"completion_tokens":$outputTokens,"total_tokens":$totalTokens} +}""" + +class LlmProxyTest : CoralTest({ + + suspend fun withProxySession( + providerName: String, + apiKey: String, + baseUrl: String, + block: suspend (secret: String, session: LocalSession) -> Unit + ) { + val client by inject() + val localSessionManager by inject() + + loadKoinModules(module { + single { + LlmProxyConfig( + enabled = true, + providers = mapOf(providerName to LlmProxyProviderConfig(apiKey = apiKey, baseUrl = baseUrl)) + ) + } + }) + + val agentName = "test-agent" + val id: SessionIdentifier = client.authenticatedPost(LocalSessions.Session()) { + setBody(sessionRequest { + agentGraphRequest { + agent(PuppetDebugAgent.identifier) { + name = agentName + provider = GraphAgentProvider.Local(RuntimeId.FUNCTION) + } + isolateAllAgents() + } + }) + }.body() + + val session = localSessionManager.getSessions(id.namespace).first() + val secret = session.agents[agentName].shouldNotBeNull().secret + + try { + block(secret, session) + } finally { + session.cancelAndJoinAgents() + } + } + + test("proxyForwardsBufferedRequestToUpstreamAndExtractsTokens").config(invocationTimeout = 15.seconds) { + val client by inject() + val application by inject() + + val upstreamPath = "/mock-upstream-${UUID.randomUUID()}" + val capturedHeaders = CompletableDeferred>>() + + application.routing { + post("$upstreamPath/v1/chat/completions") { + capturedHeaders.complete( + call.request.headers.entries().associate { (k, v) -> k.lowercase() to v } + ) + call.respondText(MOCK_OPENAI_RESPONSE, ContentType.Application.Json) + } + } + + val key = "sk-test-key-${UUID.randomUUID()}" + withProxySession("openai", key, upstreamPath) { secret, session -> + val eventDeferred = async { + withTimeout(5.seconds) { + session.events.first { it is SessionEvent.LlmProxyCall } + } + } + + val response = client.post("/llm-proxy/$secret/openai/v1/chat/completions") { + contentType(ContentType.Application.Json) + header(HttpHeaders.Authorization, "Bearer should-be-stripped") + setBody("""{"model":"gpt-test","messages":[{"role":"user","content":"hi"}]}""") + } + + response.shouldBeOK() + response.bodyAsText().shouldContain("Hello from upstream") + + val headers = capturedHeaders.await() + headers["authorization"].shouldNotBeNull().shouldContainExactly("Bearer $key") + + val event = eventDeferred.await() as SessionEvent.LlmProxyCall + event.provider.shouldBeEqual("openai") + event.model.shouldNotBeNull().shouldBeEqual("gpt-test") + event.success.shouldBeTrue() + event.streaming.shouldBeFalse() + event.inputTokens.shouldNotBeNull().shouldBeEqual(inputTokens) + event.outputTokens.shouldNotBeNull().shouldBeEqual(outputTokens) + } + } + + test("proxyForwardsQueryParametersToUpstream").config(invocationTimeout = 15.seconds) { + val client by inject() + val application by inject() + + val upstreamPath = "/mock-upstream-query-${UUID.randomUUID()}" + val capturedQueryParameters = CompletableDeferred>>() + + application.routing { + post("$upstreamPath/v1/chat/completions") { + capturedQueryParameters.complete( + call.request.queryParameters.entries().associate { (key, values) -> key to values } + ) + call.respondText(MOCK_OPENAI_RESPONSE, ContentType.Application.Json) + } + } + + val key = "sk-test-key-${UUID.randomUUID()}" + withProxySession("openai", key, upstreamPath) { secret, _ -> + val response = client.post("/llm-proxy/$secret/openai/v1/chat/completions?limit=20&after=abc&tag=x&tag=y") { + contentType(ContentType.Application.Json) + setBody("""{"model":"gpt-test","messages":[{"role":"user","content":"hi"}]}""") + } + + response.shouldBeOK() + + val queryParameters = capturedQueryParameters.await() + queryParameters["limit"].shouldNotBeNull().shouldContainExactly("20") + queryParameters["after"].shouldNotBeNull().shouldContainExactly("abc") + queryParameters["tag"].shouldNotBeNull().shouldContainExactly("x", "y") + } + } + + test("proxyStripsCookieAndPreservesIncomingJsonContentType").config(invocationTimeout = 15.seconds) { + val client by inject() + val application by inject() + + val upstreamPath = "/mock-upstream-headers-${UUID.randomUUID()}" + val capturedHeaders = CompletableDeferred>>() + + application.routing { + post("$upstreamPath/v1/chat/completions") { + capturedHeaders.complete( + call.request.headers.entries().associate { (key, values) -> key.lowercase() to values } + ) + call.respondText(MOCK_OPENAI_RESPONSE, ContentType.Application.Json) + } + } + + val key = "sk-test-key-${UUID.randomUUID()}" + withProxySession("openai", key, upstreamPath) { secret, _ -> + val response = client.post("/llm-proxy/$secret/openai/v1/chat/completions") { + contentType(ContentType.parse("application/json; charset=utf-8")) + header(HttpHeaders.Cookie, "session=abc") + setBody("""{"model":"gpt-test","messages":[{"role":"user","content":"hi"}]}""") + } + + response.shouldBeOK() + + val headers = capturedHeaders.await() + headers[HttpHeaders.Cookie.lowercase()].shouldBeNull() + headers[HttpHeaders.ContentType.lowercase()].shouldNotBeNull() + .shouldContainExactly("application/json; charset=utf-8") + } + } + + test("proxyRejectsNonJsonBodyRequests").config(invocationTimeout = 15.seconds) { + val client by inject() + + val key = "sk-test-key-${UUID.randomUUID()}" + withProxySession("openai", key, "/mock-upstream-${UUID.randomUUID()}") { secret, _ -> + val response = client.post("/llm-proxy/$secret/openai/v1/chat/completions") { + contentType(ContentType.Text.Plain) + setBody("not-json") + } + + response.status.shouldBeEqual(HttpStatusCode.UnsupportedMediaType) + response.bodyAsText().shouldContain("JSON") + } + } + + test("proxyAllowsGetRequestsAndRejectsUnsupportedMethods").config(invocationTimeout = 15.seconds) { + val client by inject() + val application by inject() + + val upstreamPath = "/mock-upstream-methods-${UUID.randomUUID()}" + + application.routing { + get("$upstreamPath/v1/models") { + call.respondText("""{"data":[]}""", ContentType.Application.Json) + } + } + + val key = "sk-test-key-${UUID.randomUUID()}" + withProxySession("openai", key, upstreamPath) { secret, _ -> + client.get("/llm-proxy/$secret/openai/v1/models").shouldBeOK() + + val putResponse = client.put("/llm-proxy/$secret/openai/v1/models") { + contentType(ContentType.Application.Json) + setBody("{}") + } + + putResponse.status.shouldBeEqual(HttpStatusCode.MethodNotAllowed) + putResponse.bodyAsText().shouldContain("Unsupported proxy method") + } + } + + test("anthropicXApiKeyPassThrough").config(invocationTimeout = 15.seconds) { + val client by inject() + val localSessionManager by inject() + val application by inject() + + val upstreamPath = "/mock-upstream-anthropic-${UUID.randomUUID()}" + val capturedHeaders = CompletableDeferred>>() + + application.routing { + post("$upstreamPath/v1/messages") { + capturedHeaders.complete( + call.request.headers.entries().associate { (k, v) -> k.lowercase() to v } + ) + call.respondText(MOCK_OPENAI_RESPONSE, ContentType.Application.Json) + } + } + + loadKoinModules(module { + single { + LlmProxyConfig( + enabled = true, + providers = mapOf("anthropic" to LlmProxyProviderConfig(baseUrl = upstreamPath)) + ) + } + }) + + val agentName = "test-agent" + val id: SessionIdentifier = client.authenticatedPost(LocalSessions.Session()) { + setBody(sessionRequest { + agentGraphRequest { + agent(PuppetDebugAgent.identifier) { + name = agentName + provider = GraphAgentProvider.Local(RuntimeId.FUNCTION) + } + isolateAllAgents() + } + }) + }.body() + + val session = localSessionManager.getSessions(id.namespace).first() + val secret = session.agents[agentName].shouldNotBeNull().secret + + try { + val key = "sk-ant-agent-key-${UUID.randomUUID()}" + val response = client.post("/llm-proxy/$secret/anthropic/v1/messages") { + contentType(ContentType.Application.Json) + header("x-api-key", key) + setBody("""{"model":"claude-sonnet-4-20250514","messages":[{"role":"user","content":"hi"}]}""") + } + + response.shouldBeOK() + capturedHeaders.await()["x-api-key"].shouldNotBeNull().shouldContainExactly(key) + } finally { + session.cancelAndJoinAgents() + } + } + + + // TODO: prototype runtime will be changed soon so that it will only use the proxy, the interface for that is WIP +// val openaiApiKey = System.getenv("CORAL_TEST_OPENAI_API_KEY") +// +// test("e2eProxyWithRealOpenai").config( +// enabled = openaiApiKey != null, +// invocationTimeout = 1.minutes +// ) { +// loadKoinModules(module { +// single { +// LlmProxyConfig( +// enabled = true, +// providers = mapOf("openai" to LlmProxyProviderConfig(apiKey = openaiApiKey!!)) +// ) +// } +// }) +// +// multiAgentPayloadTest( +// PrototypeModelProvider.OpenAI( +// PrototypeString.Inline("proxy-managed"), +// PrototypeString.Inline("gpt-4.1-nano"), +// url = PrototypeApiUrl.Proxy +// ) +// ) +// } +}) diff --git a/src/test/kotlin/org/coralprotocol/coralserver/registry/RegistryAgentTest.kt b/src/test/kotlin/org/coralprotocol/coralserver/registry/RegistryAgentTest.kt index f30d9303..7378fae2 100644 --- a/src/test/kotlin/org/coralprotocol/coralserver/registry/RegistryAgentTest.kt +++ b/src/test/kotlin/org/coralprotocol/coralserver/registry/RegistryAgentTest.kt @@ -1377,4 +1377,89 @@ class RegistryAgentTest : CoralTest({ }.validate() } } + + test("testLlmProxies") { + shouldNotThrowAny { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + proxy("GPT", "openai", "gpt-4o") + proxy("CLAUDE", "anthropic", "claude-3-5-sonnet") + } + }.validate() + } + + // too many llm proxies + shouldThrow { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + repeat(AGENT_LLM_PROXIES_MAX_ENTRIES + 1) { + proxy("P$it", "openai") + } + } + }.validate() + } + + // proxy name too short + shouldThrow { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + proxy("", "openai") + } + }.validate() + } + + // proxy name too long + shouldThrow { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + proxy("A".repeat(AGENT_LLM_PROXY_NAME_LENGTH.last + 1), "openai") + } + }.validate() + } + + // proxy name invalid pattern + shouldThrow { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + proxy("gpt", "openai") + } + }.validate() + } + + // proxy name not unique + shouldThrow { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + proxy("GPT", "openai") + proxy("GPT", "anthropic") + } + }.validate() + } + + // proxy format unknown + shouldThrow { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + proxy("GPT", "unknown-format") + } + }.validate() + } + + // proxy model too long + shouldThrow { + registryAgent("valid") { + runtime(FunctionRuntime()) + llm { + proxy("GPT", "openai", "m".repeat(AGENT_LLM_PROXY_MODEL_LENGTH.last + 1)) + } + }.validate() + } + } }) diff --git a/src/test/kotlin/org/coralprotocol/coralserver/utils/dsl/RegistryAgentBuilder.kt b/src/test/kotlin/org/coralprotocol/coralserver/utils/dsl/RegistryAgentBuilder.kt index d6a805fd..3108285c 100644 --- a/src/test/kotlin/org/coralprotocol/coralserver/utils/dsl/RegistryAgentBuilder.kt +++ b/src/test/kotlin/org/coralprotocol/coralserver/utils/dsl/RegistryAgentBuilder.kt @@ -83,6 +83,7 @@ class RegistryAgentBuilder( private val options: MutableMap = mutableMapOf() private val unresolvedExportSettings: MutableMap = mutableMapOf() private var marketplace: RegistryAgentMarketplaceSettings? = null + private var llm: AgentLlmConfig? = null fun link(name: String, value: String) { links[name] = value @@ -108,6 +109,10 @@ class RegistryAgentBuilder( marketplace = RegistryAgentMarketplaceSettingsBuilder().apply(block).build() } + fun llm(block: AgentLlmConfigBuilder.() -> Unit) { + llm = AgentLlmConfigBuilder().apply(block).build() + } + fun runtime(functionRuntime: FunctionRuntime) { runtimes = LocalAgentRuntimes( @@ -165,11 +170,23 @@ class RegistryAgentBuilder( options = options, path = path, unresolvedExportSettings = unresolvedExportSettings, - marketplace = marketplace + marketplace = marketplace, + llm = llm ) } } +@TestDsl +class AgentLlmConfigBuilder { + private val proxies = mutableListOf() + + fun proxy(name: String, format: String, model: String? = null) { + proxies += AgentLlmProxy(name, format, model) + } + + fun build() = AgentLlmConfig(proxies = proxies.toList()) +} + @TestDsl class RegistryAgentMarketplaceSettingsBuilder { private var pricing: RegistryAgentMarketplacePricing? = null diff --git a/src/test/resources/agent/coral-agent.toml b/src/test/resources/agent/coral-agent.toml index 0a20480e..c5c8171f 100644 --- a/src/test/resources/agent/coral-agent.toml +++ b/src/test/resources/agent/coral-agent.toml @@ -153,6 +153,15 @@ auth.header.parts = [ { type = "option", name = "OPTIONAL_STRING" } ] +[[llm.proxies]] +name = "MAIN" +format = "openai" +model = "gpt-4.1" + +[[llm.proxies]] +name = "REASONING" +format = "anthropic" + # option key: # len: 1 - 256 # pattern: ^[a-zA-Z_][a-zA-Z_$0-9]*$