diff --git a/.changeset/some-lights-lead.md b/.changeset/some-lights-lead.md new file mode 100644 index 000000000..0c5e41674 --- /dev/null +++ b/.changeset/some-lights-lead.md @@ -0,0 +1,8 @@ +--- +'@livekit/agents-plugin-baseten': patch +--- + +Add @livekit/agents-plugin-baseten with support for Baseten-hosted models: +- STT: WebSocket-based streaming speech-to-text with VAD and partial transcripts +- TTS: Streaming text-to-speech using Baseten's Orpheus model (24kHz) +- LLM: OpenAI-compatible API integration for Baseten inference \ No newline at end of file diff --git a/examples/package.json b/examples/package.json index 1bcac74c9..dee5a6332 100644 --- a/examples/package.json +++ b/examples/package.json @@ -12,7 +12,8 @@ "clean:build": "pnpm clean && pnpm build", "lint": "eslint -f unix \"src/**/*.ts\"", "test": "vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "baseten:cloud": "tsx --env-file=.env src/baseten_restaurant_agent.ts dev --log-level=debug" }, "devDependencies": { "@types/node": "^22.5.5", @@ -24,6 +25,7 @@ "dependencies": { "@livekit/agents": "workspace:*", "@livekit/agents-plugin-anam": "workspace:*", + "@livekit/agents-plugin-baseten": "workspace:*", "@livekit/agents-plugin-bey": "workspace:*", "@livekit/agents-plugin-cartesia": "workspace:*", "@livekit/agents-plugin-deepgram": "workspace:*", diff --git a/plugins/baseten/README.md b/plugins/baseten/README.md new file mode 100644 index 000000000..0c6ab8ecf --- /dev/null +++ b/plugins/baseten/README.md @@ -0,0 +1,92 @@ + + +# LiveKit Agents Baseten Plugin + +Node.js/TypeScript plugin for LiveKit Agents with Baseten-hosted models (LLM, STT, TTS). + +## Installation + +```bash +cd packages/livekit-plugin-baseten +pnpm install +pnpm build +``` + +## Configuration + +Create `.env` file: + +```bash +BASETEN_API_KEY=your_api_key_here +BASETEN_MODEL_ID=your_llm_model_id +BASETEN_TTS_MODEL_ID=your_tts_model_id +BASETEN_STT_MODEL_ID=your_stt_model_id +``` + +## Usage + +### LLM + +```typescript +import { LLM } from 'livekit-plugin-baseten' + +const llm = new LLM({ + model: 'openai/gpt-4o-mini', + apiKey: process.env.BASETEN_API_KEY +}) +``` + +### STT + +```typescript +import { STT } from 'livekit-plugin-baseten' + +const stt = new STT({ + apiKey: process.env.BASETEN_API_KEY, + modelId: process.env.BASETEN_STT_MODEL_ID +}) + +const stream = stt.stream() +for await (const event of stream) { + // Handle speech events +} +``` + +### TTS + +```typescript +import { TTS } from 'livekit-plugin-baseten' + +const tts = new TTS({ + apiKey: process.env.BASETEN_API_KEY, + modelEndpoint: 'your-model-endpoint-url' +}) + +const stream = tts.synthesize('Hello world') +for await (const frame of stream) { + // Process audio frames +} +``` + +## Testing + +```bash +pnpm test:llm-cli # Interactive LLM chat +pnpm test:tts-cli # TTS synthesis +pnpm test:stt-cli # STT with microphone +``` + +See [TESTING.md](./test/TESTING.md) for details. + +## Development + +```bash +pnpm build # Build +pnpm dev # Watch mode +pnpm typecheck # Type checking +pnpm lint # Linting +``` diff --git a/plugins/baseten/package.json b/plugins/baseten/package.json new file mode 100644 index 000000000..3ab7aab1a --- /dev/null +++ b/plugins/baseten/package.json @@ -0,0 +1,53 @@ +{ + "name": "@livekit/agents-plugin-baseten", + "version": "1.0.27", + "description": "Baseten plugin for LiveKit Node Agents", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "author": "LiveKit", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "lint": "eslint \"src/**/*.ts\"", + "lint:fix": "eslint --fix \"src/**/*.ts\"" + }, + "keywords": [ + "livekit", + "agents", + "baseten", + "llm", + "stt", + "tts", + "voice-ai" + ], + "dependencies": { + "@livekit/agents": "workspace:*", + "@livekit/agents-plugin-openai": "^1.0.0", + "@livekit/rtc-node": "^0.13.12", + "@livekit/agents-plugins-test": "workspace:*", + "dotenv": "^17.2.3", + "openai": "^4.0.0", + "ws": "^8.14.2" + }, + "devDependencies": { + "@types/node": "^22.18.11", + "@types/ws": "^8.5.8", + "tsx": "^4.7.0", + "typescript": "^5.9.3" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "^0.13.12" + } +} diff --git a/plugins/baseten/src/index.ts b/plugins/baseten/src/index.ts new file mode 100644 index 000000000..69d2a1393 --- /dev/null +++ b/plugins/baseten/src/index.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * LiveKit Agents Baseten Plugin + * + * Integrates Baseten-hosted models with LiveKit Agents for LLM, STT, and TTS services. + */ +import { Plugin } from '@livekit/agents'; + +class BasetenPlugin extends Plugin { + constructor() { + super({ + title: 'baseten', + version: '1.0.0', + package: 'livekit-plugin-baseten', + }); + } +} + +Plugin.registerPlugin(new BasetenPlugin()); + +// Export classes following LiveKit plugin pattern +export { LLM } from './llm.js'; +export { STT } from './stt.js'; +export { TTS } from './tts.js'; + +// Export all types +export type { BasetenLLMOptions, BasetenSttOptions, BasetenTTSOptions } from './types.js'; diff --git a/plugins/baseten/src/llm.test.ts b/plugins/baseten/src/llm.test.ts new file mode 100644 index 000000000..dad41eabe --- /dev/null +++ b/plugins/baseten/src/llm.test.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents-plugins-test'; +import { describe } from 'vitest'; +import { LLM } from './llm.js'; + +describe('Baseten', async () => { + await llm( + new LLM({ + model: 'openai/gpt-4o-mini', + temperature: 0, + }), + false, + ); +}); diff --git a/plugins/baseten/src/llm.ts b/plugins/baseten/src/llm.ts new file mode 100644 index 000000000..6b178ca3c --- /dev/null +++ b/plugins/baseten/src/llm.ts @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Baseten LLM plugin for LiveKit Agents + * Configures the OpenAI plugin to work with Baseten's OpenAI-compatible API + */ +import { LLM as OpenAILLM } from '@livekit/agents-plugin-openai'; +import type { BasetenLLMOptions } from './types.js'; + +export class LLM extends OpenAILLM { + constructor(opts: BasetenLLMOptions) { + const apiKey = opts.apiKey ?? process.env.BASETEN_API_KEY; + if (!apiKey) { + throw new Error( + 'Baseten API key is required. Set BASETEN_API_KEY environment variable or pass apiKey in options.', + ); + } + + if (!opts.model) { + throw new Error( + 'Model is required. Please specify a model name (e.g., "openai/gpt-4o-mini").', + ); + } + + const model = opts.model; + + // Configure the OpenAI plugin with Baseten's endpoint + super({ + model, + apiKey, + baseURL: 'https://inference.baseten.co/v1', + temperature: opts.temperature, + user: opts.user, + maxCompletionTokens: opts.maxTokens, + toolChoice: opts.toolChoice, + parallelToolCalls: opts.parallelToolCalls, + }); + } + + label(): string { + return 'baseten.LLM'; + } +} diff --git a/plugins/baseten/src/stt.test.ts b/plugins/baseten/src/stt.test.ts new file mode 100644 index 000000000..e77e1ed37 --- /dev/null +++ b/plugins/baseten/src/stt.test.ts @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { VAD } from '@livekit/agents-plugin-silero'; +import { stt } from '@livekit/agents-plugins-test'; +import { describe } from 'vitest'; +import { STT } from './stt.js'; + +describe('Baseten', async () => { + await stt(new STT(), await VAD.load(), { streaming: true }); +}); diff --git a/plugins/baseten/src/stt.ts b/plugins/baseten/src/stt.ts new file mode 100644 index 000000000..771802877 --- /dev/null +++ b/plugins/baseten/src/stt.ts @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { type AudioBuffer, AudioByteStream, Task, log, stt, waitForAbort } from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import { WebSocket } from 'ws'; +import type { BasetenSttOptions } from './types.js'; + +const defaultSTTOptions: Partial = { + environment: 'production', + encoding: 'pcm_s16le', + sampleRate: 16000, + bufferSizeSeconds: 0.032, + enablePartialTranscripts: true, + partialTranscriptIntervalS: 0.5, + finalTranscriptMaxDurationS: 5, + audioLanguage: 'en', + languageDetectionOnly: false, + vadThreshold: 0.5, + vadMinSilenceDurationMs: 300, + vadSpeechPadMs: 30, +}; + +export class STT extends stt.STT { + #opts: BasetenSttOptions; + #logger = log(); + label = 'baseten.STT'; + + constructor(opts: Partial = {}) { + super({ + streaming: true, + interimResults: opts.enablePartialTranscripts ?? defaultSTTOptions.enablePartialTranscripts!, + }); + + const apiKey = opts.apiKey ?? process.env.BASETEN_API_KEY; + const modelId = opts.modelId ?? process.env.BASETEN_STT_MODEL_ID; + + if (!apiKey) { + throw new Error( + 'Baseten API key is required, either pass it as `apiKey` or set $BASETEN_API_KEY', + ); + } + if (!modelId) { + throw new Error( + 'Baseten model ID is required, either pass it as `modelId` or set $BASETEN_STT_MODEL_ID', + ); + } + + this.#opts = { + ...defaultSTTOptions, + ...opts, + apiKey, + modelId, + } as BasetenSttOptions; + } + + // eslint-disable-next-line + async _recognize(_: AudioBuffer): Promise { + throw new Error('Recognize is not supported on Baseten STT'); + } + + updateOptions(opts: Partial) { + this.#opts = { ...this.#opts, ...opts }; + } + + stream(): SpeechStream { + return new SpeechStream(this, this.#opts); + } +} + +export class SpeechStream extends stt.SpeechStream { + #opts: BasetenSttOptions; + #logger = log(); + #speaking = false; + #requestId = ''; + label = 'baseten.SpeechStream'; + + constructor(stt: STT, opts: BasetenSttOptions) { + super(stt, opts.sampleRate); + this.#opts = opts; + this.closed = false; + } + + private getWsUrl(): string { + return `wss://model-${this.#opts.modelId}.api.baseten.co/environments/${this.#opts.environment}/websocket`; + } + + protected async run() { + const maxRetry = 32; + let retries = 0; + + while (!this.input.closed && !this.closed) { + const url = this.getWsUrl(); + const headers = { + Authorization: `Api-Key ${this.#opts.apiKey}`, + }; + + const ws = new WebSocket(url, { headers }); + + try { + await new Promise((resolve, reject) => { + ws.on('open', resolve); + ws.on('error', (error) => reject(error)); + ws.on('close', (code) => reject(`WebSocket returned ${code}`)); + }); + + await this.#runWS(ws); + } catch (e) { + if (!this.closed && !this.input.closed) { + if (retries >= maxRetry) { + throw new Error(`failed to connect to Baseten after ${retries} attempts: ${e}`); + } + + const delay = Math.min(retries * 5, 10); + retries++; + + this.#logger.warn( + `failed to connect to Baseten, retrying in ${delay} seconds: ${e} (${retries}/${maxRetry})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay * 1000)); + } else { + this.#logger.warn( + `Baseten disconnected, connection is closed: ${e} (inputClosed: ${this.input.closed}, isClosed: ${this.closed})`, + ); + } + } + } + + this.closed = true; + } + + async #runWS(ws: WebSocket) { + let closing = false; + + // Send initial metadata + const metadata = { + streaming_vad_config: { + threshold: this.#opts.vadThreshold, + min_silence_duration_ms: this.#opts.vadMinSilenceDurationMs, + speech_pad_ms: this.#opts.vadSpeechPadMs, + }, + streaming_params: { + encoding: this.#opts.encoding ?? 'pcm_s16le', + sample_rate: this.#opts.sampleRate ?? 16000, + enable_partial_transcripts: this.#opts.enablePartialTranscripts, + partial_transcript_interval_s: this.#opts.partialTranscriptIntervalS, + final_transcript_max_duration_s: this.#opts.finalTranscriptMaxDurationS, + }, + whisper_params: { + prompt: this.#opts.prompt, + audio_language: this.#opts.audioLanguage ?? 'en', + language_detection_only: this.#opts.languageDetectionOnly ?? false, + }, + }; + ws.send(JSON.stringify(metadata)); + + const sendTask = async () => { + const sampleRate = this.#opts.sampleRate ?? 16000; + const samplesPerChunk = sampleRate === 16000 ? 512 : 256; + const audioByteStream = new AudioByteStream(sampleRate, 1, samplesPerChunk); + + try { + while (!this.closed) { + const result = await this.input.next(); + if (result.done) { + break; + } + + const data = result.value; + + let frames: AudioFrame[]; + if (data === SpeechStream.FLUSH_SENTINEL) { + // Flush any remaining buffered audio + frames = audioByteStream.flush(); + } else { + if (data.sampleRate !== sampleRate || data.channels !== 1) { + throw new Error( + `sample rate or channel count mismatch: expected ${sampleRate}Hz/1ch, got ${data.sampleRate}Hz/${data.channels}ch`, + ); + } + frames = audioByteStream.write(data.data.buffer as ArrayBuffer); + } + + for (const frame of frames) { + const buffer = Buffer.from( + frame.data.buffer, + frame.data.byteOffset, + frame.data.byteLength, + ); + ws.send(buffer); + } + } + } finally { + closing = true; + ws.close(); + } + }; + + const listenTask = Task.from(async (controller) => { + const listenMessage = new Promise((resolve, reject) => { + ws.on('message', (data) => { + try { + let jsonString: string; + + if (typeof data === 'string') { + jsonString = data; + } else if (data instanceof Buffer) { + jsonString = data.toString('utf-8'); + } else if (Array.isArray(data)) { + jsonString = Buffer.concat(data).toString('utf-8'); + } else { + return; + } + + const msg = JSON.parse(jsonString); + + // Parse response format matching Python implementation + const isFinal = msg.is_final ?? true; + const segments = msg.segments ?? []; + const transcript = msg.transcript ?? ''; + const confidence = msg.confidence ?? 0.0; + const languageCode = msg.language_code ?? this.#opts.audioLanguage; + + // Skip if no transcript text + if (!transcript) { + this.#logger.debug('Received non-transcript message:', msg); + return; + } + + // Emit START_OF_SPEECH if not already speaking (only for interim or first final) + if (!this.#speaking && !isFinal) { + this.#speaking = true; + this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH }); + } + + // Extract timing from segments + const startTime = segments.length > 0 ? segments[0].start ?? 0.0 : 0.0; + const endTime = segments.length > 0 ? segments[segments.length - 1].end ?? 0.0 : 0.0; + + const speechData: stt.SpeechData = { + language: languageCode!, + text: transcript, + startTime, + endTime, + confidence, + }; + + // Handle interim vs final transcripts (matching Python implementation) + if (!isFinal) { + // Interim transcript + this.queue.put({ + type: stt.SpeechEventType.INTERIM_TRANSCRIPT, + alternatives: [speechData], + }); + } else { + // Final transcript + this.queue.put({ + type: stt.SpeechEventType.FINAL_TRANSCRIPT, + alternatives: [speechData], + }); + + // Emit END_OF_SPEECH after final transcript + if (this.#speaking) { + this.#speaking = false; + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); + } + } + + if (this.closed || closing) { + resolve(); + } + } catch (err) { + this.#logger.error(`STT: Error processing message: ${data}`); + reject(err); + } + }); + + ws.on('error', (err) => { + if (!closing) { + reject(err); + } + }); + + ws.on('close', () => { + if (!closing) { + resolve(); + } + }); + }); + + await Promise.race([listenMessage, waitForAbort(controller.signal)]); + }, this.abortController); + + await Promise.all([sendTask(), listenTask.result]); + closing = true; + ws.close(); + } +} diff --git a/plugins/baseten/src/tts.test.ts b/plugins/baseten/src/tts.test.ts new file mode 100644 index 000000000..80fff60e9 --- /dev/null +++ b/plugins/baseten/src/tts.test.ts @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { tts } from '@livekit/agents-plugins-test'; +import { describe } from 'vitest'; +import { STT } from './stt.js'; +import { TTS } from './tts.js'; + +describe('Baseten', async () => { + await tts(new TTS(), new STT(), { streaming: false }); +}); diff --git a/plugins/baseten/src/tts.ts b/plugins/baseten/src/tts.ts new file mode 100644 index 000000000..ee4e75ae1 --- /dev/null +++ b/plugins/baseten/src/tts.ts @@ -0,0 +1,202 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + AudioByteStream, + shortuuid, + tts, + waitForAbort, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import type { BasetenTTSOptions } from './types.js'; + +const defaultTTSOptions: Partial = { + voice: 'tara', + language: 'en', + temperature: 0.6, +}; + +/** + * Baseten TTS implementation (streaming, 24kHz mono) + */ +export class TTS extends tts.TTS { + private opts: BasetenTTSOptions; + label = 'baseten.TTS'; + private abortController = new AbortController(); + constructor(opts: Partial = {}) { + /** + * Baseten audio is 24kHz mono. + * The Orpheus model generates audio chunks that are processed as they arrive, + * which reduces latency and improves agent responsiveness. + */ + super(24000, 1, { streaming: false }); + + // Apply defaults and environment fallbacks. + const apiKey = opts.apiKey ?? process.env.BASETEN_API_KEY; + const modelEndpoint = opts.modelEndpoint ?? process.env.BASETEN_MODEL_ENDPOINT; + + if (!apiKey) { + throw new Error( + 'Baseten API key is required, either pass it as `apiKey` or set $BASETEN_API_KEY', + ); + } + if (!modelEndpoint) { + throw new Error( + 'Baseten model endpoint is required, either pass it as `modelEndpoint` or set $BASETEN_MODEL_ENDPOINT', + ); + } + + this.opts = { + ...defaultTTSOptions, + ...opts, + apiKey, + modelEndpoint, + } as BasetenTTSOptions; + } + + updateOptions(opts: Partial>) { + this.opts = { + ...this.opts, + ...opts, + } as BasetenTTSOptions; + } + + /** + * Synthesize speech for a given piece of text. Returns a `ChunkedStream` + * which will asynchronously fetch audio from Baseten and push frames into + * LiveKit's playback pipeline. If you need to cancel synthesis you can + * call {@link ChunkedStream.stop} on the returned object. + */ + synthesize( + text: string, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ): ChunkedStream { + return new ChunkedStream(this, text, this.opts, connOptions, abortSignal); + } + + stream(): tts.SynthesizeStream { + throw new Error('Streaming is not supported on Baseten TTS'); + } + + async close(): Promise { + this.abortController.abort(); + } +} + +/** + * Internal helper that performs the actual HTTP request and converts the + * response into audio frames. It inherits from `tts.ChunkedStream` to + * integrate with LiveKit's event and cancellation framework. + * + * This implementation streams audio chunks as they arrive from the Baseten + * model endpoint, processing them incrementally instead of waiting for the + * complete response. + */ +export class ChunkedStream extends tts.ChunkedStream { + label = 'baseten.ChunkedStream'; + private readonly opts: BasetenTTSOptions; + + constructor( + tts: TTS, + text: string, + opts: BasetenTTSOptions, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ) { + super(text, tts, connOptions, abortSignal); + this.opts = opts; + } + + /** + * Execute the synthesis request. This method is automatically invoked + * by the base class when the stream starts. It performs a POST request + * to the configured `modelEndpoint` with the input text and optional + * parameters. Audio chunks are streamed as they arrive and transformed + * into a sequence of `AudioFrame` objects that are enqueued immediately + * for playback. + */ + protected async run() { + const { apiKey, modelEndpoint, voice, language, temperature, maxTokens } = this.opts; + const payload: Record = { + prompt: this.inputText, + }; + if (voice) payload.voice = voice; + if (language) payload.language = language; + if (temperature !== undefined) payload.temperature = temperature; + if (maxTokens !== undefined) payload.max_tokens = maxTokens; + + const headers: Record = { + Authorization: `Api-Key ${apiKey}`, + 'Content-Type': 'application/json', + }; + + const response = await fetch(modelEndpoint, { + method: 'POST', + headers, + body: JSON.stringify(payload), + signal: this.abortSignal, + }); + + if (!response.ok) { + let errText: string; + try { + errText = await response.text(); + } catch { + errText = response.statusText; + } + throw new Error(`Baseten TTS request failed: ${response.status} ${errText}`); + } + + // Stream the response body as chunks arrive + if (!response.body) { + throw new Error('Response body is not available for streaming'); + } + + const requestId = shortuuid(); + const audioByteStream = new AudioByteStream(24000, 1); + const reader = response.body.getReader(); + + try { + let lastFrame: AudioFrame | undefined; + const sendLastFrame = (segmentId: string, final: boolean) => { + if (lastFrame) { + this.queue.put({ requestId, segmentId, frame: lastFrame, final }); + lastFrame = undefined; + } + }; + + // waitForAbort internally sets up an abort listener on the abort signal + // we need to put it outside loop to avoid constant re-registration of the listener + const abortPromise = waitForAbort(this.abortSignal); + + while (!this.abortSignal.aborted) { + const result = await Promise.race([reader.read(), abortPromise]); + + if (result === undefined) break; // aborted + + const { done, value } = result; + + if (done) { + break; + } + + // Process the chunk and convert to audio frames + // Convert Uint8Array to ArrayBuffer for AudioByteStream + const frames = audioByteStream.write(value.buffer); + + for (const frame of frames) { + sendLastFrame(requestId, false); + lastFrame = frame; + } + } + + // Send the final frame + sendLastFrame(requestId, true); + } finally { + reader.releaseLock(); + this.queue.close(); + } + } +} diff --git a/plugins/baseten/src/types.ts b/plugins/baseten/src/types.ts new file mode 100644 index 000000000..1aaac4526 --- /dev/null +++ b/plugins/baseten/src/types.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Baseten plugin types and interfaces + */ + +/** + * Options for configuring the Baseten LLM + * Since Baseten provides an OpenAI-compatible API, these options + * map to standard OpenAI parameters. + */ +export interface BasetenLLMOptions { + apiKey?: string; + model: string; + temperature?: number; + maxTokens?: number; + user?: string; + toolChoice?: 'none' | 'auto' | 'required' | { type: 'function'; function: { name: string } }; + parallelToolCalls?: boolean; +} + +/** + * Options for configuring the Baseten STT service + */ +export interface BasetenSttOptions { + apiKey: string; + modelId: string; + environment?: string; + encoding?: string; + sampleRate?: number; + bufferSizeSeconds?: number; + vadThreshold?: number; + vadMinSilenceDurationMs?: number; + vadSpeechPadMs?: number; + enablePartialTranscripts?: boolean; + partialTranscriptIntervalS?: number; + finalTranscriptMaxDurationS?: number; + audioLanguage?: string; + prompt?: string; + languageDetectionOnly?: boolean; +} + +/** + * Options for configuring the Baseten TTS service + */ +export interface BasetenTTSOptions { + apiKey: string; + modelEndpoint: string; + voice?: string; + language?: string; + temperature?: number; + maxTokens?: number; +} diff --git a/plugins/baseten/tsconfig.json b/plugins/baseten/tsconfig.json new file mode 100644 index 000000000..eb954c871 --- /dev/null +++ b/plugins/baseten/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "node", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "noImplicitThis": true, + "alwaysStrict": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 39b53189a..6700fae02 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -221,6 +221,9 @@ importers: '@livekit/agents-plugin-anam': specifier: workspace:* version: link:../plugins/anam + '@livekit/agents-plugin-baseten': + specifier: workspace:* + version: link:../plugins/baseten '@livekit/agents-plugin-bey': specifier: workspace:* version: link:../plugins/bey @@ -336,17 +339,54 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 + plugins/baseten: + dependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/agents-plugin-openai': + specifier: ^1.0.0 + version: 1.0.22(@livekit/agents@agents)(@livekit/rtc-node@0.13.13)(zod@3.25.76) + '@livekit/agents-plugins-test': + specifier: workspace:* + version: link:../test + '@livekit/rtc-node': + specifier: ^0.13.12 + version: 0.13.13 + dotenv: + specifier: ^17.2.3 + version: 17.2.3 + openai: + specifier: ^4.0.0 + version: 4.104.0(ws@8.18.3)(zod@3.25.76) + ws: + specifier: ^8.14.2 + version: 8.18.3 + devDependencies: + '@types/node': + specifier: ^22.18.11 + version: 22.19.1 + '@types/ws': + specifier: ^8.5.8 + version: 8.5.10 + tsx: + specifier: ^4.7.0 + version: 4.20.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + plugins/bey: dependencies: livekit-server-sdk: @@ -361,13 +401,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) pino: specifier: ^8.19.0 version: 8.21.0 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -392,13 +432,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -423,13 +463,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -454,13 +494,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -494,10 +534,10 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -516,16 +556,16 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.9.3) typescript: specifier: ^5.0.0 - version: 5.4.5 + version: 5.9.3 plugins/livekit: dependencies: @@ -544,13 +584,13 @@ importers: version: link:../../agents '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) onnxruntime-common: specifier: 1.21.0 version: 1.21.0 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -575,13 +615,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -612,13 +652,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -643,13 +683,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -677,13 +717,13 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -705,7 +745,7 @@ importers: version: 0.13.13 '@microsoft/api-extractor': specifier: ^7.35.0 - version: 7.43.7(@types/node@22.15.30) + version: 7.43.7(@types/node@22.19.1) '@types/ws': specifier: ^8.5.10 version: 8.5.10 @@ -714,7 +754,7 @@ importers: version: 1.21.0 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) typescript: specifier: ^5.0.0 version: 5.4.5 @@ -839,9 +879,6 @@ packages: resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} - '@bufbuild/protobuf@1.10.0': - resolution: {integrity: sha512-QDdVFLoN93Zjg36NoQPZfsVH9tZew7wKDKyV5qRdj8ntT4wQCOradQjRaTdwMhWUYsgKsvCINKKm87FdEk96Ag==} - '@bufbuild/protobuf@1.10.1': resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} @@ -1607,6 +1644,12 @@ packages: '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} + '@livekit/agents-plugin-openai@1.0.22': + resolution: {integrity: sha512-8PSM/6SRF4gcWzfbyqwlGar1vMmDjPtz6nr0cblyoW4TmCSqe+EhKuXq7aRMahhDQdgVOijszKA4TBeYzh2gnw==} + peerDependencies: + '@livekit/agents': 1.0.22 + '@livekit/rtc-node': ^0.13.12 + '@livekit/changesets-changelog-github@0.0.4': resolution: {integrity: sha512-MXaiLYwgkYciZb8G2wkVtZ1pJJzZmVx5cM30Q+ClslrIYyAqQhRbPmZDM79/5CGxb1MTemR/tfOM25tgJgAK0g==} @@ -2197,12 +2240,21 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + '@types/node@22.15.30': resolution: {integrity: sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==} + '@types/node@22.19.1': + resolution: {integrity: sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==} + '@types/pidusage@2.0.5': resolution: {integrity: sha512-MIiyZI4/MK9UGUXWt0jJcCZhVw7YdhBuTOuqP/BjuLDLZ2PmmViMIQgZiWxtaMicQfAz/kMrZ5T7PKxFSkTeUA==} @@ -2364,6 +2416,10 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -2768,6 +2824,10 @@ packages: resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} engines: {node: '>=12'} + dotenv@17.2.3: + resolution: {integrity: sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==} + engines: {node: '>=12'} + dotenv@8.6.0: resolution: {integrity: sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==} engines: {node: '>=10'} @@ -2830,10 +2890,6 @@ packages: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} - es-set-tostringtag@2.0.3: - resolution: {integrity: sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==} - engines: {node: '>= 0.4'} - es-set-tostringtag@2.1.0: resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} engines: {node: '>= 0.4'} @@ -3166,10 +3222,17 @@ packages: resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} engines: {node: '>=14'} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + fs-extra@7.0.1: resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==} engines: {node: '>=6 <7 || >=8'} @@ -3362,6 +3425,9 @@ packages: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -3846,6 +3912,11 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -3918,6 +3989,18 @@ packages: onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: resolution: {integrity: sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==} + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + openai@6.8.1: resolution: {integrity: sha512-ACifslrVgf+maMz9vqwMP4+v9qvx5Yzssydizks8n+YUJ6YwUoxj51sKRQ8HYMfR6wgKLSIlaI108ZwCk+8yig==} hasBin: true @@ -4697,12 +4780,20 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + ufo@1.5.3: resolution: {integrity: sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw==} unbox-primitive@1.0.2: resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -4826,6 +4917,10 @@ packages: resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} engines: {node: '>=10.13.0'} + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -5048,8 +5143,6 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@bufbuild/protobuf@1.10.0': {} - '@bufbuild/protobuf@1.10.1': {} '@changesets/apply-release-plan@7.0.12': @@ -5652,6 +5745,18 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.4.15 + '@livekit/agents-plugin-openai@1.0.22(@livekit/agents@agents)(@livekit/rtc-node@0.13.13)(zod@3.25.76)': + dependencies: + '@livekit/agents': link:agents + '@livekit/mutex': 1.1.1 + '@livekit/rtc-node': 0.13.13 + openai: 6.8.1(ws@8.18.3)(zod@3.25.76) + ws: 8.18.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + - zod + '@livekit/changesets-changelog-github@0.0.4': dependencies: '@changesets/get-github-info': 0.5.2 @@ -5709,7 +5814,7 @@ snapshots: '@livekit/rtc-node@0.13.13': dependencies: - '@bufbuild/protobuf': 1.10.0 + '@bufbuild/protobuf': 1.10.1 '@livekit/mutex': 1.1.1 '@livekit/typed-emitter': 3.0.0 pino: 9.6.0 @@ -5747,6 +5852,14 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@microsoft/api-extractor-model@7.28.17(@types/node@22.19.1)': + dependencies: + '@microsoft/tsdoc': 0.14.2 + '@microsoft/tsdoc-config': 0.16.2 + '@rushstack/node-core-library': 4.3.0(@types/node@22.19.1) + transitivePeerDependencies: + - '@types/node' + '@microsoft/api-extractor@7.43.7(@types/node@22.15.30)': dependencies: '@microsoft/api-extractor-model': 7.28.17(@types/node@22.15.30) @@ -5765,6 +5878,24 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@microsoft/api-extractor@7.43.7(@types/node@22.19.1)': + dependencies: + '@microsoft/api-extractor-model': 7.28.17(@types/node@22.19.1) + '@microsoft/tsdoc': 0.14.2 + '@microsoft/tsdoc-config': 0.16.2 + '@rushstack/node-core-library': 4.3.0(@types/node@22.19.1) + '@rushstack/rig-package': 0.5.2 + '@rushstack/terminal': 0.11.0(@types/node@22.19.1) + '@rushstack/ts-command-line': 4.21.0(@types/node@22.19.1) + lodash: 4.17.21 + minimatch: 3.0.8 + resolve: 1.22.8 + semver: 7.5.4 + source-map: 0.6.1 + typescript: 5.4.2 + transitivePeerDependencies: + - '@types/node' + '@microsoft/tsdoc-config@0.16.2': dependencies: '@microsoft/tsdoc': 0.14.2 @@ -6179,6 +6310,17 @@ snapshots: optionalDependencies: '@types/node': 22.15.30 + '@rushstack/node-core-library@4.3.0(@types/node@22.19.1)': + dependencies: + fs-extra: 7.0.1 + import-lazy: 4.0.0 + jju: 1.4.0 + resolve: 1.22.8 + semver: 7.5.4 + z-schema: 5.0.5 + optionalDependencies: + '@types/node': 22.19.1 + '@rushstack/operation-graph@0.2.19(@types/node@22.15.30)': dependencies: '@rushstack/node-core-library': 4.3.0(@types/node@22.15.30) @@ -6198,6 +6340,13 @@ snapshots: optionalDependencies: '@types/node': 22.15.30 + '@rushstack/terminal@0.11.0(@types/node@22.19.1)': + dependencies: + '@rushstack/node-core-library': 4.3.0(@types/node@22.19.1) + supports-color: 8.1.1 + optionalDependencies: + '@types/node': 22.19.1 + '@rushstack/ts-command-line@4.21.0(@types/node@22.15.30)': dependencies: '@rushstack/terminal': 0.11.0(@types/node@22.15.30) @@ -6207,6 +6356,15 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@rushstack/ts-command-line@4.21.0(@types/node@22.19.1)': + dependencies: + '@rushstack/terminal': 0.11.0(@types/node@22.19.1) + '@types/argparse': 1.0.38 + argparse: 1.0.10 + string-argv: 0.3.2 + transitivePeerDependencies: + - '@types/node' + '@sinclair/typebox@0.27.8': {} '@trivago/prettier-plugin-sort-imports@4.3.0(prettier@3.2.5)': @@ -6241,12 +6399,25 @@ snapshots: '@types/json5@0.0.29': {} + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 22.19.1 + form-data: 4.0.5 + '@types/node@12.20.55': {} + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + '@types/node@22.15.30': dependencies: undici-types: 6.21.0 + '@types/node@22.19.1': + dependencies: + undici-types: 6.21.0 + '@types/pidusage@2.0.5': {} '@types/semver@7.5.8': {} @@ -6257,7 +6428,7 @@ snapshots: '@types/ws@8.5.10': dependencies: - '@types/node': 22.15.30 + '@types/node': 22.19.1 '@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)': dependencies: @@ -6456,6 +6627,10 @@ snapshots: agent-base@7.1.4: {} + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -6856,6 +7031,8 @@ snapshots: dotenv@16.0.3: {} + dotenv@17.2.3: {} + dotenv@8.6.0: {} dunder-proto@1.0.1: @@ -6900,7 +7077,7 @@ snapshots: es-define-property: 1.0.0 es-errors: 1.3.0 es-object-atoms: 1.0.0 - es-set-tostringtag: 2.0.3 + es-set-tostringtag: 2.1.0 es-to-primitive: 1.2.1 function.prototype.name: 1.1.6 get-intrinsic: 1.2.4 @@ -6951,7 +7128,7 @@ snapshots: define-properties: 1.2.1 es-abstract: 1.23.3 es-errors: 1.3.0 - es-set-tostringtag: 2.0.3 + es-set-tostringtag: 2.1.0 function-bind: 1.1.2 get-intrinsic: 1.2.4 globalthis: 1.0.4 @@ -6972,12 +7149,6 @@ snapshots: dependencies: es-errors: 1.3.0 - es-set-tostringtag@2.0.3: - dependencies: - get-intrinsic: 1.2.4 - has-tostringtag: 1.0.2 - hasown: 2.0.2 - es-set-tostringtag@2.1.0: dependencies: es-errors: 1.3.0 @@ -7457,6 +7628,8 @@ snapshots: cross-spawn: 7.0.3 signal-exit: 4.1.0 + form-data-encoder@1.7.2: {} + form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -7465,6 +7638,11 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + fs-extra@7.0.1: dependencies: graceful-fs: 4.2.11 @@ -7695,6 +7873,10 @@ snapshots: human-signals@5.0.0: {} + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -8142,6 +8324,8 @@ snapshots: natural-compare@1.4.0: {} + node-domexception@1.0.0: {} + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 @@ -8223,6 +8407,21 @@ snapshots: platform: 1.3.6 protobufjs: 7.4.0 + openai@4.104.0(ws@8.18.3)(zod@3.25.76): + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + ws: 8.18.3 + zod: 3.25.76 + transitivePeerDependencies: + - encoding + openai@6.8.1(ws@8.18.3)(zod@3.25.76): optionalDependencies: ws: 8.18.3 @@ -8460,7 +8659,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 22.15.30 + '@types/node': 22.19.1 long: 5.2.3 pump@3.0.0: @@ -8998,6 +9197,62 @@ snapshots: - tsx - yaml + tsup@8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5): + dependencies: + bundle-require: 5.1.0(esbuild@0.25.2) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.0 + esbuild: 0.25.2 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(postcss@8.4.38)(tsx@4.20.4) + resolve-from: 5.0.0 + rollup: 4.40.0 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.12 + tree-kill: 1.2.2 + optionalDependencies: + '@microsoft/api-extractor': 7.43.7(@types/node@22.19.1) + postcss: 8.4.38 + typescript: 5.4.5 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + + tsup@8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.9.3): + dependencies: + bundle-require: 5.1.0(esbuild@0.25.2) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.0 + esbuild: 0.25.2 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(postcss@8.4.38)(tsx@4.20.4) + resolve-from: 5.0.0 + rollup: 4.40.0 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.12 + tree-kill: 1.2.2 + optionalDependencies: + '@microsoft/api-extractor': 7.43.7(@types/node@22.19.1) + postcss: 8.4.38 + typescript: 5.9.3 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + tsx@4.19.2: dependencies: esbuild: 0.23.1 @@ -9095,6 +9350,8 @@ snapshots: typescript@5.4.5: {} + typescript@5.9.3: {} + ufo@1.5.3: {} unbox-primitive@1.0.2: @@ -9104,6 +9361,8 @@ snapshots: has-symbols: 1.0.3 which-boxed-primitive: 1.0.2 + undici-types@5.26.5: {} + undici-types@6.21.0: {} universalify@0.1.2: {} @@ -9240,6 +9499,8 @@ snapshots: glob-to-regexp: 0.4.1 graceful-fs: 4.2.11 + web-streams-polyfill@4.0.0-beta.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} diff --git a/turbo.json b/turbo.json index 6c420a36e..aa13412ea 100644 --- a/turbo.json +++ b/turbo.json @@ -9,6 +9,9 @@ "BEY_API_KEY", "BEY_API_URL", "BEY_AVATAR_ID", + "BASETEN_API_KEY", + "BASETEN_MODEL_ENDPOINT", + "BASETEN_STT_MODEL_ID", "CARTESIA_API_KEY", "CAL_API_KEY", "CEREBRAS_API_KEY",