Add baseten plugin

.changeset/some-lights-lead.md

@@ -0,0 +1,8 @@
+---
+'@livekit/agents-plugin-baseten': patch
+---
+
+Add @livekit/agents-plugin-baseten with support for Baseten-hosted models:
+- STT: WebSocket-based streaming speech-to-text with VAD and partial transcripts
+- TTS: Streaming text-to-speech using Baseten's Orpheus model (24kHz)
+- LLM: OpenAI-compatible API integration for Baseten inference

examples/package.json

@@ -12,7 +12,8 @@
     "clean:build": "pnpm clean && pnpm build",
     "lint": "eslint -f unix \"src/**/*.ts\"",
     "test": "vitest run",
-    "test:watch": "vitest"
+    "test:watch": "vitest",
+    "baseten:cloud": "tsx --env-file=.env src/baseten_restaurant_agent.ts dev --log-level=debug"
   },
   "devDependencies": {
     "@types/node": "^22.5.5",
@@ -24,6 +25,7 @@
   "dependencies": {
     "@livekit/agents": "workspace:*",
     "@livekit/agents-plugin-anam": "workspace:*",
+    "@livekit/agents-plugin-baseten": "workspace:*",
     "@livekit/agents-plugin-bey": "workspace:*",
     "@livekit/agents-plugin-cartesia": "workspace:*",
     "@livekit/agents-plugin-deepgram": "workspace:*",

plugins/baseten/README.md

@@ -0,0 +1,92 @@
+<!--
+SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# LiveKit Agents Baseten Plugin
+
+Node.js/TypeScript plugin for LiveKit Agents with Baseten-hosted models (LLM, STT, TTS).
+
+## Installation
+
+```bash
+cd packages/livekit-plugin-baseten
+pnpm install
+pnpm build
+```
+
+## Configuration
+
+Create `.env` file:
+
+```bash
+BASETEN_API_KEY=your_api_key_here
+BASETEN_MODEL_ID=your_llm_model_id
+BASETEN_TTS_MODEL_ID=your_tts_model_id
+BASETEN_STT_MODEL_ID=your_stt_model_id
+```
+
+## Usage
+
+### LLM
+
+```typescript
+import { LLM } from 'livekit-plugin-baseten'
+
+const llm = new LLM({
+    model: 'openai/gpt-4o-mini',
+    apiKey: process.env.BASETEN_API_KEY
+})
+```
+
+### STT
+
+```typescript
+import { STT } from 'livekit-plugin-baseten'
+
+const stt = new STT({
+    apiKey: process.env.BASETEN_API_KEY,
+    modelId: process.env.BASETEN_STT_MODEL_ID
+})
+
+const stream = stt.stream()
+for await (const event of stream) {
+    // Handle speech events
+}
+```
+
+### TTS
+
+```typescript
+import { TTS } from 'livekit-plugin-baseten'
+
+const tts = new TTS({
+    apiKey: process.env.BASETEN_API_KEY,
+    modelEndpoint: 'your-model-endpoint-url'
+})
+
+const stream = tts.synthesize('Hello world')
+for await (const frame of stream) {
+    // Process audio frames
+}
+```
+
+## Testing
+
+```bash
+pnpm test:llm-cli   # Interactive LLM chat
+pnpm test:tts-cli   # TTS synthesis
+pnpm test:stt-cli   # STT with microphone
+```
+
+See [TESTING.md](./test/TESTING.md) for details.
+
+## Development
+
+```bash
+pnpm build      # Build
+pnpm dev        # Watch mode
+pnpm typecheck  # Type checking
+pnpm lint       # Linting
+```

plugins/baseten/package.json

@@ -0,0 +1,53 @@
+{
+    "name": "@livekit/agents-plugin-baseten",
+    "version": "1.0.27",
+    "description": "Baseten plugin for LiveKit Node Agents",
+    "private": true,
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts",
+    "type": "module",
+    "author": "LiveKit",
+    "repository": "git@github.com:livekit/agents-js.git",
+    "license": "Apache-2.0",
+    "files": [
+      "dist",
+      "src",
+      "README.md"
+    ],
+    "scripts": {
+        "build": "tsc",
+        "dev": "tsc --watch",
+        "clean": "rm -rf dist",
+        "typecheck": "tsc --noEmit",
+        "lint": "eslint \"src/**/*.ts\"",
+        "lint:fix": "eslint --fix \"src/**/*.ts\""
+    },
+    "keywords": [
+        "livekit",
+        "agents",
+        "baseten",
+        "llm",
+        "stt",
+        "tts",
+        "voice-ai"
+    ],
+    "dependencies": {
+        "@livekit/agents": "workspace:*",
+        "@livekit/agents-plugin-openai": "^1.0.0",
+        "@livekit/rtc-node": "^0.13.12",
+        "@livekit/agents-plugins-test": "workspace:*",
+        "dotenv": "^17.2.3",
+        "openai": "^4.0.0",
+        "ws": "^8.14.2"
+    },
+    "devDependencies": {
+        "@types/node": "^22.18.11",
+        "@types/ws": "^8.5.8",
+        "tsx": "^4.7.0",
+        "typescript": "^5.9.3"
+    },
+    "peerDependencies": {
+        "@livekit/agents": "workspace:*",
+        "@livekit/rtc-node": "^0.13.12"
+    }
+}

plugins/baseten/src/index.ts

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * LiveKit Agents Baseten Plugin
+ *
+ * Integrates Baseten-hosted models with LiveKit Agents for LLM, STT, and TTS services.
+ */
+import { Plugin } from '@livekit/agents';
+
+class BasetenPlugin extends Plugin {
+  constructor() {
+    super({
+      title: 'baseten',
+      version: '1.0.0',
+      package: 'livekit-plugin-baseten',
+    });
+  }
+}
+
+Plugin.registerPlugin(new BasetenPlugin());
+
+// Export classes following LiveKit plugin pattern
+export { LLM } from './llm.js';
+export { STT } from './stt.js';
+export { TTS } from './tts.js';
+
+// Export all types
+export type { BasetenLLMOptions, BasetenSttOptions, BasetenTTSOptions } from './types.js';

plugins/baseten/src/llm.test.ts

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents-plugins-test';
+import { describe } from 'vitest';
+import { LLM } from './llm.js';
+
+describe('Baseten', async () => {
+  await llm(
+    new LLM({
+      model: 'openai/gpt-4o-mini',
+      temperature: 0,
+    }),
+    false,
+  );
+});

plugins/baseten/src/llm.ts

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Baseten LLM plugin for LiveKit Agents
+ * Configures the OpenAI plugin to work with Baseten's OpenAI-compatible API
+ */
+import { LLM as OpenAILLM } from '@livekit/agents-plugin-openai';
+import type { BasetenLLMOptions } from './types.js';
+
+export class LLM extends OpenAILLM {
+  constructor(opts: BasetenLLMOptions) {
+    const apiKey = opts.apiKey ?? process.env.BASETEN_API_KEY;
+    if (!apiKey) {
+      throw new Error(
+        'Baseten API key is required. Set BASETEN_API_KEY environment variable or pass apiKey in options.',
+      );
+    }
+
+    if (!opts.model) {
+      throw new Error(
+        'Model is required. Please specify a model name (e.g., "openai/gpt-4o-mini").',
+      );
+    }
+
+    const model = opts.model;
+
+    // Configure the OpenAI plugin with Baseten's endpoint
+    super({
+      model,
+      apiKey,
+      baseURL: 'https://inference.baseten.co/v1',
+      temperature: opts.temperature,
+      user: opts.user,
+      maxCompletionTokens: opts.maxTokens,
+      toolChoice: opts.toolChoice,
+      parallelToolCalls: opts.parallelToolCalls,
+    });
+  }
+
+  label(): string {
+    return 'baseten.LLM';
+  }
+}

plugins/baseten/src/stt.test.ts

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { VAD } from '@livekit/agents-plugin-silero';
+import { stt } from '@livekit/agents-plugins-test';
+import { describe } from 'vitest';
+import { STT } from './stt.js';
+
+describe('Baseten', async () => {
+  await stt(new STT(), await VAD.load(), { streaming: true });
+});