diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/.gitignore b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/.gitignore new file mode 100644 index 0000000000..aa6b9396e1 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/.gitignore @@ -0,0 +1,28 @@ +# Dependencies +node_modules/ +.env +.env.local + +# Build outputs +build/ +dist/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log +logs/ + +# Temporary +*.tmp +*.bak +*~ +session_control.conf diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Dockerfile b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Dockerfile new file mode 100644 index 0000000000..7ff1124c77 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Dockerfile @@ -0,0 +1,49 @@ +# Build stage +FROM golang:1.22-alpine AS builder + +WORKDIR /app + +# Copy go module files +COPY tenapp/go.mod tenapp/go.sum ./ + +# Download dependencies +RUN go mod download + +# Copy source code +COPY tenapp/ ./ + +# Build the application +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o voice-assistant main.go + +# Final stage +FROM python:3.11-slim + +WORKDIR /app + +# Install Python dependencies +COPY ten_packages/extension/main_python/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY ten_packages/extension/openai_llm2_python/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy frontend build +COPY --from=builder /app/tenapp ./tenapp + +# Copy API server +COPY ten_apps/api_server ./ten_apps/api_server + +# Copy frontend +COPY ten_apps/voice-assistant ./ten_apps/voice-assistant + +# Copy built application +COPY --from=builder /app/voice-assistant . + +# Expose ports +EXPOSE 8080 3000 + +# Set environment variables +ENV PYTHONUNBUFFERED=1 + +# Run the application +CMD ["sh", "-c", "python ten_apps/api_server/main.py & npm run dev --prefix ten_apps/voice-assistant & ./voice-assistant"] diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/README.md b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/README.md new file mode 100644 index 0000000000..14bedcc9bc --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/README.md @@ -0,0 +1,225 @@ +# Language Learning Companion (with PowerMem) + +A voice AI language tutor enhanced with [PowerMem](https://github.com/oceanbase/powermem/) for persistent memory of vocabulary, learning progress, and personalized lessons. + +## Overview + +This is a **memory-enabled voice AI use case** for the Oceanbase Developer Challenge 2026. The Language Learning Companion uses TEN Framework with PowerMem to provide personalized language education that: + +- **Remembers vocabulary words** learned across sessions +- **Tracks grammar concepts** covered in each lesson +- **Provides personalized practice** based on the learner's level +- **Reviews previously learned material** to reinforce memory +- **Adapts to learning style** through long-term context accumulation + +## Use Case: Personalized Language Tutor + +### Memory Design + +The tutor uses PowerMem to store and retrieve different types of learning data: + +| Memory Type | Content | Retrieval Strategy | +|-------------|---------|-------------------| +| **Vocabulary** | Words, phrases, translations | Semantic search by topic/language | +| **Grammar** | Rules, examples, explanations | Context-aware retrieval by topic | +| **Progress** | Lesson history, quiz scores | Timeline-based review | +| **Preferences** | Learning style, native language | User profile retrieval | + +### How Memory Works + +1. **Learning New Words**: When a user learns a new vocabulary word, it's stored with metadata (language, level, topic) +2. **Grammar Sessions**: Each grammar lesson is summarized and stored for future reference +3. **Personalized Review**: Before each session, relevant memories are retrieved to provide context +4. **Progress Tracking**: Quiz results and practice scores are saved to track improvement + +### Example Conversation Flow + +``` +User: Teach me the Spanish word for "apple" +Tutor: "Apple" in Spanish is "manzana". Let me remember this for you. + Now, can you use "manzana" in a sentence? + +User: I eat an manzana +Tutor: Good try! The correct sentence is "Como una manzana" or "Me como una manzana". + Remember: "manzana" is feminine (la manzana). +``` + +## PowerMem Configuration + +Set the environment variables in `.env` file: + +```bash +# Database (PowerMem/Oceanbase) +DATABASE_PROVIDER=oceanbase +OCEANBASE_HOST=127.0.0.1 +OCEANBASE_PORT=2881 +OCEANBASE_USER=root +OCEANBASE_PASSWORD=password +OCEANBASE_DATABASE=oceanbase +OCEANBASE_COLLECTION=language_tutor_memories + +# LLM Provider (for PowerMem) +LLM_PROVIDER=qwen +LLM_API_KEY=your_qwen_api_key +LLM_MODEL=qwen-plus + +# Embedding Provider (for PowerMem) +EMBEDDING_PROVIDER=qwen +EMBEDDING_API_KEY=your_qwen_api_key +EMBEDDING_MODEL=text-embedding-v4 +EMBEDDING_DIMS=1536 + +# Voice Services +AGORA_APP_ID=your_agora_app_id +DEEPGRAM_API_KEY=your_deepgram_api_key +ELEVENLABS_TTS_KEY=your_elevenlabs_key + +# LLM for Chat +OPENAI_API_KEY=your_openai_api_key +OPENAI_MODEL=gpt-4 +``` + +## Prerequisites + +### Required Services + +1. **Oceanbase Database** (for PowerMem) + ```bash + docker run -d \ + --name seekdb \ + -p 2881:2881 \ + -p 2886:2886 \ + -v ./data:/var/lib/oceanbase \ + -e SEEKDB_DATABASE=powermem \ + -e ROOT_PASSWORD=password \ + oceanbase/seekdb:latest + ``` + +2. **Agora Account** - Get credentials from [Agora Console](https://console.agora.io/) + - `AGORA_APP_ID` (required) + +3. **API Keys**: + - Deepgram for ASR (`DEEPGRAM_API_KEY`) + - ElevenLabs for TTS (`ELEVENLABS_TTS_KEY`) + - OpenAI for LLM (`OPENAI_API_KEY`) + - Qwen for PowerMem embeddings (`LLM_API_KEY`, `EMBEDDING_API_KEY`) + +## Quick Start + +1. **Install dependencies:** + ```bash + task install + ``` + +2. **Run the Language Tutor:** + ```bash + task run + ``` + +3. **Access the application:** + - Frontend: http://localhost:3000 + - API Server: http://localhost:8080 + - TMAN Designer: http://localhost:49483 + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Language Learning Companion │ +├─────────────────────────────────────────────────────────────┤ +│ ┌───────────┐ ┌───────────┐ ┌───────────────────┐ │ +│ │ Agora │◄──►│ STT │◄──►│ Main Controller │ │ +│ │ RTC │ │ (Deepgram)│ │ (with PowerMem) │ │ +│ └───────────┘ └───────────┘ └───────────────────┘ │ +│ │ │ +│ ┌───────────┐ ┌───────────┐ ┌───────▼───────┐ │ +│ │ TTS │◄──►│ LLM │◄──►│ PowerMem │ │ +│ │(Elevenlabs)│ │ (GPT-4) │ │ Memory Store │ │ +│ └───────────┘ └───────────┘ └───────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Memory Flow + +1. **Input**: User speaks via Agora RTC +2. **STT**: Audio converted to text via Deepgram +3. **Processing**: Main controller extracts learning content +4. **Memory Retrieval**: PowerMem searches for relevant memories +5. **LLM Generation**: GPT-4 generates contextual response +6. **Memory Storage**: New learning content saved to PowerMem +7. **Output**: Response spoken via ElevenLabs TTS + +## Customization + +### Adjust Memory Settings + +Modify `tenapp/property.json`: + +```json +"memory_save_interval_turns": 3, // Save every N turns +"memory_idle_timeout_seconds": 60.0, // Session timeout +"max_memory_length": 20 // Max conversation history +``` + +### Customize Tutor Persona + +Update the `custom_system_prompt` in `property.json`: + +```json +"custom_system_prompt": "You are a Spanish tutor..." +``` + +## Project Structure + +``` +voice-assistant-with-PowerMem-LanguageTutor/ +├── Dockerfile +├── README.md +├── Taskfile.yml +├── Taskfile.docker.yml +└── tenapp/ + ├── .tenignore + ├── go.mod + ├── go.sum + ├── main.go + ├── manifest.json + ├── manifest-lock.json + ├── property.json + └── scripts/ + ├── install_python_deps.sh + └── start.sh +``` + +## Docker Deployment + +### Build Image + +```bash +docker build -f Dockerfile -t language-tutor-app . +``` + +### Run Container + +```bash +docker run --rm -it --env-file .env \ + -p 8080:8080 \ + -p 3000:3000 \ + language-tutor-app +``` + +## Troubleshooting + +- **Memory not persisting**: Check Oceanbase connection and `OCEANBASE_*` variables +- **Audio issues**: Verify `AGORA_APP_ID`, `DEEPGRAM_API_KEY`, `ELEVENLABS_TTS_KEY` +- **LLM errors**: Confirm `OPENAI_API_KEY` is valid +- **PowerMem errors**: Ensure `LLM_API_KEY` and `EMBEDDING_API_KEY` are set + +## License + +MIT License - Part of TEN Framework + +## References + +- [TEN Framework](https://github.com/TEN-framework/ten-framework) +- [PowerMem](https://github.com/oceanbase/powermem/) +- [Oceanbase Developer Challenge 2026](https://github.com/TEN-framework/ten-framework/issues/2013) diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Taskfile.docker.yml b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Taskfile.docker.yml new file mode 100644 index 0000000000..2590400904 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Taskfile.docker.yml @@ -0,0 +1,32 @@ +version: '3' + +vars: + APP_NAME: voice-assistant-with-PowerMem-LanguageTutor + TEN_AGENT_DIR: '{{.USER_WORKING_DIR}}/../../..' + FRONTEND_DIR: '{{.TEN_AGENT_DIR}}/ten_apps/voice-assistant' + +tasks: + install: + desc: Install dependencies for Docker build + cmds: + - | + echo "Installing Python dependencies..." + cd {{.TEN_AGENT_DIR}}/ten_packages/extension/main_python + pip install -r requirements.txt -q + + echo "Installing extension dependencies..." + cd {{.TEN_AGENT_DIR}}/ten_packages/extension/openai_llm2_python + pip install -r requirements.txt -q + + echo "Installation completed." + dir: '{{.PWD}}' + + run: + desc: Run with Docker + cmds: + - | + docker run --rm -it --env-file .env \ + -p 8080:8080 \ + -p 3000:3000 \ + voice-assistant-with-PowerMem-LanguageTutor-app + dir: '{{.PWD}}' diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Taskfile.yml b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Taskfile.yml new file mode 100644 index 0000000000..be64b6d03a --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/Taskfile.yml @@ -0,0 +1,62 @@ +version: '3' + +vars: + APP_NAME: voice-assistant-with-PowerMem-LanguageTutor + TEN_AGENT_DIR: '{{.USER_WORKING_DIR}}/../../..' + FRONTEND_DIR: '{{.TEN_AGENT_DIR}}/ten_apps/voice-assistant' + +tasks: + install: + desc: Install dependencies for the voice assistant with PowerMem Language Tutor + cmds: + - | + echo "Installing Python dependencies..." + cd {{.TEN_AGENT_DIR}}/ten_packages/extension/main_python + pip install -r requirements.txt + echo "Python dependencies installed." + + echo "Installing extension dependencies..." + cd {{.TEN_AGENT_DIR}}/ten_packages/extension/openai_llm2_python + pip install -r requirements.txt + echo "Extension dependencies installed." + + echo "Installing frontend dependencies..." + cd {{.FRONTEND_DIR}} + npm install + echo "Frontend dependencies installed." + + echo "Installation completed." + dir: '{{.PWD}}' + + run: + desc: Run the voice assistant with PowerMem Language Tutor + cmds: + - | + echo "Starting frontend..." + cd {{.FRONTEND_DIR}} + export REACT_APP_DEFAULT_AGENT=voice-assistant-with-PowerMem-LanguageTutor + npm run dev & + FRONTEND_PID=$! + echo "Frontend started with PID: $FRONTEND_PID" + + echo "Starting API server..." + cd {{.TEN_AGENT_DIR}}/ten_apps/api_server + python main.py & + API_PID=$! + echo "API server started with PID: $API_PID" + + echo "Starting TMAN Designer..." + cd {{.TEN_AGENT_DIR}} + python -m ten_app_designer & + DESIGNER_PID=$! + echo "TMAN Designer started with PID: $DESIGNER_PID" + + echo "Starting Language Tutor Agent..." + cd tenapp + go run main.go + AGENT_EXIT=$? + + echo "Stopping services..." + kill $FRONTEND_PID $API_PID $DESIGNER_PID 2>/dev/null || true + echo "All services stopped." + dir: '{{.PWD}}' diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/.tenignore b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/.tenignore new file mode 100644 index 0000000000..9d04eb81a3 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/.tenignore @@ -0,0 +1,25 @@ +# .tenignore +# Files to ignore during deployment + +# IDE files +.idea/ +.vscode/ +*.swp +*.swo + +# Build artifacts +build/ +dist/ + +# Logs +*.log +logs/ + +# OS files +.DS_Store +Thumbs.db + +# Temporary files +*.tmp +*.bak +*~ diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/go.mod b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/go.mod new file mode 100644 index 0000000000..bf74072de7 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/go.mod @@ -0,0 +1,3 @@ +module voice-assistant-with-PowerMem-LanguageTutor + +go 1.22 diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/go.sum b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/go.sum new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/main.go b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/main.go new file mode 100644 index 0000000000..c3bb4ce8c1 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/main.go @@ -0,0 +1,71 @@ +/** + * Voice Assistant with PowerMem - Language Tutor + * A language learning companion that remembers vocabulary, learning progress, + * and provides personalized lessons using PowerMem for persistent memory. + * + * Created for Oceanbase Developer Challenge 2026 + */ +package main + +import ( + "flag" + "log" + "os" + + ten "ten_framework/ten_runtime" +) + +type appConfig struct { + PropertyFilePath string +} + +type defaultApp struct { + ten.DefaultApp + + cfg *appConfig +} + +func (p *defaultApp) OnConfigure( + tenEnv ten.TenEnv, +) { + // Using the default property.json if not specified. + if len(p.cfg.PropertyFilePath) > 0 { + if b, err := os.ReadFile(p.cfg.PropertyFilePath); err != nil { + log.Fatalf("Failed to read property file %s, err %v\n", p.cfg.PropertyFilePath, err) + } else { + tenEnv.InitPropertyFromJSONBytes(b) + } + } + + tenEnv.OnConfigureDone() +} + +func startAppBlocking(cfg *appConfig) { + appInstance, err := ten.NewApp(&defaultApp{ + cfg: cfg, + }) + if err != nil { + log.Fatalf("Failed to create the app, %v\n", err) + } + + appInstance.Run(true) + appInstance.Wait() + + ten.EnsureCleanupWhenProcessExit() +} + +func setDefaultLog() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) +} + +func main() { + // Set the default log format globally, users can use `log.Println()` directly. + setDefaultLog() + + cfg := &appConfig{} + + flag.StringVar(&cfg.PropertyFilePath, "property", "", "The absolute path of property.json") + flag.Parse() + + startAppBlocking(cfg) +} diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/manifest-lock.json b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/manifest-lock.json new file mode 100644 index 0000000000..6100ccdae2 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/manifest-lock.json @@ -0,0 +1,402 @@ +{ + "manifest_lock_version": "1", + "dependencies": { + "agora_rtc": { + "addon_name": "agora_rtc", + "download_url": null, + "name": "agora_rtc", + "source": "remote", + "system_addon": false, + "type": "extension", + "version": "0.23.9-t1" + }, + "aliyun_asr": { + "addon_name": "aliyun_asr", + "download_url": null, + "name": "aliyun_asr", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "aliyun_asr_bigmodel_python": { + "addon_name": "aliyun_asr_bigmodel_python", + "download_url": null, + "name": "aliyun_asr_bigmodel_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "assemblyai_asr_python": { + "addon_name": "assemblyai_asr_python", + "download_url": null, + "name": "assemblyai_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "aws_asr_python": { + "addon_name": "aws_asr_python", + "download_url": null, + "name": "aws_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "azure_asr_python": { + "addon_name": "azure_asr_python", + "download_url": null, + "name": "azure_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "azure_tts_python": { + "addon_name": "azure_tts_python", + "download_url": null, + "name": "azure_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "bytedance_asr": { + "addon_name": "bytedance_asr", + "download_url": null, + "name": "bytedance_asr", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "bytedance_llm_based_asr": { + "addon_name": "bytedance_llm_based_asr", + "download_url": null, + "name": "bytedance_llm_based_asr", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "bytedance_tts_duplex": { + "addon_name": "bytedance_tts_duplex", + "download_url": null, + "name": "bytedance_tts_duplex", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "cartesia_tts": { + "addon_name": "cartesia_tts", + "download_url": null, + "name": "cartesia_tts", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "coze_llm2_python": { + "addon_name": "coze_llm2_python", + "download_url": null, + "name": "coze_llm2_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "cosy_tts_python": { + "addon_name": "cosy_tts_python", + "download_url": null, + "name": "cosy_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "deepgram_asr_python": { + "addon_name": "deepgram_asr_python", + "download_url": null, + "name": "deepgram_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "dify_llm2_python": { + "addon_name": "dify_llm2_python", + "download_url": null, + "name": "dify_llm2_python", + "source": "local", + "system_addon": false, + "type": "extension", + "extension_group": "dify", + "version": null + }, + "dubverse_tts": { + "addon_name": "dubverse_tts", + "download_url": null, + "name": "dubverse_tts", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "elevenlabs_tts2_python": { + "addon_name": "elevenlabs_tts2_python", + "download_url": null, + "name": "elevenlabs_tts2_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "fish_audio_tts_python": { + "addon_name": "fish_audio_tts_python", + "download_url": null, + "name": "fish_audio_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "gladia_asr_python": { + "addon_name": "gladia_asr_python", + "download_url": null, + "name": "gladia_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "google_asr_python": { + "addon_name": "google_asr_python", + "download_url": null, + "name": "google_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "google_tts_python": { + "addon_name": "google_tts_python", + "download_url": null, + "name": "google_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "groq_tts_python": { + "addon_name": "groq_tts_python", + "download_url": null, + "name": "groq_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "humeai_tts_python": { + "addon_name": "humeai_tts_python", + "download_url": null, + "name": "humeai_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "message_collector2": { + "addon_name": "message_collector2", + "download_url": null, + "name": "message_collector2", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "minimax_tts_websocket_python": { + "addon_name": "minimax_tts_websocket_python", + "download_url": null, + "name": "minimax_tts_websocket_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "neuphonic_tts": { + "addon_name": "neuphonic_tts", + "download_url": null, + "name": "neuphonic_tts", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "openai_asr_python": { + "addon_name": "openai_asr_python", + "download_url": null, + "name": "openai_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "openai_llm2_python": { + "addon_name": "openai_llm2_python", + "download_url": null, + "name": "openai_llm2_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "openai_tts2_python": { + "addon_name": "openai_tts2_python", + "download_url": null, + "name": "openai_tts2_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "polly_tts": { + "addon_name": "polly_tts", + "download_url": null, + "name": "polly_tts", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "rime_http_tts": { + "addon_name": "rime_http_tts", + "download_url": null, + "name": "rime_http_tts", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "rime_tts": { + "addon_name": "rime_tts", + "download_url": null, + "name": "rime_tts", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "soniox_asr_python": { + "addon_name": "soniox_asr_python", + "download_url": null, + "name": "soniox_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "speechmatics_asr_python": { + "addon_name": "speechmatics_asr_python", + "download_url": null, + "name": "speechmatics_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "stepfun_tts_python": { + "addon_name": "stepfun_tts_python", + "download_url": null, + "name": "stepfun_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "streamid_adapter": { + "addon_name": "streamid_adapter", + "download_url": null, + "name": "streamid_adapter", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "ten_ai_base": { + "addon_name": null, + "download_url": null, + "name": "ten_ai_base", + "source": "remote", + "system_addon": true, + "type": "system", + "version": "0.7" + }, + "ten_runtime_go": { + "addon_name": null, + "download_url": null, + "name": "ten_runtime_go", + "source": "remote", + "system_addon": true, + "type": "system", + "version": "0.11" + }, + "tencent_asr_python": { + "addon_name": "tencent_asr_python", + "download_url": null, + "name": "tencent_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "tencent_tts_python": { + "addon_name": "tencent_tts_python", + "download_url": null, + "name": "tencent_tts_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "weatherapi_tool_python": { + "addon_name": "weatherapi_tool_python", + "download_url": null, + "name": "weatherapi_tool_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "xfyun_asr_bigmodel_python": { + "addon_name": "xfyun_asr_bigmodel_python", + "download_url": null, + "name": "xfyun_asr_bigmodel_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "xfyun_asr_dialect_python": { + "addon_name": "xfyun_asr_dialect_python", + "download_url": null, + "name": "xfyun_asr_dialect_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + }, + "xfyun_asr_python": { + "addon_name": "xfyun_asr_python", + "download_url": null, + "name": "xfyun_asr_python", + "source": "local", + "system_addon": false, + "type": "extension", + "version": null + } + } +} diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/manifest.json b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/manifest.json new file mode 100644 index 0000000000..2b6f1fc94f --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/manifest.json @@ -0,0 +1,149 @@ +{ + "type": "app", + "name": "voice_assistant_powermem_language_tutor", + "version": "0.10.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.11" + }, + { + "type": "extension", + "name": "agora_rtc", + "version": "=0.23.9-t1" + }, + { + "type": "system", + "name": "ten_ai_base", + "version": "0.7" + }, + { + "path": "../../../ten_packages/extension/streamid_adapter" + }, + { + "path": "../../../ten_packages/extension/aliyun_asr" + }, + { + "path": "../../../ten_packages/extension/aliyun_asr_bigmodel_python" + }, + { + "path": "../../../ten_packages/extension/assemblyai_asr_python" + }, + { + "path": "../../../ten_packages/extension/aws_asr_python" + }, + { + "path": "../../../ten_packages/extension/azure_asr_python" + }, + { + "path": "../../../ten_packages/extension/bytedance_asr" + }, + { + "path": "../../../ten_packages/extension/bytedance_llm_based_asr" + }, + { + "path": "../../../ten_packages/extension/deepgram_asr_python" + }, + { + "path": "../../../ten_packages/extension/gladia_asr_python" + }, + { + "path": "../../../ten_packages/extension/google_asr_python" + }, + { + "path": "../../../ten_packages/extension/openai_asr_python" + }, + { + "path": "../../../ten_packages/extension/soniox_asr_python" + }, + { + "path": "../../../ten_packages/extension/speechmatics_asr_python" + }, + { + "path": "../../../ten_packages/extension/tencent_asr_python" + }, + { + "path": "../../../ten_packages/extension/xfyun_asr_bigmodel_python" + }, + { + "path": "../../../ten_packages/extension/xfyun_asr_dialect_python" + }, + { + "path": "../../../ten_packages/extension/xfyun_asr_python" + }, + { + "path": "../../../ten_packages/extension/coze_llm2_python" + }, + { + "path": "../../../ten_packages/extension/dify_llm2_python" + }, + { + "path": "../../../ten_packages/extension/openai_llm2_python" + }, + { + "path": "../../../ten_packages/extension/azure_tts_python" + }, + { + "path": "../../../ten_packages/extension/bytedance_tts_duplex" + }, + { + "path": "../../../ten_packages/extension/cartesia_tts" + }, + { + "path": "../../../ten_packages/extension/cosy_tts_python" + }, + { + "path": "../../../ten_packages/extension/dubverse_tts" + }, + { + "path": "../../../ten_packages/extension/elevenlabs_tts2_python" + }, + { + "path": "../../../ten_packages/extension/fish_audio_tts_python" + }, + { + "path": "../../../ten_packages/extension/google_tts_python" + }, + { + "path": "../../../ten_packages/extension/groq_tts_python" + }, + { + "path": "../../../ten_packages/extension/humeai_tts_python" + }, + { + "path": "../../../ten_packages/extension/minimax_tts_websocket_python" + }, + { + "path": "../../../ten_packages/extension/neuphonic_tts" + }, + { + "path": "../../../ten_packages/extension/openai_tts2_python" + }, + { + "path": "../../../ten_packages/extension/polly_tts" + }, + { + "path": "../../../ten_packages/extension/rime_http_tts" + }, + { + "path": "../../../ten_packages/extension/rime_tts" + }, + { + "path": "../../../ten_packages/extension/stepfun_tts_python" + }, + { + "path": "../../../ten_packages/extension/tencent_tts_python" + }, + { + "path": "../../../ten_packages/extension/message_collector2" + }, + { + "path": "../../../ten_packages/extension/weatherapi_tool_python" + } + ], + "scripts": { + "start": "scripts/start.sh", + "build": "scripts/install_python_deps.sh" + } +} diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/property.json b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/property.json new file mode 100644 index 0000000000..d7a4d1a6f0 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/property.json @@ -0,0 +1,246 @@ +{ + "ten": { + "predefined_graphs": [ + { + "name": "language_tutor", + "auto_start": true, + "graph": { + "nodes": [ + { + "type": "extension", + "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "app_certificate": "${env:AGORA_APP_CERTIFICATE|}", + "channel": "language_tutor_channel", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}", + "agora_asr_session_control_file_path": "session_control.conf" + } + }, + { + "type": "extension", + "name": "stt", + "addon": "deepgram_asr_python", + "extension_group": "stt", + "property": { + "params": { + "api_key": "${env:DEEPGRAM_API_KEY}", + "language": "en-US", + "model": "nova-3" + } + } + }, + { + "type": "extension", + "name": "llm", + "addon": "openai_llm2_python", + "extension_group": "chatgpt", + "property": { + "base_url": "https://api.openai.com/v1", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL|}", + "greeting": "Hello! I'm your Language Learning Companion. I can help you learn a new language by remembering your progress, vocabulary, and providing personalized lessons.", + "max_memory_length": 20 + } + }, + { + "type": "extension", + "name": "tts", + "addon": "elevenlabs_tts2_python", + "extension_group": "tts", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "key": "${env:ELEVENLABS_TTS_KEY}", + "model_id": "eleven_multilingual_v2", + "voice_id": "pNInz6obpgDQGcFmaJgB", + "output_format": "pcm_16000" + } + } + }, + { + "type": "extension", + "name": "main_control", + "addon": "main_python", + "extension_group": "control", + "property": { + "greeting": "Hello! I'm your Language Learning Companion. I can help you learn a new language by remembering your vocabulary, grammar lessons, and providing personalized practice. Let's start learning together!", + "agent_id": "language_tutor_agent", + "user_id": "student", + "enable_memorization": true, + "enable_user_memory": true, + "memory_save_interval_turns": 3, + "memory_idle_timeout_seconds": 60.0, + "powermem_config": { + "vector_store": { + "provider": "oceanbase", + "config": { + "collection_name": "${env:OCEANBASE_COLLECTION}", + "host": "${env:OCEANBASE_HOST}", + "port": "${env:OCEANBASE_PORT}", + "user": "${env:OCEANBASE_USER}", + "password": "${env:OCEANBASE_PASSWORD}", + "db_name": "${env:OCEANBASE_DATABASE}" + } + }, + "llm": { + "provider": "${env:LLM_PROVIDER}", + "config": { + "api_key": "${env:LLM_API_KEY}", + "model": "${env:LLM_MODEL}" + } + }, + "embedder": { + "provider": "${env:EMBEDDING_PROVIDER}", + "config": { + "api_key": "${env:EMBEDDING_API_KEY}", + "model": "${env:EMBEDDING_MODEL}", + "embedding_dims": "${env:EMBEDDING_DIMS}" + } + } + }, + "custom_system_prompt": "You are a helpful language learning companion. You help users learn languages by:\n\n1. Remembering vocabulary words they've learned (store as memory with tags like 'vocabulary', 'spanish', 'beginner')\n2. Tracking grammar concepts covered (store as 'grammar', 'topic')\n3. Providing personalized practice based on their level\n4. Reviewing previously learned material to reinforce memory\n\nWhen a user learns a new word, acknowledge it and ask them to practice using it in a sentence.\nWhen they make mistakes, gently correct them and explain the grammar rule.\nBefore each lesson, briefly review what was covered in previous sessions if available in memory.\n\nYour goal is to make language learning engaging, personalized, and effective through consistent memory of the user's progress." + } + }, + { + "type": "extension", + "name": "message_collector", + "addon": "message_collector2", + "extension_group": "transcriber", + "property": {} + }, + { + "type": "extension", + "name": "weatherapi_tool_python", + "addon": "weatherapi_tool_python", + "extension_group": "default", + "property": { + "api_key": "${env:WEATHERAPI_API_KEY|}" + } + }, + { + "type": "extension", + "name": "streamid_adapter", + "addon": "streamid_adapter", + "property": {} + } + ], + "connections": [ + { + "extension": "main_control", + "cmd": [ + { + "names": ["on_user_joined", "on_user_left"], + "source": [ + { + "extension": "agora_rtc" + } + ] + }, + { + "names": ["tool_register"], + "source": [ + { + "extension": "weatherapi_tool_python" + } + ] + } + ], + "data": [ + { + "name": "asr_result", + "source": [ + { + "extension": "stt" + } + ] + } + ] + }, + { + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "streamid_adapter" + } + ] + }, + { + "name": "pcm_frame", + "source": [ + { + "extension": "tts" + } + ] + } + ], + "data": [ + { + "name": "data", + "source": [ + { + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension": "streamid_adapter", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "stt" + } + ] + } + ] + } + ] + } + } + ], + "log": { + "handlers": [ + { + "matchers": [ + { + "level": "info" + } + ], + "formatter": { + "type": "plain", + "colored": true + }, + "emitter": { + "type": "console", + "config": { + "stream": "stdout" + } + } + } + ] + } + } +} diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/scripts/install_python_deps.sh b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/scripts/install_python_deps.sh new file mode 100644 index 0000000000..c0fe1a3071 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/scripts/install_python_deps.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Install Python dependencies for Language Tutor with PowerMem + +# Install main extension dependencies +echo "Installing main Python dependencies..." +cd ../../ten_packages/extension/main_python +pip install -r requirements.txt -q + +# Install openai_llm2_python dependencies +echo "Installing LLM extension dependencies..." +cd ../../ten_packages/extension/openai_llm2_python +pip install -r requirements.txt -q + +echo "Python dependencies installed successfully." diff --git a/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/scripts/start.sh b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/scripts/start.sh new file mode 100644 index 0000000000..9144935f9b --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant-with-PowerMem-LanguageTutor/tenapp/scripts/start.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Start script for Language Tutor with PowerMem + +# Set default environment variables if not set +export PYTHONUNBUFFERED=1 + +# Navigate to the app directory +cd "$(dirname "$0")" + +echo "Starting Language Tutor with PowerMem..." + +# Run the application +go run main.go