diff --git a/.github/workflows/test-mcp-examples.yaml b/.github/workflows/test-mcp-examples.yaml deleted file mode 100644 index 601c5532..00000000 --- a/.github/workflows/test-mcp-examples.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: Test MCP Examples - -on: - push: - branches: [main] - paths: - - 'modules/ai-agents/examples/**/*.yaml' - - 'modules/ai-agents/examples/test-all.sh' - pull_request: - branches: [main] - paths: - - 'modules/ai-agents/examples/**/*.yaml' - - 'modules/ai-agents/examples/test-all.sh' - -jobs: - test-all: - name: Test All Examples - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install dependencies - run: npm install - - - name: Install yq - run: | - sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 - sudo chmod +x /usr/local/bin/yq - - - name: Install tools - run: | - npx doc-tools install-test-dependencies - rpk connect install - - - name: Make test script executable - run: chmod +x modules/ai-agents/examples/test-all.sh - - - name: Run comprehensive test suite - run: | - cd modules/ai-agents/examples - ./test-all.sh - - test-mcp-tools: - name: Test MCP Tools - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install dependencies - run: npm install - - - name: Install tools - run: | - npx doc-tools install-test-dependencies - rpk connect install - - - name: Make test script executable - run: chmod +x modules/ai-agents/examples/test-all.sh - - - name: Test MCP tool definitions - run: | - cd modules/ai-agents/examples - ./test-all.sh --mcp-only - - test-pipelines: - name: Test Pipelines - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install dependencies - run: npm install - - - name: Install tools - run: | - npx doc-tools install-test-dependencies - rpk connect install - - - name: Make test script executable - run: chmod +x modules/ai-agents/examples/test-all.sh - - - name: Test pipeline examples - run: | - cd modules/ai-agents/examples - ./test-all.sh --pipelines - - test-snippets: - name: Test Snippets - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install dependencies - run: npm install - - - name: Install tools - run: | - npx doc-tools install-test-dependencies - rpk connect install - - - name: Make test script executable - run: chmod +x modules/ai-agents/examples/test-all.sh - - - name: Test config snippets - run: | - cd modules/ai-agents/examples - ./test-all.sh --snippets diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index e57600ba..d1da0771 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -15,15 +15,6 @@ ** xref:install:prebuilt-binary.adoc[Prebuilt Binary] ** xref:install:build-from-source.adoc[Build from Source] -* xref:ai-agents:mcp-server/index.adoc[MCP Servers] -** xref:ai-agents:mcp-server/overview.adoc[Overview] -** xref:ai-agents:mcp-server/quickstart.adoc[Quickstart] -** xref:ai-agents:mcp-server/concepts.adoc[Concepts] -** xref:ai-agents:mcp-server/create-tool.adoc[Create a Tool] -** xref:ai-agents:mcp-server/best-practices.adoc[Best Practices] -** xref:ai-agents:mcp-server/tool-patterns.adoc[Tool Patterns] -** xref:ai-agents:mcp-server/troubleshooting.adoc[Troubleshoot] - * xref:configuration:about.adoc[] ** xref:configuration:resources.adoc[] ** xref:configuration:allow_and_deny_lists.adoc[Allow and Deny Lists] diff --git a/modules/ai-agents/examples/best-practices/config-examples/correct-processor.yaml b/modules/ai-agents/examples/best-practices/config-examples/correct-processor.yaml deleted file mode 100644 index c1eb4216..00000000 --- a/modules/ai-agents/examples/best-practices/config-examples/correct-processor.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Test: Correct processor component configuration -# Validates: Processor with try/catch error handling - -# tag::correct-processor[] -label: fetch-example-data - -processors: - - label: safe_operation - try: - - http: - url: "https://httpbin.org/json" - verb: GET - timeout: "10s" - - mutation: | - root = this.merge({"processed": true}) - - - label: handle_errors - catch: - - mutation: | - root = { - "error": "Operation failed", - "details": error() - } - -meta: - mcp: - enabled: true - description: "Fetch data from external API with error handling" - properties: - - name: endpoint - type: string - description: "API endpoint to fetch" - required: false -# end::correct-processor[] diff --git a/modules/ai-agents/examples/best-practices/debugging/component-type-errors.yaml b/modules/ai-agents/examples/best-practices/debugging/component-type-errors.yaml deleted file mode 100644 index 2e209b80..00000000 --- a/modules/ai-agents/examples/best-practices/debugging/component-type-errors.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Test: Component type configuration patterns -# Validates: Single component type vs incorrect multi-component - -# tag::incorrect-multiple[] -# Wrong: Multiple top-level component types (DO NOT USE) -# input: -# redpanda: { ... } -# processors: -# - mutation: { ... } -# output: -# redpanda: { ... } -# end::incorrect-multiple[] - -# tag::correct-single[] -label: my-input - -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "events" ] - consumer_group: "mcp-reader" - -meta: - mcp: - enabled: true - description: "Consume events from Redpanda" - properties: - - name: count - type: number - description: "Number of messages to read" - required: false -# end::correct-single[] diff --git a/modules/ai-agents/examples/best-practices/debugging/debug-logging.yaml b/modules/ai-agents/examples/best-practices/debugging/debug-logging.yaml deleted file mode 100644 index b3347476..00000000 --- a/modules/ai-agents/examples/best-practices/debugging/debug-logging.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Test: Debug logging pattern for troubleshooting -# Validates: Log processors for debugging data flow - -label: debug-data-flow - -# tag::debug-logging[] -processors: - - label: debug_input - log: - message: "Input received: ${! json() }" - level: DEBUG - - - label: actual_processing - mutation: | - root = this - root.processed = true - - - label: debug_output - log: - message: "Output: ${! json() }" - level: DEBUG -# end::debug-logging[] - -meta: - mcp: - enabled: true - description: "Process data with debug logging" - properties: - - name: data - type: string - description: "Data to process" - required: false diff --git a/modules/ai-agents/examples/best-practices/error-handling-snippets/common-mistakes.yaml b/modules/ai-agents/examples/best-practices/error-handling-snippets/common-mistakes.yaml deleted file mode 100644 index ed89d28b..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling-snippets/common-mistakes.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Test: Common error handling mistakes (documentation only) -# Shows incorrect patterns to avoid - -label: correct-error-handling - -processors: - - label: proper_try - try: - - mutation: | - root = this - root.processed = true - - - label: proper_catch - catch: - - mutation: | - root = { - "error": "Processing failed", - "details": error() - } - -meta: - mcp: - enabled: true - description: "Example of correct error handling pattern" - -# tag::wrong-single-processor[] -# Wrong: try/catch cannot be a single processor -# - label: operation -# try: -# - http: { url: "..." } -# catch: -# - mapping: 'root = {"error": error()}' -# end::wrong-single-processor[] - -# tag::wrong-no-handling[] -# Wrong: No error handling - tool fails silently -# - label: fetch_data -# http: -# url: "https://api.example.com/data" -# end::wrong-no-handling[] diff --git a/modules/ai-agents/examples/best-practices/error-handling-snippets/timeout-config.yaml b/modules/ai-agents/examples/best-practices/error-handling-snippets/timeout-config.yaml deleted file mode 100644 index 43e35a45..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling-snippets/timeout-config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Test: HTTP timeout configuration -# Validates: Proper timeout, retry, and retry_period settings - -label: fetch-with-timeout - -processors: - # tag::timeout-config[] - - label: fetch_with_timeout - try: - - http: - url: "https://httpbin.org/get" - verb: GET - timeout: "10s" # Fail after 10 seconds - retries: 2 # Retry twice before failing - retry_period: "1s" - # end::timeout-config[] - - - label: handle_timeout_errors - catch: - - mutation: | - root = { - "error": "Request timed out", - "details": error() - } - -meta: - mcp: - enabled: true - description: "Fetch data with timeout and retry configuration" - properties: - - name: endpoint - type: string - description: "API endpoint to fetch" - required: false diff --git a/modules/ai-agents/examples/best-practices/error-handling/basic-try-catch.yaml b/modules/ai-agents/examples/best-practices/error-handling/basic-try-catch.yaml deleted file mode 100644 index 0cdb226e..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling/basic-try-catch.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Test: Wrap operations in try/catch -# Source: best-practices.adoc - Error handling section -# Expected: Error caught and formatted with message -# Note: Uses simulated error since we can't rely on external APIs in tests - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"request": "data"} - -pipeline: - # tag::basic-try-catch[] - processors: - - try: - - http: - url: "https://api.example.com/data" - verb: GET - - - catch: - - mutation: | - root.error = true - root.message = "Request failed: " + error() - # end::basic-try-catch[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/error-handling/error-with-context.yaml b/modules/ai-agents/examples/best-practices/error-handling/error-with-context.yaml deleted file mode 100644 index 32806566..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling/error-with-context.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Test: Include context in error responses -# Source: best-practices.adoc - Error handling section -# Expected: Error includes original input and timestamp -# Note: Uses simulated error since we can't rely on database in tests - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"customer_id": "cust_12345"} - -pipeline: - processors: - - try: - # Simulate database failure for testing - - mutation: | - root = throw("connection refused") - - # tag::error-with-context[] - - catch: - - mutation: | - root.error = true - root.message = "Failed to look up customer: " + error() - root.input = {"customer_id": this.customer_id} - root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") - # end::error-with-context[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/error-handling/error-with-timestamp.yaml b/modules/ai-agents/examples/best-practices/error-handling/error-with-timestamp.yaml deleted file mode 100644 index 770a83f2..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling/error-with-timestamp.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Test: Error response with timestamp from error-handling.adoc -# Expected: Error captured with success=false and formatted timestamp - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"action": "trigger_error"} - -pipeline: - processors: - - label: simulate_error - try: - - mutation: | - root = throw("Database connection failed") - - # tag::error-with-timestamp[] - - label: handle_errors - catch: - - mutation: | - root = { - "success": false, - "error_message": error(), - "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") - } - # end::error-with-timestamp[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/error-handling/handle-error-types.yaml b/modules/ai-agents/examples/best-practices/error-handling/handle-error-types.yaml deleted file mode 100644 index 168bf512..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling/handle-error-types.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# Test: Handle specific error types -# Source: best-practices.adoc - Error handling section -# Expected: Categorized errors with retry suggestion - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"simulate": "timeout"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"simulate": "connection"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"simulate": "not_found"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"simulate": "other"}' - -pipeline: - processors: - - try: - - mutation: | - root = match this.simulate { - "timeout" => throw("Request timeout after 30s") - "connection" => throw("connection refused") - "not_found" => throw("404 resource not found") - _ => throw("Something unexpected happened") - } - - # tag::handle-error-types[] - - catch: - - mutation: | - let err = error() - root.error = true - root.error_type = if $err.contains("timeout") { - "TIMEOUT" - } else if $err.contains("connection refused") { - "CONNECTION_ERROR" - } else if $err.contains("404") { - "NOT_FOUND" - } else { - "UNKNOWN" - } - root.message = $err - root.retry_suggested = root.error_type == "TIMEOUT" || root.error_type == "CONNECTION_ERROR" - # end::handle-error-types[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/error-handling/http-timeout.yaml b/modules/ai-agents/examples/best-practices/error-handling/http-timeout.yaml deleted file mode 100644 index 01ffe9a8..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling/http-timeout.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Test: Set timeouts on external calls -# Source: best-practices.adoc - Error handling section -# Expected: HTTP processor with timeout and retry configuration -# Note: This is a config example, not a runnable test - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"endpoint": "/data"} - -pipeline: - processors: - - try: - # tag::http-timeout[] - - http: - url: "https://api.example.com/data" - verb: GET - timeout: 10s - retries: 2 - retry_period: 1s - # end::http-timeout[] - - - catch: - - mutation: | - root.error = true - root.message = error() - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/error-handling/log-errors.yaml b/modules/ai-agents/examples/best-practices/error-handling/log-errors.yaml deleted file mode 100644 index 7790792d..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling/log-errors.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Test: Log errors for debugging -# Source: best-practices.adoc - Error handling section -# Expected: Error logged with structured fields and formatted response returned - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"user_id": 12345, "action": "fetch_profile"} - -pipeline: - processors: - - try: - - mutation: | - root = throw("Connection refused to database") - - # tag::log-errors[] - - catch: - - log: - message: "Tool failed: ${! error() }" - level: ERROR - fields_mapping: | - root.input = this - root.error = error() - - - mutation: | - root.error = true - root.message = error() - # end::log-errors[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/error-handling/preserve-input-context.yaml b/modules/ai-agents/examples/best-practices/error-handling/preserve-input-context.yaml deleted file mode 100644 index 6e333d5c..00000000 --- a/modules/ai-agents/examples/best-practices/error-handling/preserve-input-context.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Test: Preserving input context in errors from error-handling.adoc -# Expected: Original input preserved in error response via metadata - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"user_id": "usr_abc123", "action": "fetch_profile"} - -pipeline: - processors: - # tag::preserve-input-context[] - - label: validate_and_fetch - try: - - mutation: | - meta original_input = this - - mutation: | - root = throw("User not found in database") - - - label: handle_errors - catch: - - mutation: | - root = { - "error": "Failed to fetch user", - "details": error(), - "input_received": @original_input, - "suggestion": "Verify the user_id exists" - } - # end::preserve-input-context[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/check-required-fields.yaml b/modules/ai-agents/examples/best-practices/input-validation/check-required-fields.yaml deleted file mode 100644 index e0d93508..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/check-required-fields.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Test: Check required fields -# Source: best-practices.adoc - Input validation section -# Expected: Empty/missing city returns error object, valid city passes through - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": "London"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": " "}' - - generate: - count: 1 - interval: "" - mapping: 'root = {}' - -pipeline: - processors: - # tag::check-required-fields[] - - mutation: | - root = if this.city.or("").trim() == "" { - {"error": true, "message": "city is required"} - } else { - this - } - # end::check-required-fields[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/sanitize-string-input.yaml b/modules/ai-agents/examples/best-practices/input-validation/sanitize-string-input.yaml deleted file mode 100644 index be0120f9..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/sanitize-string-input.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Test: Sanitizing string inputs from input-validation.adoc -# Expected: HTML entities removed, special chars sanitized - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": "New York!@#$"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": "San-Francisco"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": "!@#$%"}' - -pipeline: - processors: - # tag::sanitize-input[] - - label: sanitize_input - mutation: | - let clean_city = this.city.or("").trim().re_replace_all("[^a-zA-Z\\s-]", "") - root = if $clean_city == "" { - {"error": "City name contains only invalid characters"} - } else { - {"city": $clean_city} - } - # end::sanitize-input[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/sanitize-strings.yaml b/modules/ai-agents/examples/best-practices/input-validation/sanitize-strings.yaml deleted file mode 100644 index 39d7ed53..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/sanitize-strings.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Test: Sanitize string inputs -# Source: best-practices.adoc - Input validation section -# Expected: Stores sanitized value in metadata - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": "New-York"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": "Paris!@#123"}' - -pipeline: - processors: - # tag::sanitize-strings[] - - mutation: | - # Remove any character that isn't a letter, space, or hyphen - meta sanitized_city = this.city.re_replace_all("[^a-zA-Z\\s\\-]", "") - root = this - # end::sanitize-strings[] - - # For testing: show what was stored in metadata - - mutation: | - root.sanitized = @sanitized_city - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/throw-on-critical.yaml b/modules/ai-agents/examples/best-practices/input-validation/throw-on-critical.yaml deleted file mode 100644 index 8b0b64e5..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/throw-on-critical.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Test: Stop processing on critical failures -# Source: best-practices.adoc - Input validation section -# Expected: Missing api_key throws error, valid api_key passes through - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"api_key": "sk-12345", "data": "test"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"data": "test"}' - -pipeline: - processors: - - try: - # tag::throw-on-critical[] - - mutation: | - root = if !this.exists("api_key") || this.api_key == "" { - throw("API key is required for this operation") - } else { - this - } - # end::throw-on-critical[] - - - catch: - - mutation: | - root.error = true - root.message = error() - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/validate-enum-values.yaml b/modules/ai-agents/examples/best-practices/input-validation/validate-enum-values.yaml deleted file mode 100644 index f09f9ff2..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/validate-enum-values.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Test: Enum value validation from input-validation.adoc -# Expected: Status normalized and validated against allowed values - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"status": "PENDING"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"status": "approved"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"status": "invalid_status"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {}' - -pipeline: - processors: - # tag::validate-status[] - - label: validate_status - mutation: | - let allowed = ["pending", "approved", "rejected"] - let status = this.status.or("").lowercase() - root = if $status == "" { - {"error": "status is required", "allowed": $allowed} - } else if !$allowed.contains($status) { - {"error": "invalid status", "received": $status, "allowed": $allowed} - } else { - {"status": $status, "valid": true} - } - # end::validate-status[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/validate-enum.yaml b/modules/ai-agents/examples/best-practices/input-validation/validate-enum.yaml deleted file mode 100644 index da688cdc..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/validate-enum.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Test: Validate enum values -# Source: best-practices.adoc - Input validation section -# Expected: Invalid status returns error with allowed values, valid normalizes to lowercase - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"status": "ACTIVE", "id": "123"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"status": "invalid", "id": "456"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"status": "Completed", "id": "789"}' - -pipeline: - processors: - # tag::validate-enum[] - - mutation: | - let status = this.status.lowercase() - let allowed = ["pending", "active", "completed", "cancelled"] - root = if !$allowed.contains($status) { - {"error": true, "message": "invalid status", "allowed_values": $allowed, "received": $status} - } else { - this.without("status").assign({"status": $status}) - } - # end::validate-enum[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/validate-multiple-fields.yaml b/modules/ai-agents/examples/best-practices/input-validation/validate-multiple-fields.yaml deleted file mode 100644 index a469ab90..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/validate-multiple-fields.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Test: Multiple field validation from input-validation.adoc -# Expected: All required fields validated, errors collected - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"order_id": "123", "items": [{"id": 1}], "email": "test@example.com"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"items": [{"id": 1}]}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"order_id": "123", "items": [], "email": "invalid-email"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {}' - -pipeline: - processors: - # tag::validate-order[] - - label: validate_order - mutation: | - let errors = [] - let errors = if !this.exists("order_id") || this.order_id == "" { - $errors.append("order_id is required") - } else { $errors } - let errors = if !this.exists("items") || this.items.length() == 0 { - $errors.append("at least one item is required") - } else { $errors } - let errors = if this.exists("email") && !this.email.contains("@") { - $errors.append("invalid email format") - } else { $errors } - root = if $errors.length() > 0 { - {"valid": false, "errors": $errors} - } else { - {"valid": true, "order_id": this.order_id} - } - # end::validate-order[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/validate-numeric-range.yaml b/modules/ai-agents/examples/best-practices/input-validation/validate-numeric-range.yaml deleted file mode 100644 index 75e305b5..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/validate-numeric-range.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Test: Numeric range validation from input-validation.adoc -# Expected: Quantity validated against min/max range - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"quantity": 0}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"quantity": 50}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"quantity": 1001}' - - generate: - count: 1 - interval: "" - mapping: 'root = {}' - -pipeline: - processors: - # tag::validate-quantity[] - - label: validate_quantity - mutation: | - let qty = this.quantity.or(0).number() - root = if $qty < 1 { - {"error": "Quantity must be at least 1", "received": $qty} - } else if $qty > 1000 { - {"error": "Quantity cannot exceed 1000", "received": $qty} - } else { - {"quantity": $qty, "valid": true} - } - # end::validate-quantity[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/validate-numeric-ranges.yaml b/modules/ai-agents/examples/best-practices/input-validation/validate-numeric-ranges.yaml deleted file mode 100644 index ae803440..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/validate-numeric-ranges.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Test: Validate numeric ranges -# Source: best-practices.adoc - Input validation section -# Expected: quantity < 1 or > 100 returns error, valid passes through - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"quantity": 50}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"quantity": 0}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"quantity": 150}' - - generate: - count: 1 - interval: "" - mapping: 'root = {}' - -pipeline: - processors: - # tag::validate-numeric-ranges[] - - mutation: | - let quantity = this.quantity.or(0).number() - root = if $quantity < 1 { - {"error": true, "message": "quantity must be at least 1", "received": $quantity} - } else if $quantity > 100 { - {"error": true, "message": "quantity cannot exceed 100", "received": $quantity} - } else { - this - } - # end::validate-numeric-ranges[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/input-validation/validate-required-field.yaml b/modules/ai-agents/examples/best-practices/input-validation/validate-required-field.yaml deleted file mode 100644 index e28d6fba..00000000 --- a/modules/ai-agents/examples/best-practices/input-validation/validate-required-field.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Test: Basic input validation from input-validation.adoc -# Expected: Empty city returns error, valid city returns data - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": "London"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"city": " "}' - - generate: - count: 1 - interval: "" - mapping: 'root = {}' - -pipeline: - processors: - # tag::validate-input[] - - label: validate_input - mutation: | - let city = this.city.or("").trim() - root = if $city == "" { - {"error": "City name is required"} - } else { - {"city": $city} - } - # end::validate-input[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/mcp-metadata/basic-mcp-block.yaml b/modules/ai-agents/examples/best-practices/mcp-metadata/basic-mcp-block.yaml deleted file mode 100644 index a481747d..00000000 --- a/modules/ai-agents/examples/best-practices/mcp-metadata/basic-mcp-block.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Test: Basic MCP metadata block structure -# Validates: MCP server can parse basic tool metadata - -# tag::basic-mcp-block[] -label: get-weather - -meta: - tags: [ weather, api, example ] - mcp: - enabled: true - description: "Get current weather for a city. Returns temperature, humidity, and conditions." - properties: - - name: city - type: string - description: "City name (e.g., 'London', 'Tokyo', 'New York')" - required: true -# end::basic-mcp-block[] - -processors: - - mutation: | - root = { - "temperature": 22, - "humidity": 65, - "conditions": "partly cloudy", - "city": this.city - } diff --git a/modules/ai-agents/examples/best-practices/mcp-metadata/descriptions.yaml b/modules/ai-agents/examples/best-practices/mcp-metadata/descriptions.yaml deleted file mode 100644 index cb69e526..00000000 --- a/modules/ai-agents/examples/best-practices/mcp-metadata/descriptions.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Test: Tool description best practices -# Validates: Good descriptions explain what tool does and returns - -label: get-city-weather - -processors: - - mutation: | - root = { - "temperature_celsius": 22, - "temperature_fahrenheit": 72, - "humidity_percent": 65, - "description": "Partly cloudy" - } - -meta: - mcp: - enabled: true - # tag::good-description[] - description: "Get current weather for a city. Returns temperature in Celsius and Fahrenheit, humidity percentage, and weather description." - # end::good-description[] - properties: - - name: city - type: string - description: "City name" - required: true - -# tag::with-limitations[] - description: "Search product catalog by name or category. Returns up to 10 results. Requires at least one search term." -# end::with-limitations[] diff --git a/modules/ai-agents/examples/best-practices/mcp-metadata/property-definitions.yaml b/modules/ai-agents/examples/best-practices/mcp-metadata/property-definitions.yaml deleted file mode 100644 index 27a0f655..00000000 --- a/modules/ai-agents/examples/best-practices/mcp-metadata/property-definitions.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Test: Property definition patterns -# Validates: Properties have proper types, descriptions, and required flags - -label: get-weather-detailed - -processors: - - mutation: | - let units = this.units | "celsius" - let limit = this.limit | 10 - let include_forecast = this.include_forecast | false - root = { - "city": this.city, - "temperature": if $units == "fahrenheit" { 72 } else { 22 }, - "units": $units, - "forecast": if $include_forecast { [{"day": 1, "temp": 23}] } else { null } - } - -meta: - mcp: - enabled: true - description: "Get detailed weather with optional forecast" - # tag::property-descriptions[] - properties: - - name: city - type: string - description: "City name (e.g., 'London', 'New York', 'Tokyo')" - required: true - - - name: units - type: string - description: "Temperature units: 'celsius' or 'fahrenheit'. Defaults to 'celsius'." - required: false - - - name: limit - type: number - description: "Maximum results to return (1-100). Defaults to 10." - required: false - - - name: include_forecast - type: boolean - description: "If true, include 5-day forecast. Defaults to false." - required: false - # end::property-descriptions[] - -# tag::required-property[] - properties: - - name: city - type: string - description: "City name to get weather for" - required: true -# end::required-property[] - -# tag::optional-property[] - - name: units - type: string - description: "Temperature units. Defaults to 'celsius'." - required: false -# end::optional-property[] diff --git a/modules/ai-agents/examples/best-practices/mcp-metadata/search-customer-orders.yaml b/modules/ai-agents/examples/best-practices/mcp-metadata/search-customer-orders.yaml deleted file mode 100644 index a6955405..00000000 --- a/modules/ai-agents/examples/best-practices/mcp-metadata/search-customer-orders.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# Test: Complete MCP tool example with all metadata patterns -# Validates: Full tool with description, multiple properties, and tags - -# tag::complete-example[] -label: search-customer-orders - -processors: - - mutation: | - let customer_id = this.customer_id | "" - let status = this.status | "" - let limit = (this.limit | 10).number() - root = { - "orders": [ - { - "order_id": "ord_001", - "customer_id": $customer_id, - "status": if $status != "" { $status } else { "delivered" }, - "total": 125.99 - } - ], - "count": 1, - "filters_applied": { - "customer_id": $customer_id, - "status": $status, - "limit": $limit - } - } - -meta: - tags: [ orders, database, production ] - mcp: - enabled: true - description: "Search customer orders by customer ID, date range, or status. Returns order summaries with totals. Maximum 50 results per query." - properties: - - name: customer_id - type: string - description: "Customer ID (e.g., 'cust_12345'). Required if no other filters provided." - required: false - - name: status - type: string - description: "Order status filter: 'pending', 'shipped', 'delivered', or 'cancelled'." - required: false - - name: start_date - type: string - description: "Start date for date range filter (ISO 8601, e.g., '2024-01-01')." - required: false - - name: end_date - type: string - description: "End date for date range filter (ISO 8601, e.g., '2024-12-31')." - required: false - - name: limit - type: number - description: "Maximum results to return (1-50). Defaults to 10." - required: false -# end::complete-example[] diff --git a/modules/ai-agents/examples/best-practices/mcp-metadata/tags-example.yaml b/modules/ai-agents/examples/best-practices/mcp-metadata/tags-example.yaml deleted file mode 100644 index 9524e268..00000000 --- a/modules/ai-agents/examples/best-practices/mcp-metadata/tags-example.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Test: Using tags for tool organization -# Validates: Tags array in meta section - -label: get-external-weather - -processors: - - mutation: | - root = { - "source": "external-api", - "temperature": 22, - "conditions": "sunny" - } - -# tag::tags-example[] -meta: - tags: [ weather, api, external, production ] - mcp: - enabled: true - description: "Get weather from external API" - properties: - - name: city - type: string - description: "City name" - required: true -# end::tags-example[] diff --git a/modules/ai-agents/examples/best-practices/production-workflows/conditional-processing.yaml b/modules/ai-agents/examples/best-practices/production-workflows/conditional-processing.yaml deleted file mode 100644 index af0ef0d1..00000000 --- a/modules/ai-agents/examples/best-practices/production-workflows/conditional-processing.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Test: Conditional processing with switch -# Validates: Branch processing based on data type - -label: parse-data-by-type - -# tag::conditional-processing[] -processors: - - label: conditional_processing - switch: - - check: this.data_type == "json" - processors: - - mapping: | - root.parsed_data = this.content.parse_json() - root.format = "json" - - check: this.data_type == "csv" - processors: - - mapping: | - root.parsed_data = this.content.parse_csv() - root.format = "csv" - - processors: - - mapping: | - root.error = "Unsupported data type" - root.supported_types = ["json", "csv"] -# end::conditional-processing[] - -meta: - mcp: - enabled: true - description: "Parse data based on specified type" - properties: - - name: data_type - type: string - description: "Data format: json or csv" - required: true - - name: content - type: string - description: "Content to parse" - required: true diff --git a/modules/ai-agents/examples/best-practices/production-workflows/dynamic-config.yaml b/modules/ai-agents/examples/best-practices/production-workflows/dynamic-config.yaml deleted file mode 100644 index db48a598..00000000 --- a/modules/ai-agents/examples/best-practices/production-workflows/dynamic-config.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Test: Dynamic configuration based on input -# Validates: Environment-based branching and conditional logic - -label: dynamic-order-query - -# tag::dynamic-config[] -processors: - - label: dynamic_config - mutation: | - # Choose data source based on environment - meta env = this.environment | "production" - meta table_name = match @env { - "dev" => "dev_orders", - "staging" => "staging_orders", - "production" => "prod_orders", - _ => "dev_orders" - } - - # Adjust query complexity based on urgency - meta columns = if this.detailed.bool().catch(false) { - ["order_id", "customer_id", "total", "items", "shipping_address"] - } else { - ["order_id", "customer_id", "total"] - } -# end::dynamic-config[] - - - mutation: | - root = { - "environment": @env, - "table": @table_name, - "columns": @columns, - "config_applied": true - } - -meta: - mcp: - enabled: true - description: "Configure query dynamically based on environment" - properties: - - name: environment - type: string - description: "Environment: dev, staging, or production" - required: false - - name: detailed - type: boolean - description: "Include detailed columns" - required: false diff --git a/modules/ai-agents/examples/best-practices/production-workflows/error-fallbacks.yaml b/modules/ai-agents/examples/best-practices/production-workflows/error-fallbacks.yaml deleted file mode 100644 index 87dfa102..00000000 --- a/modules/ai-agents/examples/best-practices/production-workflows/error-fallbacks.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Test: Error handling with fallback sources -# Validates: Try/catch with fallback API pattern - -label: fetch-with-fallback - -# tag::error-fallbacks[] -processors: - - label: primary_fetch - try: - - http: - url: "https://httpbin.org/json" - verb: GET - timeout: "10s" - catch: - - log: - message: "Primary API failed, trying fallback" - level: WARN - - label: fallback_fetch - http: - url: "https://httpbin.org/get" - verb: GET - timeout: "15s" - - mutation: | - root.metadata.source = "fallback" - root.metadata.warning = "Primary source unavailable" -# end::error-fallbacks[] - - - mutation: | - root.fetch_completed = true - -meta: - mcp: - enabled: true - description: "Fetch data with automatic fallback on failure" - properties: - - name: query - type: string - description: "Query parameter" - required: false diff --git a/modules/ai-agents/examples/best-practices/production-workflows/param-validation.yaml b/modules/ai-agents/examples/best-practices/production-workflows/param-validation.yaml deleted file mode 100644 index de85b04c..00000000 --- a/modules/ai-agents/examples/best-practices/production-workflows/param-validation.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Test: Parameter validation patterns -# Validates: Required field checking and type coercion - -label: validate-user-params - -# tag::param-validation[] -processors: - - label: validate_params - mutation: | - # Validate required parameters - root = if !this.exists("user_id") { - throw("user_id parameter is required") - } else { this } - - # Type coercion with validation - meta user_id = this.user_id.string() - meta limit = this.limit.number().catch(10) - meta start_date = this.start_date.parse_timestamp("2006-01-02").catch(now() - duration("24h")) -# end::param-validation[] - - - mutation: | - root = { - "user_id": @user_id, - "limit": @limit, - "validated": true - } - -meta: - mcp: - enabled: true - description: "Validate and coerce user parameters" - properties: - - name: user_id - type: string - description: "User ID (required)" - required: true - - name: limit - type: number - description: "Result limit (defaults to 10)" - required: false - - name: start_date - type: string - description: "Start date in YYYY-MM-DD format" - required: false diff --git a/modules/ai-agents/examples/best-practices/response-formatting/add-timestamps.yaml b/modules/ai-agents/examples/best-practices/response-formatting/add-timestamps.yaml deleted file mode 100644 index ed4d48f8..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/add-timestamps.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Test: Timestamp formatting from response-formatting.adoc -# Expected: Original data preserved, three timestamp fields added - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"user": "test", "action": "login"} - -pipeline: - processors: - # tag::add-timestamps[] - - label: add_timestamps - mutation: | - root = this - root.timestamp = now().format_timestamp("2006-01-02T15:04:05Z07:00") - root.date = now().format_timestamp("2006-01-02") - root.time = now().format_timestamp("15:04:05") - # end::add-timestamps[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/conditional-fields.yaml b/modules/ai-agents/examples/best-practices/response-formatting/conditional-fields.yaml deleted file mode 100644 index 3b7ca4df..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/conditional-fields.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Test: Include fields conditionally -# Source: best-practices.adoc - Response formatting section -# Expected: Omits empty/missing fields using deleted() - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"id": "123", "name": "Alice", "email": "alice@example.com", "phone": "+1234567890"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"id": "456", "name": "Bob", "email": ""}' - -pipeline: - processors: - # tag::conditional-fields[] - - mutation: | - root.id = this.id - root.name = this.name - root.email = if this.exists("email") && this.email != "" { this.email } else { deleted() } - root.phone = if this.exists("phone") { this.phone } else { deleted() } - # end::conditional-fields[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/extract-nested-fields.yaml b/modules/ai-agents/examples/best-practices/response-formatting/extract-nested-fields.yaml deleted file mode 100644 index ac3c5751..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/extract-nested-fields.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Test: Extracting nested fields from response-formatting.adoc -# Expected: Deeply nested structure flattened to simple object - -input: - generate: - count: 1 - interval: "" - mapping: | - root = { - "data": { - "user": { - "id": 123, - "profile": {"display_name": "John Doe"}, - "contact": {"email": "john@example.com"}, - "status": {"verified": true}, - "metadata": {"created_at": "2024-01-15"} - } - } - } - -pipeline: - processors: - # tag::extract-nested[] - - label: extract_user_data - mapping: | - root = { - "user_id": this.data.user.id.string(), - "name": this.data.user.profile.display_name, - "email": this.data.user.contact.email, - "is_verified": this.data.user.status.verified.bool(), - "created_at": this.data.user.metadata.created_at - } - # end::extract-nested[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/extract-nested.yaml b/modules/ai-agents/examples/best-practices/response-formatting/extract-nested.yaml deleted file mode 100644 index cea46da9..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/extract-nested.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Test: Extract nested fields -# Source: best-practices.adoc - Response formatting section -# Expected: Flattens deeply nested structure - -input: - generate: - count: 1 - interval: "" - mapping: | - root = { - "data": { - "user": { - "id": 12345, - "profile": {"display_name": "Alice"}, - "contact": {"email": "alice@example.com"}, - "status": {"verified": true} - } - } - } - -pipeline: - processors: - # tag::extract-nested[] - - mutation: | - root.user_id = this.data.user.id.string() - root.display_name = this.data.user.profile.display_name - root.email = this.data.user.contact.email - root.is_verified = this.data.user.status.verified.bool() - # end::extract-nested[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/filter-sensitive-data.yaml b/modules/ai-agents/examples/best-practices/response-formatting/filter-sensitive-data.yaml deleted file mode 100644 index 002ef5ae..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/filter-sensitive-data.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Test: Filter sensitive data -# Source: best-practices.adoc - Response formatting section -# Expected: Sensitive fields removed, other fields preserved - -input: - generate: - count: 1 - interval: "" - mapping: | - root = { - "user_id": 123, - "name": "John Doe", - "email": "john@example.com", - "password": "secret123", - "ssn": "123-45-6789", - "api_key": "sk-abc123", - "internal_notes": "VIP customer" - } - -pipeline: - processors: - # tag::filter-sensitive[] - - mutation: | - root = this.without("password", "ssn", "api_key", "internal_notes") - # end::filter-sensitive[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/format-basic-response.yaml b/modules/ai-agents/examples/best-practices/response-formatting/format-basic-response.yaml deleted file mode 100644 index 79e3f28c..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/format-basic-response.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Test: Basic response formatting from response-formatting.adoc -# Expected: Nested weather data flattened with timestamp - -input: - generate: - count: 1 - interval: "" - mapping: | - root = { - "location": {"name": "London"}, - "current": { - "temp_c": "15.5", - "condition": {"text": "Partly cloudy"} - } - } - -pipeline: - processors: - # tag::format-response[] - - label: format_response - mapping: | - root = { - "city": this.location.name, - "temperature_c": this.current.temp_c.number(), - "description": this.current.condition.text, - "timestamp": now().ts_format("2006-01-02T15:04:05Z") - } - # end::format-response[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/handle-arrays.yaml b/modules/ai-agents/examples/best-practices/response-formatting/handle-arrays.yaml deleted file mode 100644 index df1d83a6..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/handle-arrays.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Test: Array handling from response-formatting.adoc -# Expected: Array processed with length, map_each, index - -input: - generate: - count: 1 - interval: "" - mapping: | - root = { - "items": [ - {"id": 1, "name": "Apple", "price": "1.50"}, - {"id": 2, "name": "Banana", "price": "0.75"}, - {"id": 3, "name": "Orange", "price": "2.00"} - ] - } - -pipeline: - processors: - # tag::handle-arrays[] - - label: format_items - mapping: | - root = { - "total_items": this.items.length(), - "items": this.items.map_each(item -> { - "id": item.id.string(), - "name": item.name, - "price": item.price.number() - }), - "first_item": this.items.index(0).name, - "item_names": this.items.map_each(i -> i.name) - } - # end::handle-arrays[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/structure-output.yaml b/modules/ai-agents/examples/best-practices/response-formatting/structure-output.yaml deleted file mode 100644 index 3c1a2c77..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/structure-output.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Test: Structure output fields -# Source: best-practices.adoc - Response formatting section -# Expected: Groups fields into customer object with timestamp - -input: - generate: - count: 1 - interval: "" - mapping: | - root = { - "id": "cust_123", - "name": "Alice Smith", - "email": "alice@example.com", - "plan": "premium" - } - -pipeline: - processors: - # tag::structure-output[] - - mutation: | - root.customer = { - "id": this.id, - "name": this.name, - "email": this.email, - "plan": this.plan - } - root.retrieved_at = now() - # end::structure-output[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/success-envelope.yaml b/modules/ai-agents/examples/best-practices/response-formatting/success-envelope.yaml deleted file mode 100644 index d0c1ca94..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/success-envelope.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Test: Success envelope structure from response-formatting.adoc -# Expected: Data wrapped in success envelope with timestamp - -input: - generate: - count: 1 - interval: "" - mapping: | - root = {"id": 123, "name": "Test User"} - -pipeline: - processors: - # tag::success-envelope[] - - label: format_success - mapping: | - root = { - "success": true, - "data": { - "user_id": this.id, - "name": this.name - }, - "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") - } - # end::success-envelope[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/success-error-envelope.yaml b/modules/ai-agents/examples/best-practices/response-formatting/success-error-envelope.yaml deleted file mode 100644 index e4270490..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/success-error-envelope.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Test: Wrap in success/error structure -# Source: best-practices.adoc - Response formatting section -# Expected: Consistent envelope with success boolean and timestamp - -input: - sequence: - inputs: - - generate: - count: 1 - interval: "" - mapping: 'root = {"id": "123", "name": "Alice"}' - - generate: - count: 1 - interval: "" - mapping: 'root = {"trigger_error": true}' - -pipeline: - processors: - - try: - - mutation: | - root = if this.exists("trigger_error") { - throw("Simulated error for testing") - } else { - this - } - - # tag::success-envelope[] - # Success response - - mutation: | - root.success = true - root.data = this - root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") - # end::success-envelope[] - - - catch: - # tag::error-envelope[] - # Error response (in catch block) - - mutation: | - root.success = false - root.error = {"message": error(), "code": "PROCESSING_ERROR"} - root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") - # end::error-envelope[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/response-formatting/transform-arrays.yaml b/modules/ai-agents/examples/best-practices/response-formatting/transform-arrays.yaml deleted file mode 100644 index db166370..00000000 --- a/modules/ai-agents/examples/best-practices/response-formatting/transform-arrays.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Test: Transform arrays -# Source: best-practices.adoc - Response formatting section -# Expected: Maps array items to new structure with count - -input: - generate: - count: 1 - interval: "" - mapping: | - root = { - "items": [ - {"id": "prod_1", "name": "Widget", "price": "19.99", "category": "tools"}, - {"id": "prod_2", "name": "Gadget", "price": "29.99", "category": "electronics"} - ] - } - -pipeline: - processors: - # tag::transform-arrays[] - - mutation: | - root.total_items = this.items.length() - root.items = this.items.map_each(item -> { - "id": item.id, - "name": item.name, - "price": item.price.number() - }) - # end::transform-arrays[] - -output: - stdout: {} diff --git a/modules/ai-agents/examples/best-practices/tool-implementation/component-type.yaml b/modules/ai-agents/examples/best-practices/tool-implementation/component-type.yaml deleted file mode 100644 index 8a7f385c..00000000 --- a/modules/ai-agents/examples/best-practices/tool-implementation/component-type.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Test: Correct component type structure for MCP tools -# Validates: Single component type without wrappers - -# tag::correct-component[] -label: read-events - -redpanda: - topics: [ "events" ] - seed_brokers: [ "${REDPANDA_BROKERS}" ] - consumer_group: "mcp-reader" - -meta: - mcp: - enabled: true - description: "Read events from Redpanda topic" - properties: - - name: count - type: number - description: "Number of messages to read" - required: false -# end::correct-component[] - -# tag::wrong-pipeline[] -# Wrong: This is a pipeline, not an MCP tool -# label: my-tool -# input: -# redpanda: -# topics: [ "events" ] -# processors: -# - mutation: | -# root.processed = true -# output: -# redpanda: -# topic: "processed-events" -# end::wrong-pipeline[] - -# tag::wrong-wrapper[] -# Wrong: Unnecessary wrapper -# label: my-input -# input: -# redpanda: -# topics: [ "events" ] -# end::wrong-wrapper[] diff --git a/modules/ai-agents/examples/best-practices/tool-implementation/secrets-handling.yaml b/modules/ai-agents/examples/best-practices/tool-implementation/secrets-handling.yaml deleted file mode 100644 index 21f5b907..00000000 --- a/modules/ai-agents/examples/best-practices/tool-implementation/secrets-handling.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Test: Proper secrets handling in MCP tools -# Validates: Environment variables used for secrets, not hardcoded values - -label: fetch-api-data - -processors: - # tag::correct-vs-wrong[] - # Correct: Secret from environment variable - - http: - url: "https://httpbin.org/get" - verb: GET - headers: - Authorization: "Bearer ${API_TOKEN}" - timeout: "10s" - - # Wrong: Hardcoded - # - http: - # url: "https://api.example.com" - # headers: - # Authorization: "Bearer sk-1234567890abcdef" - # end::correct-vs-wrong[] - - - mutation: | - root = { - "status": "success", - "data_fetched": true - } - -meta: - mcp: - enabled: true - description: "Fetch data from external API using secure credentials" - properties: - - name: query - type: string - description: "Query parameter" - required: false diff --git a/modules/ai-agents/examples/best-practices/tool-implementation/single-responsibility.yaml b/modules/ai-agents/examples/best-practices/tool-implementation/single-responsibility.yaml deleted file mode 100644 index 8d4420f0..00000000 --- a/modules/ai-agents/examples/best-practices/tool-implementation/single-responsibility.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Test: Single responsibility principle for MCP tools -# Validates: Each tool should do one thing well - -# tag::good-user-profile[] -label: get-user-profile -# end::good-user-profile[] - -processors: - - mutation: | - root = { - "user_id": this.user_id, - "name": "John Doe", - "email": "john@example.com", - "created_at": "2024-01-15" - } - -meta: - mcp: - enabled: true - description: "Fetch user profile data by user ID" - properties: - - name: user_id - type: string - description: "User ID to fetch profile for" - required: true - -# tag::good-user-settings[] -# Another good example (separate file): -# label: update-user-settings -# Updates user settings only - single responsibility -# end::good-user-settings[] - -# tag::bad-manage-user[] -# Bad example (DO NOT USE): -# label: manage-user -# Fetches profile AND updates settings AND deletes user -# Too broad - split into focused tools -# end::bad-manage-user[] diff --git a/modules/ai-agents/examples/o11y/metrics.yaml b/modules/ai-agents/examples/o11y/metrics.yaml deleted file mode 100644 index 709eb7b6..00000000 --- a/modules/ai-agents/examples/o11y/metrics.yaml +++ /dev/null @@ -1 +0,0 @@ -prometheus: {} diff --git a/modules/ai-agents/examples/o11y/tracer.yaml b/modules/ai-agents/examples/o11y/tracer.yaml deleted file mode 100644 index aed02bf4..00000000 --- a/modules/ai-agents/examples/o11y/tracer.yaml +++ /dev/null @@ -1,4 +0,0 @@ -open_telemetry_collector: - service: rpcn-mcp - grpc: [] - http: [] diff --git a/modules/ai-agents/examples/resources/inputs/event-workflow.yaml b/modules/ai-agents/examples/resources/inputs/event-workflow.yaml deleted file mode 100644 index 0f252b82..00000000 --- a/modules/ai-agents/examples/resources/inputs/event-workflow.yaml +++ /dev/null @@ -1,11 +0,0 @@ -label: event_triggered_workflow -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "order-events" ] - consumer_group: "workflow-trigger" - -meta: - tags: [ workflows, events, example ] - mcp: - enabled: true - description: "Consume order events to trigger workflows" diff --git a/modules/ai-agents/examples/resources/inputs/generate-input.yaml b/modules/ai-agents/examples/resources/inputs/generate-input.yaml deleted file mode 100644 index fc7a66e1..00000000 --- a/modules/ai-agents/examples/resources/inputs/generate-input.yaml +++ /dev/null @@ -1,9 +0,0 @@ -generate: - interval: 1s - count: 0 - mapping: | - root.id = uuid_v4() - root.timestamp = now() - root.user_id = random_int(min: 1000, max: 9999) - root.event_type = ["login", "purchase", "logout"].index(random_int(max: 2)) - root.amount = if this.event_type == "purchase" { random_int(min: 10, max: 500) } diff --git a/modules/ai-agents/examples/resources/inputs/redpanda-consume.yaml b/modules/ai-agents/examples/resources/inputs/redpanda-consume.yaml deleted file mode 100644 index 3db88100..00000000 --- a/modules/ai-agents/examples/resources/inputs/redpanda-consume.yaml +++ /dev/null @@ -1,10 +0,0 @@ -label: get_recent_events -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "user-events" ] - consumer_group: "mcp-event-reader" -meta: - tags: [ consuming, events, example ] - mcp: - enabled: true - description: "Retrieve recent events from a Redpanda topic. Use the count property to limit results." diff --git a/modules/ai-agents/examples/resources/inputs/stream-analytics.yaml b/modules/ai-agents/examples/resources/inputs/stream-analytics.yaml deleted file mode 100644 index 5a4d28eb..00000000 --- a/modules/ai-agents/examples/resources/inputs/stream-analytics.yaml +++ /dev/null @@ -1,11 +0,0 @@ -label: stream_analytics -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "click-events" ] - consumer_group: "mcp-analytics" - -meta: - tags: [ analytics, streaming, example ] - mcp: - enabled: true - description: "Consume click stream data for analytics processing" diff --git a/modules/ai-agents/examples/resources/outputs/redpanda-output-with-processors.yaml b/modules/ai-agents/examples/resources/outputs/redpanda-output-with-processors.yaml deleted file mode 100644 index 5f3796f8..00000000 --- a/modules/ai-agents/examples/resources/outputs/redpanda-output-with-processors.yaml +++ /dev/null @@ -1,27 +0,0 @@ -label: summarize_and_store -processors: - - label: generate_summary - openai_chat_completion: - api_key: "${OPENAI_API_KEY}" - model: "gpt-4" - prompt: "Summarize this document in 2-3 sentences: ${! json(\"content\") }" - - label: prepare_record - mutation: | - root = { - "original": this.content, - "summary": this.choices.0.message.content, - "processed_at": now().ts_format("2006-01-02T15:04:05Z") - } -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topic: "summaries" -meta: - tags: [ publishing, llm, example ] - mcp: - enabled: true - description: "Summarize a document using an LLM and store the result in Redpanda" - properties: - - name: content - type: string - description: "Document content to summarize" - required: true diff --git a/modules/ai-agents/examples/resources/outputs/redpanda-publish.yaml b/modules/ai-agents/examples/resources/outputs/redpanda-publish.yaml deleted file mode 100644 index 412ec019..00000000 --- a/modules/ai-agents/examples/resources/outputs/redpanda-publish.yaml +++ /dev/null @@ -1,26 +0,0 @@ -label: publish_order_events -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topic: "order-events" - key: "${! this.order_id }" - partitioner: "manual" - partition: "${! this.customer_id.hash(algorithm:\"xxhash64\") % 8 }" - max_in_flight: 10 -meta: - tags: [ publishing, orders, example ] - mcp: - enabled: true - description: "Publish order events to Redpanda for downstream processing" - properties: - - name: order_id - type: string - description: "Unique order identifier" - required: true - - name: customer_id - type: string - description: "Customer ID for partitioning" - required: true - - name: order_data - type: string - description: "Order details as JSON string (items, total, etc.)" - required: true diff --git a/modules/ai-agents/examples/resources/processors/customer-enrichment.yaml b/modules/ai-agents/examples/resources/processors/customer-enrichment.yaml deleted file mode 100644 index 89886a8b..00000000 --- a/modules/ai-agents/examples/resources/processors/customer-enrichment.yaml +++ /dev/null @@ -1,47 +0,0 @@ -label: customer_enrichment -processors: - - label: fetch_customer_base - branch: - processors: - - sql_select: - driver: "postgres" - dsn: "${POSTGRES_DSN}" - table: "customers" - where: "customer_id = ?" - args_mapping: 'root = [this.customer_id]' - result_map: 'root.customers = this' - - - label: enrich_with_orders - branch: - processors: - - sql_select: - driver: "postgres" - dsn: "${POSTGRES_DSN}" - table: "orders" - where: "customer_id = ? AND created_at >= NOW() - INTERVAL ''30 days''" - args_mapping: 'root = [this.customer_id]' - result_map: 'root.orders = this' - - - label: combine_data - mutation: | - let order_totals = this.orders.map_each(o -> o.total) - root = { - "customer": this.customers.index(0), - "recent_orders": this.orders, - "metrics": { - "total_orders": this.orders.length(), - "total_spent": $order_totals.sum(), - "avg_order_value": if $order_totals.length() > 0 { $order_totals.sum() / $order_totals.length() } else { 0 } - } - } - -meta: - tags: [ example ] - mcp: - enabled: true - description: "Get comprehensive customer profile with recent order history and metrics" - properties: - - name: customer_id - type: string - description: "Customer ID to analyze" - required: true diff --git a/modules/ai-agents/examples/resources/processors/database-query.yaml b/modules/ai-agents/examples/resources/processors/database-query.yaml deleted file mode 100644 index 3e04e5e6..00000000 --- a/modules/ai-agents/examples/resources/processors/database-query.yaml +++ /dev/null @@ -1,41 +0,0 @@ -label: user_orders -processors: - - label: prepare_parameters - mutation: | - meta user_id = this.user_id.string() - meta limit = this.limit.number().catch(10) - - label: query_database - sql_select: - driver: "postgres" - dsn: "${DATABASE_URL}" - table: "orders" - columns: ["id", "total", "status", "created_at"] - where: "user_id = ? AND created_at > NOW() - INTERVAL '30 days'" - suffix: "ORDER BY created_at DESC LIMIT ?" - args_mapping: root = [@user_id, @limit] - - label: format_response - mutation: | - root = { - "user_id": @user_id, - "orders": this, - "total_count": this.length(), - "metadata": { - "source": "PostgreSQL", - "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") - } - } - -meta: - tags: [ database, orders, example ] - mcp: - enabled: true - description: "Get recent orders for a user" - properties: - - name: user_id - type: string - description: "User ID to fetch orders for" - required: true - - name: limit - type: number - description: "Maximum number of orders to return (default: 10)" - required: false diff --git a/modules/ai-agents/examples/resources/processors/observable-tool.yaml b/modules/ai-agents/examples/resources/processors/observable-tool.yaml deleted file mode 100644 index e2596a79..00000000 --- a/modules/ai-agents/examples/resources/processors/observable-tool.yaml +++ /dev/null @@ -1,60 +0,0 @@ -label: observable_tool -processors: - - label: init_tracing - mutation: | - # Generate correlation ID for request tracing - meta req_id = uuid_v7() - meta start_time = now() - - # Log request start with structured data - root.trace = { - "request_id": @req_id, - "timestamp": @start_time.ts_format("2006-01-02T15:04:05.000Z"), - "tool": "observable_tool", - "version": "1.0.0" - } - - - label: log_request_start - log: - message: "MCP tool request started" - fields: - request_id: "${! @req_id }" - tool_name: "observable_tool" - input_params: "${! this.without(\"trace\") }" - user_agent: "${! meta(\"User-Agent\").catch(\"unknown\") }" - level: "INFO" - - - label: finalize_response - mutation: | - # Calculate total execution time - meta duration = (now().ts_unix_nano() - @start_time.ts_unix_nano()) / 1000000 - - # Add trace information to response - root.metadata = { - "request_id": @req_id, - "execution_time_ms": @duration, - "timestamp": now().ts_format("2006-01-02T15:04:05.000Z"), - "tool": "observable_tool", - "success": !this.exists("error") - } - - - label: log_completion - log: - message: "MCP tool request completed" - fields: - request_id: "${! @req_id }" - duration_ms: "${! this.metadata.execution_time_ms }" - success: "${! this.metadata.success }" - result_size: "${! content().length() }" - level: "INFO" - -meta: - tags: [ example ] - mcp: - enabled: true - description: "Example tool with comprehensive observability and error handling" - properties: - - name: user_id - type: string - description: "User ID to fetch data for" - required: true diff --git a/modules/ai-agents/examples/resources/processors/order-workflow.yaml b/modules/ai-agents/examples/resources/processors/order-workflow.yaml deleted file mode 100644 index c7562f26..00000000 --- a/modules/ai-agents/examples/resources/processors/order-workflow.yaml +++ /dev/null @@ -1,108 +0,0 @@ -label: order_workflow -processors: - - label: validate_order - mutation: | - # Validation logic - root = if this.total <= 0 { - throw("Invalid order total") - } else { this } - - - label: mock_inventory_check - mutation: | - # Mock inventory data for testing - let inventory = { - "widget-001": {"quantity": 100, "name": "Standard Widget"}, - "widget-premium": {"quantity": 25, "name": "Premium Widget"}, - "widget-limited": {"quantity": 2, "name": "Limited Edition Widget"} - } - - let product = $inventory.get(this.product_id) - root = if $product == null { - throw("Product not found: " + this.product_id) - } else if $product.quantity < this.quantity { - throw("Insufficient inventory. Available: " + $product.quantity.string()) - } else { - this.merge({ - "inventory_check": "passed", - "available_quantity": $product.quantity, - "product_name": $product.name - }) - } - - - label: route_by_priority - switch: - - check: 'this.total > 1000' - processors: - - label: mock_high_value_processing - mutation: | - # Mock premium processing - root = this.merge({ - "processing_tier": "premium", - "processing_time_estimate": "2-4 hours", - "assigned_rep": "premium-team@company.com", - "priority_score": 95 - }) - - - check: 'this.customer_tier == "vip"' - processors: - - label: mock_vip_processing - mutation: | - # Mock VIP processing - root = this.merge({ - "processing_tier": "vip", - "processing_time_estimate": "1-2 hours", - "assigned_rep": "vip-team@company.com", - "priority_score": 90, - "perks": ["expedited_shipping", "white_glove_service"] - }) - - - processors: - - label: mock_standard_processing - mutation: | - # Mock standard processing - root = this.merge({ - "processing_tier": "standard", - "processing_time_estimate": "24-48 hours", - "assigned_rep": "support@company.com", - "priority_score": 50 - }) - - - label: finalize_order - mutation: | - # Add final processing metadata - # Calculate estimated fulfillment by parsing processing time - let max_hours = this.processing_time_estimate.split("-").index(1).split(" ").index(0).number() - - root = this.merge({ - "order_status": "processed", - "processed_at": now().ts_format("2006-01-02T15:04:05.000Z"), - "estimated_fulfillment": "TBD - calculated based on processing tier", - "processing_time_hours": $max_hours - }) - -meta: - tags: [ example ] - mcp: - enabled: true - description: "Process orders with validation, inventory check, and tiered routing (with mocks for testing)" - properties: - - name: order_id - type: string - description: "Unique order identifier" - required: true - - name: product_id - type: string - description: "Product ID (try: widget-001, widget-premium, widget-limited)" - required: true - - name: quantity - type: number - description: "Quantity to order" - required: true - - name: total - type: number - description: "Order total in dollars" - required: true - - name: customer_tier - type: string - description: "Customer tier (optional: vip, standard)" - required: false diff --git a/modules/ai-agents/examples/resources/processors/weather-api.yaml b/modules/ai-agents/examples/resources/processors/weather-api.yaml deleted file mode 100644 index 33b802a1..00000000 --- a/modules/ai-agents/examples/resources/processors/weather-api.yaml +++ /dev/null @@ -1,38 +0,0 @@ -label: fetch-weather -processors: - - label: prepare_parameters - mutation: | - meta city_name = this.city_name - - label: fetch_weather - http: - url: 'https://wttr.in/${! @city_name }?format=j1' - verb: GET - headers: - Accept: "application/json" - User-Agent: "redpanda-mcp-server/1.0" - - label: format_response - mutation: | - root = { - "city": @city_name, - "temperature": this.current_condition.0.temp_C.number(), - "feels_like": this.current_condition.0.FeelsLikeC.number(), - "humidity": this.current_condition.0.humidity.number(), - "pressure": this.current_condition.0.pressure.number(), - "description": this.current_condition.0.weatherDesc.0.value, - "wind_speed": this.current_condition.0.windspeedKmph.number(), - "metadata": { - "source": "wttr.in", - "fetched_at": now().ts_format("2006-01-02T15:04:05.000Z") - } - } - -meta: - tags: [ example, weather, api ] - mcp: - enabled: true - description: "Fetch current weather information for a specified city" - properties: - - name: city_name - type: string - description: "Name of the city to get weather information for" - required: true diff --git a/modules/ai-agents/examples/resources/processors/weather-service.yaml b/modules/ai-agents/examples/resources/processors/weather-service.yaml deleted file mode 100644 index 3ebe920f..00000000 --- a/modules/ai-agents/examples/resources/processors/weather-service.yaml +++ /dev/null @@ -1,60 +0,0 @@ -label: weather-service -processors: - - label: validate_inputs # <1> - mutation: | - # Validate and sanitize city input - meta city = this.city.string(). - re_replace_all("[^a-zA-Z\\s\\-]", ""). - trim() - - # Check for empty input - root = if @city == "" { - throw("City name cannot be empty") - } else { "" } - - - label: fetch_weather_data # <2> - try: - - http: # <3> - url: "https://wttr.in/${! @city }?format=j1" - verb: GET - headers: - User-Agent: "redpanda-connect-mcp/1.0" - timeout: "10s" - - mutation: | # <4> - root = { - "city": @city, - "temperature_c": this.current_condition.0.temp_C.number(), - "temperature_f": this.current_condition.0.temp_F.number(), - "feels_like_c": this.current_condition.0.FeelsLikeC.number(), - "humidity": this.current_condition.0.humidity.number(), - "description": this.current_condition.0.weatherDesc.0.value, - "wind_speed_kmh": this.current_condition.0.windspeedKmph.number(), - "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") - } - - log: - message: "Weather data fetched for city: ${! @city }" - level: "INFO" - - - label: handle_weather_errors # <5> - catch: - - mutation: | - root = { - "error": "Failed to fetch weather data", - "city": @city, - "details": error(), - "timestamp": now().format_timestamp("2006-01-02T15:04:05Z07:00") - } - - log: - message: "Weather API error for city ${! @city }: ${! error() }" - level: "ERROR" - -meta: # <6> - tags: [ weather, example ] - mcp: - enabled: true - description: "Get current weather conditions for any city worldwide" - properties: - - name: city - type: string - description: "Name of the city (such as 'San Francisco', 'London')" - required: true diff --git a/modules/ai-agents/examples/test-all.sh b/modules/ai-agents/examples/test-all.sh deleted file mode 100755 index f03760c4..00000000 --- a/modules/ai-agents/examples/test-all.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env bash -# -# Master test script for all Redpanda Connect MCP examples -# -# This script tests: -# 1. MCP tool definitions using `rpk connect mcp-server lint` -# 2. Full pipelines using `rpk connect run` -# 3. Config snippets using `rpk connect lint` -# -# Usage: -# ./test-all.sh # Run all tests -# ./test-all.sh --mcp-only # Only test MCP tools -# ./test-all.sh --pipelines # Only test pipelines -# ./test-all.sh --snippets # Only test snippets - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -CYAN='\033[0;36m' -NC='\033[0m' - -# Get script directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -# Counters -TOTAL_MCP=0 -PASSED_MCP=0 -TOTAL_PIPELINES=0 -PASSED_PIPELINES=0 -TOTAL_SNIPPETS=0 -PASSED_SNIPPETS=0 - -echo "🧪 Redpanda Connect Examples - Complete Test Suite" -echo "==================================================" -echo "" - -# Parse arguments -RUN_MCP=true -RUN_PIPELINES=true -RUN_SNIPPETS=true - -if [[ $# -gt 0 ]]; then - case "$1" in - --mcp-only) - RUN_PIPELINES=false - RUN_SNIPPETS=false - ;; - --pipelines) - RUN_MCP=false - RUN_SNIPPETS=false - ;; - --snippets) - RUN_MCP=false - RUN_PIPELINES=false - ;; - esac -fi - -# ============================================================================ -# SECTION 1: MCP Tool Definitions -# These are complete MCP tools that should be tested with mcp-server lint -# rpk connect mcp-server lint works on directories, so we test each directory -# ============================================================================ - -if $RUN_MCP; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo -e "📦 ${CYAN}SECTION 1: MCP Tool Definitions${NC}" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "" - - # Directories containing MCP tool definitions - MCP_DIRS=( - "resources/inputs" - "resources/outputs" - "resources/processors" - "best-practices/mcp-metadata" - "best-practices/production-workflows" - "best-practices/tool-implementation" - ) - - for dir in "${MCP_DIRS[@]}"; do - if [[ -d "$dir" ]]; then - file_count=$(find "$dir" -name "*.yaml" | wc -l | tr -d ' ') - TOTAL_MCP=$((TOTAL_MCP + file_count)) - - echo -n -e "${BLUE}📁 $dir${NC} ($file_count files)... " - - if output=$(rpk connect mcp-server lint --skip-env-var-check "$dir" 2>&1); then - echo -e "${GREEN}✓ PASSED${NC}" - PASSED_MCP=$((PASSED_MCP + file_count)) - else - echo -e "${RED}✗ FAILED${NC}" - echo "$output" | sed 's/^/ /' | head -10 - echo "" - fi - fi - done -fi - -# ============================================================================ -# SECTION 2: Full Pipelines (runnable examples) -# These have input/pipeline/output and can be executed with rpk connect run -# ============================================================================ - -if $RUN_PIPELINES; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo -e "🔄 ${CYAN}SECTION 2: Full Pipelines${NC}" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "" - - PIPELINE_DIRS=( - "best-practices/input-validation" - "best-practices/response-formatting" - "best-practices/error-handling" - ) - - for dir in "${PIPELINE_DIRS[@]}"; do - if [[ -d "$dir" ]]; then - echo -e "${BLUE}📁 Testing: $dir${NC}" - - for file in "$dir"/*.yaml; do - if [[ -f "$file" ]]; then - TOTAL_PIPELINES=$((TOTAL_PIPELINES + 1)) - filename=$(basename "$file") - echo -n " $filename " - - # First lint - echo -n "[lint: " - if rpk connect lint --skip-env-var-check "$file" > /dev/null 2>&1; then - echo -n -e "${GREEN}✓${NC}] " - else - echo -e "${RED}✗${NC}]" - rpk connect lint --skip-env-var-check "$file" 2>&1 | sed 's/^/ /' - continue - fi - - # Then run - echo -n "[run: " - if timeout 30s rpk connect run "$file" > /dev/null 2>&1; then - echo -e "${GREEN}✓${NC}]" - PASSED_PIPELINES=$((PASSED_PIPELINES + 1)) - else - echo -e "${RED}✗${NC}]" - fi - fi - done - echo "" - fi - done -fi - -# ============================================================================ -# SECTION 3: Config Snippets -# These are partial configs or examples that may not run but should lint -# ============================================================================ - -if $RUN_SNIPPETS; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo -e "📝 ${CYAN}SECTION 3: Config Snippets${NC}" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "" - - SNIPPET_DIRS=( - "best-practices/debugging" - "best-practices/error-handling-snippets" - "best-practices/config-examples" - ) - - for dir in "${SNIPPET_DIRS[@]}"; do - if [[ -d "$dir" ]]; then - file_count=$(find "$dir" -name "*.yaml" | wc -l | tr -d ' ') - TOTAL_SNIPPETS=$((TOTAL_SNIPPETS + file_count)) - - echo -n -e "${BLUE}📁 $dir${NC} ($file_count files)... " - - # Try MCP lint on directory (these should be MCP tools) - if output=$(rpk connect mcp-server lint --skip-env-var-check "$dir" 2>&1); then - echo -e "${GREEN}✓ PASSED${NC} (mcp)" - PASSED_SNIPPETS=$((PASSED_SNIPPETS + file_count)) - else - echo -e "${RED}✗ FAILED${NC}" - echo "$output" | sed 's/^/ /' | head -10 - fi - fi - done - - # Handle o11y separately (these are just config fragments) - if [[ -d "o11y" ]]; then - echo "" - echo -e "${BLUE}📁 o11y (config fragments)${NC}" - - for file in o11y/*.yaml; do - if [[ -f "$file" ]]; then - TOTAL_SNIPPETS=$((TOTAL_SNIPPETS + 1)) - filename=$(basename "$file") - echo -n " $filename... " - - # These are just YAML fragments - check valid YAML - if python3 -c "import yaml; yaml.safe_load(open('$file'))" 2>/dev/null; then - echo -e "${GREEN}✓${NC} (yaml)" - PASSED_SNIPPETS=$((PASSED_SNIPPETS + 1)) - else - echo -e "${RED}✗${NC} (invalid yaml)" - fi - fi - done - echo "" - fi -fi - -# ============================================================================ -# SUMMARY -# ============================================================================ - -echo "" -echo "==================================================" -echo "📊 Test Summary" -echo "==================================================" - -TOTAL=$((TOTAL_MCP + TOTAL_PIPELINES + TOTAL_SNIPPETS)) -PASSED=$((PASSED_MCP + PASSED_PIPELINES + PASSED_SNIPPETS)) -FAILED=$((TOTAL - PASSED)) - -if $RUN_MCP; then - echo -e "MCP Tools: ${PASSED_MCP}/${TOTAL_MCP} passed" -fi -if $RUN_PIPELINES; then - echo -e "Pipelines: ${PASSED_PIPELINES}/${TOTAL_PIPELINES} passed" -fi -if $RUN_SNIPPETS; then - echo -e "Snippets: ${PASSED_SNIPPETS}/${TOTAL_SNIPPETS} passed" -fi -echo "──────────────────────────────────────────────────" -echo -e "Total: ${PASSED}/${TOTAL} passed" -echo "" - -if [[ $FAILED -gt 0 ]]; then - echo -e "${RED}❌ Some tests failed ($FAILED failures)${NC}" - exit 1 -else - echo -e "${GREEN}✅ All tests passed!${NC}" - exit 0 -fi diff --git a/modules/ai-agents/examples/testing.adoc b/modules/ai-agents/examples/testing.adoc deleted file mode 100644 index ed904970..00000000 --- a/modules/ai-agents/examples/testing.adoc +++ /dev/null @@ -1,546 +0,0 @@ -= Test MCP Examples -:description: Automated testing strategies for Redpanda Connect MCP server examples and best practices. - -This document describes the automated testing strategies for Redpanda Connect MCP server examples and best practices documentation. - -== Directory structure - -[cols="1,2,1"] -|=== -|Directory |Purpose |Test approach - -|`resources/inputs/` -|MCP input tools (streaming sources) -|`rpk connect mcp-server lint` - -|`resources/outputs/` -|MCP output tools (data sinks) -|`rpk connect mcp-server lint` - -|`resources/processors/` -|MCP processor tools (transformations, API calls) -|`rpk connect mcp-server lint` - -|`best-practices/mcp-metadata/` -|MCP metadata examples (descriptions, properties) -|`rpk connect mcp-server lint` - -|`best-practices/production-workflows/` -|Production workflow patterns -|`rpk connect mcp-server lint` - -|`best-practices/tool-implementation/` -|Tool design patterns -|`rpk connect mcp-server lint` - -|`best-practices/input-validation/` -|Full pipelines demonstrating input validation -|`rpk connect run` - -|`best-practices/response-formatting/` -|Full pipelines demonstrating response formatting -|`rpk connect run` - -|`best-practices/error-handling/` -|Full pipelines demonstrating error handling -|`rpk connect run` - -|`best-practices/debugging/` -|Debug and troubleshooting patterns -|`rpk connect mcp-server lint` - -|`best-practices/error-handling-snippets/` -|Error handling snippets -|`rpk connect mcp-server lint` - -|`best-practices/config-examples/` -|Configuration examples -|`rpk connect mcp-server lint` - -|`o11y/` -|Observability config fragments -|YAML validation -|=== - -== Running all tests - -Run the comprehensive test suite: - -[,bash] ----- -cd modules/ai-agents/examples -./test-all.sh ----- - -This tests: - -. **MCP Tools (25 files)**: Using `rpk connect mcp-server lint` on tool directories -. **Full Pipelines (28 files)**: Using `rpk connect lint` and `rpk connect run` -. **Config Snippets (7 files)**: Using `rpk connect mcp-server lint` or YAML validation - -=== Selective testing - -[,bash] ----- -./test-all.sh --mcp-only # Only MCP tool definitions -./test-all.sh --pipelines # Only full pipelines -./test-all.sh --snippets # Only config snippets ----- - -== MCP tool examples - -All MCP tool examples are automatically tested to ensure: - -. YAML syntax and structure are correct -. MCP metadata is complete and valid -. Component schemas match Redpanda Connect specifications -. Environment variables are handled gracefully (both `${secrets.X}` and `${X}` syntax) - -== Testing approaches - -=== Configuration linting - -Validate MCP tool configurations using `rpk connect lint`: - -[,bash] ----- -# Lint a single MCP tool -rpk connect lint resources/processors/weather-api.yaml - -# Lint all processor examples -rpk connect lint resources/processors/*.yaml - -# Lint with environment variable checking skipped (recommended for MCP) -rpk connect lint --skip-env-var-check resources/processors/*.yaml ----- - -This checks for common issues such as: - -* YAML syntax errors -* Unknown component types -* Invalid field names -* Type mismatches -* Missing required fields - -=== MCP metadata validation - -The test script validates MCP-specific metadata for all tool examples: - -[,bash] ----- -# Run all tests (includes linting + MCP validation) -./test-all.sh - -# Test only MCP tools -./test-all.sh --mcp-only ----- - -MCP metadata validation checks: - -* Presence of `meta.mcp` section -* `enabled: true` is set -* `description` field exists and is non-empty -* `properties` are properly structured (if present) - -=== Unit testing limitations - -[IMPORTANT] -==== -MCP tool examples are standalone component definitions, not full pipelines with `input:`, `pipeline:`, `output:` sections. This means they cannot use inline `tests:` sections like cookbook examples do. - -The `rpk connect test` command requires full pipeline structure with paths like `/pipeline/processors/0`, which don't exist in MCP tool definitions. -==== - -For testing MCP tools: - -Linting is the primary validation:: Ensures syntax and schema correctness. -MCP metadata validation:: Verifies tool has proper description and properties. -Manual testing:: Use `rpk connect mcp-server` to start a server and test tools with an MCP client. - -== MCP tool structure - -MCP tools are structured as standalone components: - -[,yaml] ----- -label: fetch-weather -processors: - - label: prepare_parameters - mutation: | - meta city_name = this.city_name - - - label: fetch_weather - http: - url: 'https://wttr.in/${! @city_name }?format=j1' - verb: GET - - - label: format_response - mutation: | - root = { - "city": @city_name, - "temperature": this.current_condition.0.temp_C.number() - } - -meta: - mcp: - enabled: true - description: "Fetch current weather information for a specified city" - properties: - - name: city_name - type: string - description: "Name of the city to get weather information for" - required: true ----- - -This differs from full pipeline configurations used in cookbooks: - -[,yaml] ----- -# Cookbook style (full pipeline) -input: - generate: { ... } - -pipeline: - processors: [ ... ] - -output: - stdout: {} - -tests: [ ... ] # Can use inline tests ----- - -== Test script usage - -The `test-all.sh` script provides automated validation: - -[,bash] ----- -# Test everything (MCP tools, pipelines, snippets) -./test-all.sh - -# Test only MCP tool definitions -./test-all.sh --mcp-only - -# Test only full pipelines -./test-all.sh --pipelines - -# Test only config snippets -./test-all.sh --snippets ----- - -The script provides color-coded output: - -[,console] ----- -🧪 Redpanda Connect MCP Examples Test Suite -============================================ - -📄 Testing: resources/processors/weather-api.yaml (processor) - Linting weather-api.yaml... PASSED - Validating MCP metadata... PASSED - -============================================ -📊 Test Summary -============================================ -Total configs tested: 7 -Passed: 7 -Failed: 0 - -✅ All tests passed! ----- - -== Manual end-to-end testing - -For comprehensive validation, test MCP tools with an actual MCP server: - -[,bash] ----- -# Navigate to examples directory -cd modules/ai-agents/examples - -# Start MCP server with your tools -rpk connect mcp-server --address localhost:4195 - -# In another terminal, connect with an MCP client -# Example: Claude Code with mcp-remote -claude mcp add local -- npx mcp-remote http://localhost:4195/sse - -# Test your tool through the MCP client -# The tool should appear in the tools list and be invocable ----- - -This validates: - -* Tool loads correctly in MCP server -* MCP metadata is properly exposed -* Tool executes with provided parameters -* Responses are formatted correctly - -== GitHub Actions CI/CD - -Automated tests run on every push and pull request using GitHub Actions. - -The workflow includes two jobs: - -. `lint-and-test` tests all examples at once. -. `test-matrix` tests each component type in parallel for faster feedback. - -See `.github/workflows/test-mcp-examples.yaml` for the complete workflow. - -== Best practices examples - -The `best-practices/` directory contains executable examples that demonstrate patterns from the MCP development best practices documentation. Unlike MCP tool definitions, these are full pipelines that can be run directly with `rpk connect run`. - -=== Directory structure - -[cols="1,2"] -|=== -|Directory |Purpose - -|`best-practices/input-validation/` -|Input validation patterns (required fields, sanitization, ranges, enums) - -|`best-practices/response-formatting/` -|Response formatting patterns (timestamps, arrays, conditional fields, filtering) - -|`best-practices/error-handling/` -|Error handling patterns (try/catch, error types, logging, context preservation) -|=== - -=== Running best practices tests - -[,bash] ----- -# Navigate to examples directory -cd modules/ai-agents/examples - -# Test all pipelines (including best-practices) -./test-all.sh --pipelines ----- - -The test script: - -. Lints each configuration with `rpk connect lint` -. Runs each pipeline with `rpk connect run` -. Validates successful execution (exit code 0) -. Shows sample output for verification - -=== Example output - -[,console] ----- -🧪 Redpanda Connect Best Practices Test Suite -============================================== - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -📁 Category: input-validation -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -📄 Testing: input-validation/validate-required-field - Linting... PASSED - Running... PASSED - Output: {"city":"London"}... - -📄 Testing: input-validation/sanitize-string-input - Linting... PASSED - Running... PASSED - Output: {"city":"New York"}... - -============================================== -📊 Test Summary -============================================== -Total tests: 17 -Passed: 17 - -✅ All tests passed! ----- - -=== Best practices example structure - -Best practices examples use full pipeline structure with generated test data: - -[,yaml] ----- -# Test: Input validation pattern -# Expected: Valid input passes, invalid input returns error - -input: - sequence: - inputs: - - generate: - count: 1 - mapping: 'root = {"city": "London"}' # Valid input - - generate: - count: 1 - mapping: 'root = {"city": " "}' # Whitespace only - - generate: - count: 1 - mapping: 'root = {}' # Missing field - -pipeline: - processors: - - label: validate_input - mutation: | - let city = this.city.or("").trim() - root = if $city == "" { - {"error": "City name is required"} - } else { - {"city": $city} - } - -output: - stdout: {} ----- - -The `sequence` input runs multiple test cases through the same pipeline, allowing validation of both success and error paths. - -=== CI/CD for best practices - -Best practices tests run automatically via GitHub Actions as part of the unified MCP examples workflow when: - -* Any YAML file in `modules/ai-agents/examples/` is modified -* The test-all.sh script is updated - -See `.github/workflows/test-mcp-examples.yaml` for the workflow configuration. - -== Writing good examples - -=== Use descriptive tool names - -[,yaml] ----- -# Good -label: fetch-customer-orders - -# Bad -label: tool1 ----- - -=== Write clear MCP descriptions - -[,yaml] ----- -# Good -meta: - mcp: - description: "Fetch a customer's order history and calculate spending metrics over the last 30 days" - -# Bad -meta: - mcp: - description: "Get orders" ----- - -=== Document all properties - -[,yaml] ----- -# Good -properties: - - name: customer_id - type: string - description: "Unique identifier for the customer" - required: true - - name: days - type: number - description: "Number of days to look back (default: 30)" - required: false - -# Bad -properties: - - name: id - type: string - required: true ----- - -=== Use environment variables for secrets - -[,yaml] ----- -# For Cloud (Secrets Store) -dsn: "${secrets.POSTGRES_DSN}" - -# For self-managed Connect (environment variables) -dsn: "${POSTGRES_DSN}" ----- - -=== Tag your examples - -[,yaml] ----- -meta: - tags: [ example, weather, api ] # Helps organize and filter - mcp: - enabled: true ----- - -== Adding new examples - -When adding new MCP tool examples: - -. Choose the appropriate directory: -+ -[cols="1,2"] -|=== -|Directory |Purpose - -|`resources/processors/` -|Most MCP tools (data transformations, API calls) - -|`resources/inputs/` -|Streaming data sources - -|`resources/outputs/` -|Data sinks for batch operations - -|`resources/caches/` -|Caching components - -|`o11y/` -|Observability configurations (metrics, tracing) -|=== - -. Include complete MCP metadata: -+ -[,yaml] ----- -meta: - mcp: - enabled: true - description: "Clear, task-oriented description" - properties: - - name: param_name - type: string - description: "Parameter purpose and constraints" - required: true ----- - -. Lint your example: -+ -[,bash] ----- -rpk connect lint --skip-env-var-check resources/processors/my-tool.yaml ----- - -. Run automated tests: -+ -[,bash] ----- -cd modules/ai-agents/examples -./test-all.sh --mcp-only ----- - -. Test manually (recommended): -+ -[,bash] ----- -rpk connect mcp-server --address localhost:4195 -# Connect with MCP client and verify tool works end-to-end ----- - -. Commit your example: -+ -[,bash] ----- -git add modules/ai-agents/examples/resources/processors/my-tool.yaml -git commit -m "Add my-tool MCP example" ----- \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp-server/best-practices.adoc b/modules/ai-agents/pages/mcp-server/best-practices.adoc deleted file mode 100644 index 3cc24430..00000000 --- a/modules/ai-agents/pages/mcp-server/best-practices.adoc +++ /dev/null @@ -1,44 +0,0 @@ -= MCP Tool Design -:page-aliases: ai-agents:mcp-server/developer-guide.adoc -:description: Design effective MCP tool interfaces with clear names, descriptions, and input properties. -:page-topic-type: best-practices -:personas: ai_agent_developer -// Reader journey: "I want AI clients to discover and use my tools effectively" -// Learning objectives - what readers should be able to do after reading this page: -:learning-objective-1: Write tool names and descriptions that help AI clients select the right tool -:learning-objective-2: Define input properties with appropriate types and constraints -:learning-objective-3: Design focused tools that complete quickly - -After xref:ai-agents:mcp-server/create-tool.adoc[creating your first tool], apply these design guidelines so AI clients can discover, understand, and invoke your tools correctly. - -After reading this page, you will be able to: - -* [ ] {learning-objective-1} -* [ ] {learning-objective-2} -* [ ] {learning-objective-3} - -[[tool-discovery]] -== Tool discovery - -include::ai-agents:partial$mcp/best-practices/mcp-metadata-design.adoc[] - -[[tool-execution]] -== Tool execution - -include::ai-agents:partial$mcp/best-practices/tool-implementation-practices.adoc[] - -== Complete example - -This example combines all the best practices: - -[source,yaml] ----- -include::example$best-practices/mcp-metadata/search-customer-orders.yaml[tags=complete-example] ----- - -== Next steps - -* xref:ai-agents:mcp-server/create-tool.adoc#secrets[Use secrets]: Store credentials securely -* xref:ai-agents:mcp-server/create-tool.adoc#tool-filtering[Use tags]: Organize and filter tools -* xref:ai-agents:mcp-server/tool-patterns.adoc[]: Find reusable patterns including validation, error handling, and response formatting -* xref:ai-agents:mcp-server/troubleshooting.adoc[]: Diagnose common issues diff --git a/modules/ai-agents/pages/mcp-server/concepts.adoc b/modules/ai-agents/pages/mcp-server/concepts.adoc deleted file mode 100644 index 23296167..00000000 --- a/modules/ai-agents/pages/mcp-server/concepts.adoc +++ /dev/null @@ -1,61 +0,0 @@ -= MCP Tool Execution and Components -:description: Understand the MCP execution model and choose the right component type for your tools. -:page-topic-type: concepts -:personas: ai_agent_developer, streaming_developer -// Reader journey: "I want to understand how it works" -// Learning objectives - what readers should know after reading this page: -:learning-objective-1: Describe the request/response execution model -:learning-objective-2: Explain how Redpanda Connect components map to MCP tools -:learning-objective-3: Choose the right component type (processor, input, output, cache) for a use case - -This page explains how MCP tools execute and how to choose the right component type for your use case. - -After reading this page, you will be able to: - -* [ ] {learning-objective-1} -* [ ] {learning-objective-2} -* [ ] {learning-objective-3} - -// Execution model - single-sourced from partial -include::ai-agents:partial$mcp/concepts/execution-model.adoc[] - -// Component mapping - single-sourced from partial -include::ai-agents:partial$mcp/concepts/component-mapping.adoc[] - -[[component-selection]] -== Choose the right component type - -// Component selection guide - single-sourced from partial -include::ai-agents:partial$mcp/concepts/component-selection.adoc[] - -[[support-and-licensing]] -== Component support levels and licensing - -Redpanda Connect components have different support levels and licensing requirements. Understanding these helps you make informed decisions when building MCP tools. - -=== Support levels - -Components fall into two support categories: - -Certified components:: Officially supported by Redpanda. These components receive regular updates, bug fixes, and are tested for production use. Use certified components when reliability and support are priorities. - -Community components:: Contributed by the community. These may have varying levels of maintenance and testing. Review the component documentation and consider your risk tolerance before using in production. - -=== Enterprise licensing - -Some components require a Redpanda Enterprise license. These include certain cloud-provider integrations, advanced security features, and some enterprise data source connectors. - -If you use an Enterprise-licensed component without a valid license, the MCP server will not start, and you'll see an error indicating the missing license. - -The xref:components:about.adoc[component reference documentation] displays support level and license badges on each component page. - -For more information about licenses, see xref:get-started:licensing.adoc[]. - -== Next steps - -Continue your learning journey with these resources: - -* xref:ai-agents:mcp-server/create-tool.adoc[]: Create custom tools step by step -* xref:ai-agents:mcp-server/best-practices.adoc[]: Apply naming and design guidelines -* xref:ai-agents:mcp-server/tool-patterns.adoc[]: Find reusable patterns -* xref:ai-agents:mcp-server/troubleshooting.adoc[]: Diagnose common issues diff --git a/modules/ai-agents/pages/mcp-server/create-tool.adoc b/modules/ai-agents/pages/mcp-server/create-tool.adoc deleted file mode 100644 index defe29f5..00000000 --- a/modules/ai-agents/pages/mcp-server/create-tool.adoc +++ /dev/null @@ -1,519 +0,0 @@ -= Create an MCP Tool -:description: Create an MCP tool with the correct YAML structure, metadata, and parameter mapping. -:page-topic-type: how-to -:personas: ai_agent_developer, streaming_developer, data_engineer -// Reader journey: "I want to create a tool for my AI agent" -// Learning objectives - what readers can do after reading this page: -:learning-objective-1: Create a tool file with the correct structure and MCP metadata -:learning-objective-2: Map MCP parameters to component configuration fields using Bloblang -:learning-objective-3: Test tools locally before connecting AI clients - -After xref:ai-agents:mcp-server/quickstart.adoc[deploying your first MCP server], create custom tools that AI clients can discover and invoke. This guide walks you through the process using any Redpanda Connect component. - -After reading this page, you will be able to: - -* [ ] {learning-objective-1} -* [ ] {learning-objective-2} -* [ ] {learning-objective-3} - -== Prerequisites - -* At least version 4.66.1 of xref:install:rpk.adoc[Redpanda Connect installed] -* You can describe the MCP execution model (see xref:ai-agents:mcp-server/concepts.adoc#execution-model[The MCP execution model]) -* You have chosen the right component type for your use case (see xref:ai-agents:mcp-server/concepts.adoc#component-selection[Choose the right component type]) - -== Set up the project - -If you don't have an MCP server project yet, create one: - -[source,bash] ----- -rpk connect mcp-server init -cd ----- - -This command creates the following structure: - -[source,text] ----- -/ -├── resources/ - ├── caches/ # Cache components - ├── inputs/ # Input components - ├── outputs/ # Output components - └── processors/ # Processor components (most common) ----- - -The `resources/` directory holds your MCP tool files, organized by component type. - -== Create the tool file - -Create a YAML file in the appropriate directory. For example, to create a processor tool: - -[source,bash] ----- -touch resources/processors/.yaml ----- - -The file name can be anything, but use descriptive names that reflect the tool's purpose. - -Each YAML file must contain exactly one component. The directory in your project determines the component type: - -[cols="1,2", options="header"] -|=== -| Directory | Component type - -| `resources/processors/` -| Processor - -| `resources/inputs/` -| Input - -| `resources/outputs/` -| Output - -| `resources/caches/` -| Cache -|=== - -For example, a file in `resources/processors/` defines a processor component. Do not mix component types in the same file. - -[[anatomy]] -== Add the tool structure - -An MCP tool wraps a xref:components:about.adoc[Redpanda Connect component] and exposes it to AI clients. Each tool file has three parts: - -* **Label**: The tool name AI clients see -* **Component configuration**: A Redpanda Connect component (processor, input, output, or cache) -* **MCP metadata**: The tool's purpose and input parameters - -Here's an example using the xref:components:processors/sql_select.adoc[`sql_select` processor]: - -[source,yaml] ----- -label: lookup-customer # <1> - -sql_select: # <2> - driver: postgres - dsn: "${DATABASE_URL}" - table: customers - columns: ["id", "name", "email", "plan"] - where: id = ? - args_mapping: '[this.customer_id]' - -meta: # <3> - mcp: - enabled: true - description: "Look up a customer by ID and return their profile." - properties: - - name: customer_id - type: string - description: "The customer's unique identifier" - required: true ----- - -<1> The label becomes the tool name that AI clients see and invoke. -<2> The `sql_select` processor queries a PostgreSQL database. -<3> The MCP metadata tells AI clients what this tool does and what parameters it accepts. - -The following sections show how to structure tools for each component type. - -=== Label naming rules - -include::ai-agents:partial$mcp/create-tool/label-naming-rules.adoc[] - -=== Component types - -xref:components:processors/about.adoc[Processors] transform, filter, or enrich data. Use a `processors:` array with one or more processors: - -[source,yaml] ----- -label: enrich-order - -processors: - - http: - url: "https://api.example.com/lookup" - verb: GET - -meta: - mcp: - enabled: true - description: "Enrich order with customer data" ----- - -xref:components:inputs/about.adoc[Inputs] read data from sources, xref:components:outputs/about.adoc[outputs] write data to destinations, and xref:components:caches/about.adoc[caches] store and retrieve data. Define these components directly at the top level. Do not wrap components in `input:`, `output:`, or `cache:` blocks. This syntax is for pipelines, not MCP tools. - -.Input tool -[source,yaml] ----- -label: read-events - -redpanda: # <1> -ifdef::env-cloud[] - seed_brokers: ["${secrets.REDPANDA_BROKERS}"] -endif::[] -ifndef::env-cloud[] - seed_brokers: ["${REDPANDA_BROKERS}"] -endif::[] - topics: ["events"] - consumer_group: "mcp-reader" - -meta: - mcp: - enabled: true - description: "Read events from Redpanda" ----- -<1> The component name (`redpanda`) is at the top level, not wrapped in `input:`. - -.Output tool -[source,yaml] ----- -label: publish-event - -redpanda: -ifdef::env-cloud[] - seed_brokers: ["${secrets.REDPANDA_BROKERS}"] -endif::[] -ifndef::env-cloud[] - seed_brokers: ["${REDPANDA_BROKERS}"] -endif::[] - topic: "processed-events" - -meta: - mcp: - enabled: true - description: "Publish event to Redpanda" ----- - -.Cache tool -[source,yaml] ----- -label: session-cache - -memory: - default_ttl: 300s - -meta: - mcp: - enabled: true - description: "In-memory cache for session data" ----- - -Outputs can include a `processors:` section to transform data before publishing: - -.Output with processors -[source,yaml] ----- -label: publish-with-timestamp - -processors: - - mutation: | - root = this - root.published_at = now() - -redpanda: -ifdef::env-cloud[] - seed_brokers: ["${secrets.REDPANDA_BROKERS}"] -endif::[] -ifndef::env-cloud[] - seed_brokers: ["${REDPANDA_BROKERS}"] -endif::[] - topic: "processed-events" - -meta: - mcp: - enabled: true - description: "Add timestamp and publish to Redpanda" ----- - -ifdef::env-cloud[] -For more examples, see xref:ai-agents:mcp/remote/tool-patterns.adoc#outputs-with-processors[outputs with processors]. -endif::[] -ifndef::env-cloud[] -For more examples, see xref:ai-agents:mcp-server/tool-patterns.adoc#outputs-with-processors[outputs with processors]. -endif::[] - -[[mcp-metadata]] -=== MCP metadata fields - -The `meta.mcp` block defines how AI clients discover and interact with your tool. These fields control tool visibility, naming, and input parameters. - -include::ai-agents:partial$mcp/create-tool/mcp-metadata-fields-table.adoc[] - -==== Property fields - -include::ai-agents:partial$mcp/create-tool/property-fields-table.adoc[] - -[[property-restrictions]] -==== Property restrictions by component type - -include::ai-agents:partial$mcp/create-tool/property-restrictions-table.adoc[] - -[[parameter-mapping]] -== Map parameters to component fields - -When an AI client calls your tool, the `arguments` object becomes the message body. You can access these arguments using xref:guides:bloblang/about.adoc[Bloblang], but the syntax depends on where you're using it: - -* **Inside Bloblang contexts** (mutation, mapping, args_mapping): Use `this.field_name` -* **Inside string fields** (URLs, topics, headers): Use interpolation `${! json("field_name") }` - -=== In Bloblang contexts - -Use `this` to access message fields directly in processors like `mutation`, `mapping`, or in `args_mapping` fields: - -[source,yaml] ----- -mutation: | - root.search_query = this.query.lowercase() - root.max_results = this.limit.or(10) ----- - -[source,yaml] ----- -sql_select: - table: orders - where: customer_id = ? AND status = ? - args_mapping: '[this.customer_id, this.status.or("active")]' ----- - -=== In string fields (interpolation) - -Use `${! ... }` interpolation to embed Bloblang expressions inside string values like URLs or topic names: - -[source,yaml] ----- -http: - url: 'https://api.weather.com/v1/current?city=${! json("city") }&units=${! json("units").or("metric") }' ----- - -[source,yaml] ----- -redpanda: - seed_brokers: ["${REDPANDA_BROKERS}"] # <1> - topic: '${! json("topic_name") }' # <2> ----- -<1> `$\{VAR}` without `!` is environment variable substitution, not Bloblang. -<2> `${! ... }` with `!` is Bloblang interpolation that accesses message data. - -TIP: For more on Bloblang syntax, see xref:guides:bloblang/about.adoc[]. For interpolation details, see xref:configuration:interpolation.adoc[]. - -=== Provide defaults for optional parameters - -Use `.or(default)` to handle missing optional parameters: - -[source,yaml] ----- -mutation: | - root.city = this.city # Required - will error if missing - root.units = this.units.or("metric") # Optional with default - root.limit = this.limit.or(10).number() # Optional, converted to number ----- - -Declare which parameters are required in your `meta.mcp.properties`: - -[source,yaml] ----- -properties: - - name: city - type: string - description: "City name to look up" - required: true - - name: units - type: string - description: "Temperature units: 'metric' or 'imperial' (default: metric)" - required: false ----- - -[[secrets]] -== Use secrets and environment variables - -Never hardcode credentials, API keys, or connection strings in your tool files. Use xref:configuration:interpolation.adoc#environment-variables[environment variable substitution] to inject secrets at runtime. - -Reference environment variables using `$\{VARIABLE_NAME}` syntax. Redpanda Connect replaces these placeholders when loading the configuration: - -[source,yaml] ----- -http: - url: "https://api.example.com/data" - headers: - Authorization: "Bearer ${API_TOKEN}" - -sql_select: - driver: postgres - dsn: "${DATABASE_URL}" - table: customers ----- - -NOTE: `$\{VAR}` is environment variable substitution (resolved at startup). `$\{! expr }` is Bloblang interpolation (resolved at runtime from message data). See <> for the difference. - -Set environment variables before starting the MCP server: - -[source,bash] ----- -export API_TOKEN="your-secret-token" -export DATABASE_URL="postgres://user:password@localhost:5432/mydb" -rpk connect mcp-server --address localhost:4195 ----- - -For production deployments, load secrets from a secrets manager or inject them from your deployment system rather than exporting them in a shell. - -For naming conventions and security guidelines, see xref:ai-agents:mcp-server/best-practices.adoc[]. - -== Test the tool - -. Navigate to the root of your MCP project directory. - -. Lint your configuration: -+ -[source,bash] ----- -rpk connect mcp-server lint ----- - -. Start the MCP server: -+ -[source,bash] ----- -rpk connect mcp-server --address localhost:8080 ----- - -. Test tool calls using curl: -+ -[source,bash] ----- -#!/bin/bash -# Start SSE connection and capture session ID -exec 3< <(curl -s -N http://localhost:8080/sse) -read -r line <&3 # event: endpoint -read -r line <&3 # data: /sse?sessionid=XXX -SESSION_ID=$(echo "$line" | sed 's/.*sessionid=//') -echo "Session ID: $SESSION_ID" - -# Initialize the session -curl -s -X POST "http://localhost:8080/message?sessionid=$SESSION_ID" \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}}' - -sleep 1 - -# Call your tool (replace with your tool name and arguments) -curl -s -X POST "http://localhost:8080/message?sessionid=$SESSION_ID" \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"lookup-customer","arguments":{"customer_id":"cust_12345"}}}' - -# Read SSE responses -sleep 2 -while read -r -t 1 line <&3; do - echo "$line" -done - -exec 3<&- ----- - -. Connect to an AI client and verify the tool appears. For example, with Claude Code: -+ -[source,bash] ----- -claude mcp add local -- npx mcp-remote http://localhost:8080/sse -claude /mcp ----- - -. Test end-to-end with realistic prompts to verify the AI client uses your tool correctly. - -[[tool-filtering]] -== Control which tools are exposed - -By default, `rpk connect mcp-server` exposes all tools in your project that have `meta.mcp.enabled: true`. Use tags to selectively expose subsets of tools. - -Add tags to your tool's metadata: - -[source,yaml] ----- -meta: - mcp: - enabled: true - description: "Query production database" - tags: - - production - - database ----- - -Start the server with the `--tag` flag to expose only tools matching specific tags. The flag supports regular expressions: - -[source,bash] ----- -# Expose only tools tagged "production" -rpk connect mcp-server --tag production - -# Expose tools with tags starting with "db-" -rpk connect mcp-server --tag "db-.*" ----- - -For guidance on organizing tools by environment, feature, or access level, see xref:ai-agents:mcp-server/best-practices.adoc#tags[Tag strategies]. - -== Complete example - -Here's a complete tool that wraps the `http` processor to fetch weather data: - -[source,yaml] ----- -label: get-weather - -processors: - # Validate and sanitize input - - label: validate_city - mutation: | - root.city = if this.city.or("").trim() == "" { - throw("city is required") - } else { - this.city.trim().lowercase().re_replace_all("[^a-z\\s\\-]", "") - } - root.units = this.units.or("metric") - - # Fetch weather data - - label: fetch_weather - try: - - http: - url: 'https://wttr.in/${! json("city") }?format=j1' - verb: GET - timeout: 10s - - - mutation: | - root.weather = { - "location": this.nearest_area.0.areaName.0.value, - "country": this.nearest_area.0.country.0.value, - "temperature_c": this.current_condition.0.temp_C, - "temperature_f": this.current_condition.0.temp_F, - "condition": this.current_condition.0.weatherDesc.0.value, - "humidity": this.current_condition.0.humidity, - "wind_kph": this.current_condition.0.windspeedKmph - } - - # Handle errors gracefully - - label: handle_errors - catch: - - mutation: | - root.error = true - root.message = "Failed to fetch weather: " + error() - -meta: - mcp: - enabled: true - description: "Get current weather for a city. Returns temperature, conditions, humidity, and wind speed." - properties: - - name: city - type: string - description: "City name (e.g., 'London', 'New York', 'Tokyo')" - required: true - - name: units - type: string - description: "Temperature units: 'metric' or 'imperial' (default: metric)" - required: false ----- - -== Next steps - -* xref:ai-agents:mcp-server/best-practices.adoc[]: Apply naming and design guidelines. -* xref:ai-agents:mcp-server/tool-patterns.adoc[]: Find patterns for databases, APIs, and Redpanda. -* xref:ai-agents:mcp-server/troubleshooting.adoc[]: Diagnose common issues. -* xref:components:about.adoc[]: Browse all available components. diff --git a/modules/ai-agents/pages/mcp-server/index.adoc b/modules/ai-agents/pages/mcp-server/index.adoc deleted file mode 100644 index 3fde0ea7..00000000 --- a/modules/ai-agents/pages/mcp-server/index.adoc +++ /dev/null @@ -1,3 +0,0 @@ -= MCP Servers for Redpanda Connect -:description: Expose your Redpanda Connect configurations and internal tools as AI-consumable HTTP endpoints. -:page-layout: index \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp-server/overview.adoc b/modules/ai-agents/pages/mcp-server/overview.adoc deleted file mode 100644 index 9842b82f..00000000 --- a/modules/ai-agents/pages/mcp-server/overview.adoc +++ /dev/null @@ -1,89 +0,0 @@ -= MCP Server Overview -:description: Discover the business value, use cases, and architecture for Redpanda Connect MCP servers. -:page-topic-type: overview -:personas: ai_agent_developer, evaluator -// Reader journey: "I'm evaluating this" -// Learning objectives - what readers should understand after reading this page: -:learning-objective-1: Explain what an MCP server is and how tools differ from pipelines -:learning-objective-2: Identify use cases where MCP servers provide business value -:learning-objective-3: Describe how MCP tools expose Redpanda Connect components to AI - -This page introduces MCP servers and helps you decide if they're right for your use case. - -After reading this page, you will be able to: - -* [ ] {learning-objective-1} -* [ ] {learning-objective-2} -* [ ] {learning-objective-3} - -== What is MCP? - -// What is MCP - single-sourced from partial -include::ai-agents:partial$mcp/overview/what-is-mcp.adoc[] - -== What is Redpanda Connect MCP server? - -Redpanda Connect's MCP server lets you expose data pipelines and automations as AI-consumable tools (MCP tools) with no custom API code. You write tools as YAML configurations, and the MCP server makes them available to AI agents. - -This means AI agents can: - -* Pull information from databases, APIs, message queues, or Redpanda topics -* Publish messages, update records, trigger workflows, or call external services -* Only access the specific tools you configure and enable - -If you already use Redpanda Connect for data integration and processing, adding an MCP server gives AI agents controlled access to your existing systems without extra development effort. - -// MCP tools vs pipelines - single-sourced from partial -include::ai-agents:partial$mcp/overview/mcp-vs-pipelines.adoc[] - -// Use cases table - single-sourced from partial -include::ai-agents:partial$mcp/overview/use-cases-table.adoc[] - -== How it works - -When you start an MCP server, it runs locally on your system and acts as a bridge between AI clients and your data: - -. A user asks their AI agent something like "What's the weather in London?" -. The AI client connects to the local MCP server and finds the matching tool -. The server runs your MCP tools -. Your tools fetch data, transform it, and return a response -. The AI agent gets the data and can use it to answer the user - -=== What an MCP tool looks like - -An MCP tool in Redpanda Connect is a YAML file with two parts: the logic (what the tool does) and the metadata (how AI understands it). - -Here's a minimal example that returns weather data: - -[source,yaml] ----- -http: - url: "https://wttr.in/${! this.city }?format=j1" - verb: GET - -meta: - mcp: - enabled: true - name: get_weather - description: "Get current weather for a city" - properties: - - name: city - type: string - description: "City name" - required: true ----- - -When an AI client asks about weather, it calls this tool with the city name. The tool fetches data from the weather API and returns it. - -// MCP specification support - single-sourced from partial -include::ai-agents:partial$mcp/overview/specification-support.adoc[] - -== Next steps - -Continue your learning journey with these resources: - -* xref:ai-agents:mcp-server/quickstart.adoc[]: Start your first MCP server -* xref:ai-agents:mcp-server/concepts.adoc[]: Learn about execution and component types -* xref:ai-agents:mcp-server/create-tool.adoc[]: Create custom tools -* link:https://modelcontextprotocol.io/[Model Context Protocol documentation^] - diff --git a/modules/ai-agents/pages/mcp-server/quickstart.adoc b/modules/ai-agents/pages/mcp-server/quickstart.adoc deleted file mode 100644 index 93042f5b..00000000 --- a/modules/ai-agents/pages/mcp-server/quickstart.adoc +++ /dev/null @@ -1,319 +0,0 @@ -= MCP Server Quickstart -:description: Expose Redpanda Connect components as AI-consumable tools using the Model Context Protocol (MCP). -:page-topic-type: tutorial -:personas: ai_agent_developer, streaming_developer, evaluator -// Reader journey: "I want to try it now" -// Learning objectives - what readers will achieve by completing this quickstart: -:learning-objective-1: Initialize an MCP server project using the Redpanda CLI -:learning-objective-2: Create a tool with MCP metadata -:learning-objective-3: Connect the server to Claude Code and invoke the tool - -This quickstart builds a local MCP server that exposes Redpanda Connect components as tools that AI agents can use. It creates a tool that searches Bluesky posts, then asks Claude Code to perform the search using natural language: - -> Find recent posts on Bluesky mentioning Redpanda. - -By completing this quickstart, you will be able to: - -* [ ] {learning-objective-1} -* [ ] {learning-objective-2} -* [ ] {learning-objective-3} - -TIP: For background on how MCP tools work and when to use each component type, see xref:ai-agents:mcp-server/concepts.adoc[]. - -== Prerequisites - -You'll need the following to complete this quickstart: - -- xref:ROOT:get-started:rpk-install.adoc[Redpanda CLI (`rpk`)] -- link:https://docs.anthropic.com/en/docs/claude-code/setup[Claude Code^] -- link:https://docs.npmjs.com/downloading-and-installing-node-js-and-npm[Node.js and npm^] (required for running `npx` commands) - -== Initialize an MCP server project - -To get started, create a new MCP server project using the Redpanda CLI. - -. Create a directory for your MCP server project: -+ -[source,bash] ----- -mkdir redpanda-connect-mcp -cd redpanda-connect-mcp ----- - -. Initialize the MCP server project: -+ -[,bash] ----- -rpk connect mcp-server init ----- -+ -This command scaffolds your project with the necessary directories and template YAML files for defining tools and observability resources. -+ -[.no-copy] ----- -├── o11y -│ ├── metrics.yaml -│ └── tracer.yaml -└── resources - ├── caches - │ └── example-cache.yaml - ├── inputs - │ └── example-input.yaml - ├── outputs - │ └── example-output.yaml - └── processors - └── example-processor.yaml ----- - -. Make sure your Redpanda Connect version is at least 4.66.1: -+ -[,bash] ----- -rpk connect --version ----- -+ -If you need to upgrade, see xref:install:rpk.adoc#upgrade[Upgrade Redpanda Connect]. - -== Create a tool definition - -In this step, you'll create an MCP tool that searches public link:https://bsky.app/[Bluesky^] posts using a customizable query. You'll define this configuration as a tool that Claude can discover and use. - -Save the following YAML to a file named `bluesky.yaml` and save it in the `resources/processors/` directory: - -[source,yaml] ----- -label: search-bluesky-posts <1> -try: <2> - - mutation: | - root.limit = root.limit.number(10) - root.limit = [ root.limit, 100 ].min() - root.limit = [ root.limit, 1 ].max() - meta query_string = "q=" + root.query.escape_url_query() + "&limit=%v".format(root.limit) - root = "" - - http: - url: "https://api.bsky.app/xrpc/app.bsky.feed.searchposts?${! @query_string }" - verb: GET - - mapping: 'root = this.posts' - - unarchive: - format: json_array - -meta: - tags: [ bluesky ] <3> - mcp: - enabled: true - description: Search public Bluesky posts based on a query string. <4> - properties: <5> - - name: query - type: string - required: true - description: A Lucene-style query string to search posts. - - name: limit - type: number - description: Number of posts to return (1-100). Defaults to 10. ----- - -This example defines a tool that searches public Bluesky posts based on a query string. Important parts of the configuration include: - -<1> A unique label for identification. Claude uses this label to discover and call the tool. -<2> A series of xref:ai-agents:mcp-server/concepts.adoc#component-selection[processors] to build the HTTP request, handle the response, and format the output. -<3> A unique tag (`bluesky`) to control exposure. You can use tags to group related tools or isolate experiments. -<4> A clear description for LLMs helps Claude understand what the tool does. For more details, see xref:ai-agents:mcp-server/create-tool.adoc#anatomy[Anatomy of an MCP tool]. -<5> Structured properties for inputs like `query` and `limit`. - -== Start the MCP server - -. Navigate to your MCP server project directory if you're not already there: -+ -[,bash] ----- -cd redpanda-connect-mcp ----- - -. Start the MCP server to expose the tool over HTTP: -+ -[source,bash] ----- -rpk connect mcp-server --address localhost:4195 --tag bluesky ----- -+ -You should see output like this: -+ -[.no-copy] ----- -time=2025-06-27T15:20:27.976+01:00 level=INFO msg="Registering processor tool" label=search-bluesky-posts -time=2025-06-27T15:20:27.978+01:00 level=INFO msg="Successfully loaded Redpanda license" expires_at=2035-06-25T15:20:27+01:00 license_org="" license_type="open source" ----- -+ -This command creates an MCP server listening on `localhost:4195`, and makes your tool discoverable to AI agents. Leave this terminal open while you interact with Claude Code. - -:tip-caption: Limit exposure - -[TIP] -==== -Only tools with the specified `--tag` are exposed. This helps you: - -- Keep experiments isolated -- Avoid exposing sensitive functionality accidentally -- Create sets of tools that are relevant to specific agents or workflows -==== - -:tip-caption: Tip - -== Connect Claude Code to your MCP server - -Now that your MCP server is running and exposing tools, you can connect Claude Code so it can discover and use them. - -Claude Code communicates with MCP servers using the standard input/output (stdio) transport protocol. Because your Redpanda Connect MCP server exposes tools over HTTP with Server-Sent Events (SSE), you need a bridge to convert between these protocols. The link:https://www.npmjs.com/package/mcp-remote[`mcp-remote` utility^] provides this bridge. - -When you connect Claude Code: - -- `mcp-remote` subscribes to your server's SSE endpoint (`http://localhost:4195/sse`). -- It translates HTTP/SSE messages into stdio format that Claude understands. -- Claude automatically discovers your tools and their metadata. -- You can ask Claude in natural language to use your tools. -- Claude calls the appropriate tool based on your request. - -To set up this connection: - -. Open a new terminal window. - -. To install `mcp-remote`, run: -+ -[,bash] ----- -claude mcp add local -- npx mcp-remote http://localhost:4195/sse ----- -+ -You should see output like this: -+ -[.no-copy] ----- -Added stdio MCP server local with command: npx mcp-remote http://localhost:4195/sse to local config ----- - -. Verify that the local input channel is set up correctly by running: -+ -[source,bash] ----- -claude /mcp ----- -+ -You should see an entry for `local`. - -. Press *Enter* until you see the tools list. -+ -[.no-copy,role="no-wrap"] ----- -Tools for local (1 tools) -│ ❯ 1. search-bluesky-posts ----- - -. Press *Esc* until you return to the main prompt. - -== Use the tool with Claude - -Now you can see MCP in action. When you ask Claude to search Bluesky, watch what happens behind the scenes. - -. Enter the following prompt to start a conversation with Claude Code: -+ -[source,bash] ----- -Search Bluesky for the latest news about Redpanda Data ----- -+ -[TIP] -==== -If you exited the prompt, you can re-enter it by running: - -[source,bash] ----- -claude Search Bluesky for the latest news about Redpanda Data ----- -==== - -. When Claude prompts you for permissions, select *Yes*: -+ -[role="no-wrap no-copy"] ----- -> Search Bluesky for the latest news about Redpanda Data - -⏺ local:search-bluesky-posts (MCP)(value: "Redpanda") - -╭───────────────────────────────────────────── -│ Tool use -│ local:search-bluesky-posts(value: "Redpanda") (MCP) -│ Search public Bluesky posts based on a query string. -│ Do you want to proceed? -│ ❯ 1. Yes -│ 2. Yes, and don't ask again for local:search-bluesky-posts commands in /my-agent │ -│ 3. No, and tell Claude what to do differently (esc) ----- -+ -This is what happens: -+ --- -* Claude analyzes your natural language request. -* Claude identifies the `search-bluesky-posts` tool as the right one to use. -* Claude extracts "Redpanda Data" as the query parameter. -* Claude calls your local MCP server via `mcp-remote`. -* Your Redpanda Connect configuration executes the Bluesky API request. -* Results flow back through the MCP server to Claude. -* Claude synthesizes the results into a conversational response. --- - -Here's an example of what the result might look like: - -[role="no-wrap no-copy"] ----- -⏺ Based on the search results from Bluesky, here are the latest posts about Redpanda Data: - - Recent Company Activity: - - Product Development: Expanding Redpanda Connect beyond data infrastructure to support AI agent tools like - Slack, GitHub, and Google Drive - - New Snowflake Connector: Launched with 2x faster throughput than Kafka Connect for data streaming - - Events & Marketing: - - Participated in Data+AI Summit (Booth E500) - - Hosting disaster recovery streamcast on June 12th - - Fireside chat with Entain's Chief Data Officer about replacing Confluent with Redpanda in 2 weeks - - Success Stories: - - Gaming company Entain replaced Confluent with Redpanda in just two weeks - - Focus on real-time financial data streaming to Snowflake for analytics - - The posts show Redpanda positioning itself as a faster, simpler alternative to Kafka/Confluent with strong - momentum in hiring and customer wins. ----- - -[NOTE] -==== -If you change the YAML configuration of your tools, restart the MCP server to pick up the changes. -==== - -== Stop or disconnect the MCP server - -To disconnect or stop the MCP server, press kbd:[Ctrl+C] in the terminal where the server is running. This will gracefully shut down the MCP server process and disconnect any connected clients. - -You can also close the terminal window or kill the process using standard OS commands (such as `kill ` on Linux/macOS). - -== Troubleshoot - -If you encounter issues during this quickstart: - -- **Tool not appearing**: Verify `meta.mcp.enabled: true` is set and the tag matches your `--tag` argument. -- **Connection issues**: Ensure the MCP server is running and listening on the expected port. -- **JSON schema errors**: Upgrade to at least version 4.66.1 of Redpanda Connect. - -For detailed solutions, see xref:ai-agents:mcp-server/troubleshooting.adoc[]. - -== Next steps - -You've built a working MCP tool and connected it to Claude Code. Here's where to go next: - -* xref:ai-agents:mcp-server/concepts.adoc[]: Understand how MCP tools differ from pipelines -* xref:ai-agents:mcp-server/create-tool.adoc[]: Build production-quality tools with validation -* xref:ai-agents:mcp-server/best-practices.adoc[]: Apply naming and design guidelines -* xref:ai-agents:mcp-server/tool-patterns.adoc[]: Find reusable patterns -* xref:ai-agents:mcp-server/troubleshooting.adoc[]: Diagnose common issues - - diff --git a/modules/ai-agents/pages/mcp-server/tool-patterns.adoc b/modules/ai-agents/pages/mcp-server/tool-patterns.adoc deleted file mode 100644 index bdb12e5b..00000000 --- a/modules/ai-agents/pages/mcp-server/tool-patterns.adoc +++ /dev/null @@ -1,305 +0,0 @@ -= MCP Tool Patterns -:description: Cookbook of reusable patterns for building MCP tools with Redpanda Connect. -:page-aliases: ai-agents:mcp-server/pipeline-patterns.adoc -:page-topic-type: cookbook -:personas: ai_agent_developer, data_engineer -// Reader journey: "I need an example for X" -:learning-objective-1: Find reusable patterns for common MCP tool scenarios -:learning-objective-2: Apply validation and error handling patterns for production robustness -:learning-objective-3: Format responses consistently for AI client consumption - -When building tools, use these patterns as starting points for common scenarios. For step-by-step instructions, see xref:ai-agents:mcp-server/create-tool.adoc[]. For design guidelines, see xref:ai-agents:mcp-server/best-practices.adoc[]. - -After reading this page, you will be able to: - -* [ ] {learning-objective-1} -* [ ] {learning-objective-2} -* [ ] {learning-objective-3} - - - -[[read-data]] -== Read data - -Use xref:components:inputs/about.adoc[inputs] to create tools that read from data sources or generate sample data. - -[[data-generators]] -=== Generate test data - -*When to use:* Development and testing environments where you need synthetic data, load testing scenarios, or demonstrating data flows without real data sources. - -*Example use cases:* Mock user events, test order data, synthetic sensor readings, demo data for presentations. - -[source,yaml] ----- -include::ai-agents:example$resources/inputs/generate-input.yaml[] ----- - -See also: xref:components:inputs/generate.adoc[`generate` input component] - -[[consume-from-redpanda]] -=== Consume from Redpanda topics - -*When to use:* Processing events from Redpanda topics, building event-driven AI agents, consuming audit logs, or subscribing to data change streams. - -*Example use cases:* Monitor order events, process user activity streams, consume IoT sensor data, react to system notifications. - -NOTE: This example requires setting the `REDPANDA_BROKERS` environment variable with your Redpanda broker addresses. - -[source,yaml] ----- -include::ai-agents:example$resources/inputs/redpanda-consume.yaml[] ----- - -[[stream-processing]] -=== Process streaming data - -*When to use:* Real-time analytics, windowed aggregations, computing metrics over time, or building streaming dashboards. - -*Example use cases:* Calculate rolling averages, count events per time window, detect anomalies in streams, aggregate metrics. - -[source,yaml] ----- -include::ai-agents:example$resources/inputs/stream-analytics.yaml[] ----- - -[[call-external-services]] -== Call external services - -Use xref:components:processors/about.adoc[processors] to fetch data from external APIs, databases, or AI services. - -[[external-api-calls]] -=== Call REST APIs - -*When to use:* Integrating with third-party services, fetching real-time data, calling internal microservices, or enriching event data with external information. - -*Example use cases:* Fetch user profile from CRM, get product pricing from inventory API, validate addresses with geocoding service, retrieve weather data. - -[source,yaml] ----- -include::ai-agents:example$resources/processors/weather-api.yaml[] ----- - -See also: xref:components:processors/http.adoc[`http` processor], xref:components:processors/mutation.adoc[`mutation` processor] - -[[database-queries]] -=== Query databases - -*When to use:* Retrieving customer records, querying analytics data, looking up configuration values, or joining streaming data with dimensional data from data warehouses. - -*Example use cases:* Fetch customer details from PostgreSQL, query sales data from BigQuery, retrieve product catalog from MongoDB, look up reference data. - -NOTE: This example requires setting the `DATABASE_URL` environment variable with your PostgreSQL connection string. - -[source,yaml] ----- -include::ai-agents:example$resources/processors/database-query.yaml[] ----- - -See also: xref:components:processors/sql_select.adoc[`sql_select` processor] - -[[jira-queries]] -=== Query Jira issues - -*When to use:* Fetching tickets by status, checking assignments, finding recent issues, or building AI agents that interact with project management data. - -*Example use cases:* Get open bugs for a sprint, find issues assigned to a user, list recently updated tickets, search by custom fields. - -NOTE: The `jira` processor requires an Enterprise license. For more information, see xref:get-started:licensing.adoc[]. - -[source,yaml] ----- -label: search-jira - -processors: - - generate: - count: 1 - mapping: | - root.jql = this.jql - root.maxResults = this.max_results.or(50) - root.fields = ["key", "summary", "status", "assignee", "priority"] - - jira: - base_url: "${JIRA_BASE_URL}" - username: "${JIRA_USERNAME}" - api_token: "${JIRA_API_TOKEN}" - -meta: - mcp: - enabled: true - description: "Search Jira issues using JQL. Returns matching issues with key, summary, status, assignee, and priority." - properties: - - name: jql - type: string - description: "JQL query (e.g., 'project = DOC AND status = Open')" - required: true - - name: max_results - type: number - description: "Maximum issues to return (default: 50)" - required: false ----- - -For more patterns including pagination, custom fields, and creating issues via the HTTP processor, see xref:cookbooks:jira.adoc[]. - -[[ai-llm-integration]] -=== Integrate with AI/LLM services - -*When to use:* Generating embeddings for semantic search, calling LLM APIs for text generation, building RAG pipelines, or analyzing sentiment. - -*Example use cases:* Generate embeddings for documents, classify customer feedback, summarize long text, extract entities, answer questions with context. - -==== OpenAI chat completion - -[source,yaml] ----- -openai_chat_completion: - api_key: "${OPENAI_API_KEY}" - model: "gpt-4" - prompt: | - Analyze this customer feedback and provide: - 1. Sentiment (positive/negative/neutral) - 2. Key themes - 3. Actionable insights - - Feedback: ${! json("feedback_text") } - max_tokens: 500 ----- - -See also: xref:components:processors/openai_chat_completion.adoc[`openai_chat_completion`], xref:components:processors/openai_embeddings.adoc[`openai_embeddings`] - -==== Generate embeddings - -[source,yaml] ----- -openai_embeddings: - api_key: "${OPENAI_API_KEY}" - model: "text-embedding-3-small" - text: ${! json("content") } ----- - -See also: xref:components:processors/cohere_embeddings.adoc[`cohere_embeddings`], xref:components:processors/gcp_vertex_ai_embeddings.adoc[`gcp_vertex_ai_embeddings`] - -[[write-data]] -== Write data - -Use xref:components:outputs/about.adoc[outputs] to write data to Redpanda topics or cache stores. - -[[publish-to-redpanda]] -=== Publish to Redpanda topics - -*When to use:* Publishing events to Redpanda for consumption by other services, creating event sourcing patterns, building audit trails, or triggering downstream workflows. - -*Example use cases:* Publish order confirmations, emit audit events, trigger notifications, create event-driven workflows. - -NOTE: This example requires setting the `REDPANDA_BROKERS` environment variable. - -[source,yaml] ----- -include::ai-agents:example$resources/outputs/redpanda-publish.yaml[] ----- - -==== Outputs with processors - -Output tools can include processors to transform data before publishing. This pattern is useful when you need to process data and save the result to a destination in a single tool. - -*When to use:* Processing user input with an LLM and saving the response, transforming data before publishing to a topic, enriching events before writing to external systems. - -[source,yaml] ----- -include::ai-agents:example$resources/outputs/redpanda-output-with-processors.yaml[] ----- - -[[caching]] -=== Cache data - -*When to use:* Reducing repeated API calls, storing lookup tables, caching database query results, or maintaining session state across tool invocations. - -*Example use cases:* Cache user profiles, store API rate limit counters, maintain configuration values, cache product catalogs. - -[source,yaml] ----- -memory: - default_ttl: 300s - compaction_interval: 60s ----- - -See also: xref:components:caches/memory.adoc[`memory` cache], xref:components:outputs/redpanda.adoc[`redpanda` output] - -[[transform-data]] -== Transform data - -Use Bloblang and processors to transform, validate, and route data. - -[[data-transformation]] -=== Transform and validate - -*When to use:* Converting data formats, validating schemas, filtering events, enriching messages with computed fields, or normalizing data structures. - -*Example use cases:* Parse JSON payloads, validate required fields, add timestamps, convert units, mask sensitive data, aggregate nested objects. - -[source,yaml] ----- -mapping: | - # Parse and validate incoming data - root.user_id = this.user_id.or(throw("user_id is required")) - root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") - - # Transform and enrich - root.email_domain = this.email.split("@").index(1) - root.is_premium = this.subscription_tier == "premium" - - # Filter sensitive data - root.profile = this.profile.without("ssn", "credit_card") ----- - -See also: xref:components:processors/mapping.adoc[`mapping` processor], xref:guides:bloblang/about.adoc[Bloblang guide] - -[[event-driven-workflows]] -=== Build event-driven workflows - -*When to use:* Orchestrating multi-step processes, responding to business events, implementing saga patterns, or coordinating microservices. - -*Example use cases:* Order fulfillment workflows, approval processes, notification cascades, data pipeline orchestration. - -[source,yaml] ----- -include::ai-agents:example$resources/inputs/event-workflow.yaml[] ----- - -See also: xref:components:inputs/redpanda.adoc[`redpanda` input] - -[[production-readiness]] -== Production readiness - -Build production-ready tools with proper input validation, error handling, and response formatting. - -[[input-validation]] -=== Validate input - -AI clients may send unexpected or malformed input. Validate early to return helpful error messages instead of cryptic failures from downstream components. - -include::ai-agents:partial$mcp/tool-patterns/input-validation.adoc[leveloffset=+1] - -[[error-handling]] -=== Handle errors - -External services fail. Databases go down. APIs return unexpected responses. Wrap risky operations in error handling so your tool returns useful error messages instead of crashing. - -include::ai-agents:partial$mcp/tool-patterns/error-handling.adoc[leveloffset=+1] - -[[response-formatting]] -=== Format responses - -AI clients work best with clean, predictable response structures. Transform raw component output into consistent formats. - -include::ai-agents:partial$mcp/tool-patterns/response-formatting.adoc[leveloffset=+1] - -// Production workflows section - single-sourced from partial -include::ai-agents:partial$mcp/tool-patterns/production-workflows.adoc[] - -NOTE: The multi-step enrichment example requires setting the `POSTGRES_DSN` environment variable with your PostgreSQL connection string. - -== Next steps - -* xref:ai-agents:mcp-server/create-tool.adoc[]: Step-by-step tool creation guide -* xref:ai-agents:mcp-server/best-practices.adoc[]: Apply naming and design guidelines -* xref:ai-agents:mcp-server/troubleshooting.adoc[]: Diagnose common issues diff --git a/modules/ai-agents/pages/mcp-server/troubleshooting.adoc b/modules/ai-agents/pages/mcp-server/troubleshooting.adoc deleted file mode 100644 index e22eedec..00000000 --- a/modules/ai-agents/pages/mcp-server/troubleshooting.adoc +++ /dev/null @@ -1,43 +0,0 @@ -= Troubleshoot MCP Servers -:description: Diagnose and fix common issues when building and running MCP servers with Redpanda Connect. -:page-topic-type: troubleshooting -:personas: ai_agent_developer, streaming_developer, platform_operator -// Reader journey: "Something went wrong" -// Learning objectives - what readers can do with this page: -:learning-objective-1: Diagnose and fix lint and YAML configuration errors -:learning-objective-2: Resolve runtime issues when tools don't appear or return unexpected results -:learning-objective-3: Debug client connection problems - -This page helps you diagnose and fix common issues when building and running MCP servers. - -Use this page to: - -* [ ] {learning-objective-1} -* [ ] {learning-objective-2} -* [ ] {learning-objective-3} - -== Lint errors - -include::ai-agents:partial$mcp/troubleshooting/troubleshooting-lint.adoc[] - -== Runtime issues - -include::ai-agents:partial$mcp/troubleshooting/troubleshooting-runtime.adoc[] - -== Connection issues - -include::ai-agents:partial$mcp/troubleshooting/troubleshooting-connection.adoc[] - -[[debugging]] -== Debugging techniques - -Use these techniques to systematically isolate and fix issues with your MCP tools. - -include::ai-agents:partial$mcp/troubleshooting/debugging-techniques.adoc[] - -== Next steps - -If you're still experiencing issues: - -* xref:ai-agents:mcp-server/create-tool.adoc[]: Review YAML structure rules -* xref:ai-agents:mcp-server/concepts.adoc[]: Review component type selection diff --git a/modules/ai-agents/partials/mcp/best-practices/mcp-metadata-design.adoc b/modules/ai-agents/partials/mcp/best-practices/mcp-metadata-design.adoc deleted file mode 100644 index b11cf610..00000000 --- a/modules/ai-agents/partials/mcp/best-practices/mcp-metadata-design.adoc +++ /dev/null @@ -1,89 +0,0 @@ -// ============================================================================= -// PARTIAL: mcp-metadata-design.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/best-practices.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/best-practices.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/best-practices/mcp-metadata-design.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/best-practices/mcp-metadata-design.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -AI clients use tool names and descriptions to decide which tool to call. Good metadata helps AI select the right tool and provide correct inputs. - -The `label` field defines the tool name. The `meta.mcp` block defines the description and input properties. - -[source,yaml] ----- -include::example$best-practices/mcp-metadata/basic-mcp-block.yaml[tags=basic-mcp-block] ----- - -ifdef::env-cloud[] -For the complete field reference, see xref:ai-agents:mcp/remote/create-tool.adoc#mcp-metadata[MCP metadata fields]. -endif::[] -ifndef::env-cloud[] -For the complete field reference, see xref:ai-agents:mcp-server/create-tool.adoc#mcp-metadata[MCP metadata fields]. -endif::[] - -=== Choose a clear tool name - -AI clients see the `label` value when selecting tools, so make it descriptive and consistent. - -The name should reflect the tool's primary action and target resource. Focus on clarity over brevity. For example, use `get-weather-forecast` instead of just `get-data`. - -=== Write effective descriptions - -Tool descriptions help AI clients decide when to use your tool. Start with an action verb and explain what the tool returns. - -[source,yaml] ----- -include::example$best-practices/mcp-metadata/descriptions.yaml[tags=good-description,indent=0] ----- - -Include any limitations or requirements in the description: - -[source,yaml] ----- -include::example$best-practices/mcp-metadata/descriptions.yaml[tags=with-limitations,indent=0] ----- - -=== Define input properties clearly - -Each property needs a name, type, description, and required status. Use `string`, `number`, or `boolean` types. - -Include example values in descriptions: - -[source,yaml] ----- -include::example$best-practices/mcp-metadata/property-definitions.yaml[tags=property-descriptions,indent=0] ----- - -Mark properties as required only if the tool cannot function without them: - -[source,yaml] ----- -include::example$best-practices/mcp-metadata/property-definitions.yaml[tags=required-property,indent=0] ----- - -Optional properties should have sensible defaults: - -[source,yaml] ----- -include::example$best-practices/mcp-metadata/property-definitions.yaml[tags=optional-property,indent=0] ----- - -ifdef::env-cloud[] -For patterns that apply defaults and validate values at runtime, see xref:ai-agents:mcp/remote/tool-patterns.adoc#input-validation[input validation patterns]. -endif::[] -ifndef::env-cloud[] -For patterns that apply defaults and validate values at runtime, see xref:ai-agents:mcp-server/tool-patterns.adoc#input-validation[input validation patterns]. -endif::[] - diff --git a/modules/ai-agents/partials/mcp/best-practices/tool-implementation-practices.adoc b/modules/ai-agents/partials/mcp/best-practices/tool-implementation-practices.adoc deleted file mode 100644 index 2fe66961..00000000 --- a/modules/ai-agents/partials/mcp/best-practices/tool-implementation-practices.adoc +++ /dev/null @@ -1,51 +0,0 @@ -// ============================================================================= -// PARTIAL: tool-implementation-practices.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/best-practices.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/best-practices.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/best-practices/tool-implementation-practices.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/best-practices/tool-implementation-practices.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -=== Keep tools focused - -Each tool should do one thing well. If you find yourself adding multiple unrelated operations, split them into separate tools. - -Tools that do too much or have vague purposes cause problems because AI clients rely on descriptions to choose tools. Vague descriptions lead to wrong tool selection. Also, tools that do too much are harder to test and debug. - -Write descriptions that clearly state what the tool does, what input it needs, and what it returns. If a tool is doing multiple things, split it into focused tools. - -=== Design for quick completion - -MCP tools should complete quickly. AI clients wait for responses, and long-running tools cause poor user experiences. - -Tools that wait indefinitely, poll continuously, or never return cause problems because MCP tools use a request/response model. A tool that never completes will time out and fail, and resources remain allocated while waiting. - -Follow these guidelines: - -ifdef::env-cloud[] -* Set explicit timeouts on all external calls. For timeout options, see xref:develop:connect/components/processors/http.adoc[`http` processor]. -endif::[] -ifndef::env-cloud[] -* Set explicit timeouts on all external calls. For timeout options, see xref:components:processors/http.adoc[`http` processor]. -endif::[] -* Avoid unbounded reads. Read N messages, not all messages. -* Consider pagination for large datasets. -* Return partial results if full processing takes too long. - -ifdef::env-cloud[] -For patterns that handle timeout failures gracefully, see xref:ai-agents:mcp/remote/tool-patterns.adoc#error-handling[error handling patterns]. -endif::[] -ifndef::env-cloud[] -For patterns that handle timeout failures gracefully, see xref:ai-agents:mcp-server/tool-patterns.adoc#error-handling[error handling patterns]. -endif::[] diff --git a/modules/ai-agents/partials/mcp/concepts/component-mapping.adoc b/modules/ai-agents/partials/mcp/concepts/component-mapping.adoc deleted file mode 100644 index b56519f8..00000000 --- a/modules/ai-agents/partials/mcp/concepts/component-mapping.adoc +++ /dev/null @@ -1,63 +0,0 @@ -// ============================================================================= -// PARTIAL: component-mapping.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/concepts.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/concepts.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/concepts/component-mapping.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/concepts/component-mapping.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches xref targets for Cloud context -// -// DEPENDENCIES: -// None -// ============================================================================= - -== How components map to MCP tools - -Each MCP tool is implemented as a single Redpanda Connect component. The component type determines what the tool can do. - -The following table shows which component types are available and their purposes: - -[cols="1,2", options="header"] -|=== -| Component Type | Purpose as an MCP Tool - -ifdef::env-cloud[] -| <> -endif::[] -ifndef::env-cloud[] -| <> -endif::[] -| Transforms, validates, or computes data. Calls external APIs. Returns results to the AI client. - -ifdef::env-cloud[] -| <> -endif::[] -ifndef::env-cloud[] -| <> -endif::[] -| Writes data to external systems (Redpanda topics, databases, APIs). Can include processors for transformation before writing. - -ifdef::env-cloud[] -| <> -endif::[] -ifndef::env-cloud[] -| <> -endif::[] -| Reads data from external systems. Returns the read data to the AI client. - -ifdef::env-cloud[] -| <> -endif::[] -ifndef::env-cloud[] -| <> -endif::[] -| Stores and retrieves data for use by other tools. -|=== - -Most MCP tools are processors. Use outputs when you need to write data. Use inputs when you need to read from external data sources. diff --git a/modules/ai-agents/partials/mcp/concepts/component-selection.adoc b/modules/ai-agents/partials/mcp/concepts/component-selection.adoc deleted file mode 100644 index 31734855..00000000 --- a/modules/ai-agents/partials/mcp/concepts/component-selection.adoc +++ /dev/null @@ -1,263 +0,0 @@ -// ============================================================================= -// PARTIAL: component-selection.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/concepts.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/concepts.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/concepts/component-selection.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/concepts/component-selection.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches xref targets for Cloud context -// -// DEPENDENCIES: -// None -// ============================================================================= - -Every MCP tool is implemented as a single component. Choosing the right component type is a critical design decision that affects what your tool can do and how it behaves. - -=== Decision framework - -To choose the right component type, ask what the tool's primary purpose is. - -Use the following table to match your tool's intent to a component type: - -[cols="2,1", options="header"] -|=== -| Question | Component Type - -| Does the tool compute or transform data and return results? -| <> - -| Does the tool call external APIs and return the response? -| <> - -| Does the tool write data to an external system (database, topic, API)? -| <> - -| Does the tool read data from an external source and return it? -| <> - -| Does the tool store and retrieve temporary data for other tools? -| <> -|=== - -The core principle is to choose the component type that matches the tool's primary intent. - -[[processor-tools]] -=== Processor tools - -ifdef::env-cloud[] -Processor tools transform, validate, compute, or fetch data and return results to the AI client. This is the most common tool type. See the xref:develop:connect/components/processors/about.adoc[processors reference] for available processors. -endif::[] -ifndef::env-cloud[] -Processor tools transform, validate, compute, or fetch data and return results to the AI client. This is the most common tool type. See the xref:components:processors/about.adoc[processors reference] for available processors. -endif::[] - -==== When to choose a processor tool - -Choose a processor tool when the tool's purpose is to compute or transform data, call an external API and return the response, or validate inputs and return errors or results. - -==== Use case: Fetch and transform external data - -Consider a scenario where an AI agent needs current weather data to answer a user's question about whether to bring an umbrella. - -The following prompts should trigger this type of tool: - -* "What's the weather in Berlin?" -* "Is it raining in Tokyo right now?" -* "Get me the current temperature for Seattle." - -A processor is the right choice because the tool fetches data from an API, transforms it into a useful format, and returns it. - -==== Use case: Validate and normalize data - -Consider a scenario where an AI agent needs to validate user-submitted data and return structured feedback about any issues. - -The following prompts should trigger this type of tool: - -* "Validate this customer record before saving." -* "Check if this order has all required fields." -* "Normalize this JSON and tell me what's missing." - -A processor is the right choice because the tool examines data, applies validation rules, and returns results. No data is written anywhere. - -[[output-tools]] -=== Output tools - -ifdef::env-cloud[] -Output tools write data to external systems. Use them when the primary purpose is to create a side effect such as persisting data, publishing an event, or triggering an action. See the xref:develop:connect/components/outputs/about.adoc[outputs reference] for available outputs. -endif::[] -ifndef::env-cloud[] -Output tools write data to external systems. Use them when the primary purpose is to create a side effect such as persisting data, publishing an event, or triggering an action. See the xref:components:outputs/about.adoc[outputs reference] for available outputs. -endif::[] - -==== When to choose an output tool - -Choose an output tool when the tool's purpose is to write data to Redpanda, a database, or an external API. The side effect (writing) should be the primary intent, not incidental. You can use `processors:` within the output to transform data before writing. Output tools are appropriate when you want the AI to trigger real-world actions. - -==== Understanding tool response vs. side effect - -Output tools have two outcomes: the side effect (data is written to the destination) and the tool response (the AI client receives confirmation that the write succeeded). - -The AI client does not receive the written data back. It receives status information. If you need to return the written data, consider using a processor tool instead. - -==== Use case: Publish events to Redpanda - -Consider a scenario where an AI agent needs to publish order events to Redpanda for downstream processing. - -The following prompts should trigger this type of tool: - -* "Publish this order to Redpanda." -* "Send the order event to the orders topic." -* "Record this new order for processing." - -An output is the right choice because the purpose is to write data to Redpanda. The AI needs to create a persistent record, not just compute something. - -==== Use case: Transform and publish - -Output components can include a `processors:` section that transforms data before writing to the destination. This is a single output component, not a combination of component types. - -Consider a scenario where an AI agent asks an LLM to summarize a document, then stores both the original and summary in Redpanda. - -The following prompts should trigger this type of tool: - -* "Summarize this document and save it." -* "Process this feedback with GPT and store the analysis." -* "Analyze this text and publish the results." - -An output with processors is the right choice because the primary intent is to store data. The processors provide pre-processing before writing. - -The execution flow for this pattern is as follows: - -. AI client calls the tool with input data. -. The `processors` section transforms the data. -. The output component writes the transformed data to the destination. -. The tool returns a response to the AI client. - -ifdef::env-cloud[] -For implementation examples, see xref:ai-agents:mcp/remote/tool-patterns.adoc#outputs-with-processors[outputs with processors] in the tool patterns guide. -endif::[] -ifndef::env-cloud[] -For implementation examples, see xref:ai-agents:mcp-server/tool-patterns.adoc#outputs-with-processors[outputs with processors] in the tool patterns guide. -endif::[] - -[[input-tools]] -=== Input tools - -ifdef::env-cloud[] -Input tools read data from external sources and return it to the AI client. They're useful when you need to query or fetch existing data. See the xref:develop:connect/components/inputs/about.adoc[inputs reference] for available inputs. -endif::[] -ifndef::env-cloud[] -Input tools read data from external sources and return it to the AI client. They're useful when you need to query or fetch existing data. See the xref:components:inputs/about.adoc[inputs reference] for available inputs. -endif::[] - -==== When to choose an input tool - -Choose an input tool when the tool's purpose is to read and return data from an external source, consume messages from a Redpanda topic, or build a query-style tool that retrieves existing data. - -==== Bounded vs. unbounded reads - -Input tools must return a finite result. Use bounded reads that fetch a specific number of messages or read until a condition is met. For example, "get me the latest N events" or "read messages from the last hour". Unbounded reads that poll continuously are not appropriate for MCP tools because the tool would never return a response to the AI client. - -==== Latency and scope considerations - -Keep these factors in mind when building input tools: - -* Input tools may have variable latency depending on the data source. -* Scope your reads appropriately. Don't try to read entire topics. -* Consider consumer group behavior: with a consumer group, each invocation advances through the stream. Without one, each invocation may read the same data. - -==== Use case: Query recent events - -Consider a scenario where an AI agent needs to retrieve recent user activity events to understand user behavior. - -The following prompts should trigger this type of tool: - -* "Show me recent user events." -* "Get the last 10 login events." -* "What events happened in the user-events topic recently?" - -An input is the right choice because the tool reads from an existing data source (topic) and returns what it finds. - -[[cache-tools]] -=== Cache tools - -ifdef::env-cloud[] -Cache tools store and retrieve temporary data that other tools can access. They're useful for sharing state between tool calls or storing frequently accessed data. See the xref:develop:connect/components/caches/about.adoc[caches reference] for available caches. -endif::[] -ifndef::env-cloud[] -Cache tools store and retrieve temporary data that other tools can access. They're useful for sharing state between tool calls or storing frequently accessed data. See the xref:components:caches/about.adoc[caches reference] for available caches. -endif::[] - -==== When to choose a cache tool - -Choose a cache tool when the tool's purpose is to store temporary data that expires after a set time, share state between multiple tool calls in a conversation, or reduce repeated calls to slow external APIs by caching results. - -==== Use case: Session state management - -Consider a scenario where an AI agent needs to remember user preferences across multiple tool calls within a conversation. - -The following prompts should trigger this type of tool: - -* "Remember that I prefer metric units." -* "Store my timezone as America/New_York." -* "Save this search filter for later." - -A cache is the right choice because the data is temporary, session-scoped, and needs to be accessible by other tools during the conversation. - -==== Use case: API response caching - -Consider a scenario where an AI agent frequently looks up the same reference data (like exchange rates or product catalogs) and you want to avoid repeated API calls. - -The following prompts should trigger cache usage: - -* "Get the current exchange rate" (cached for 5 minutes) -* "Look up product details" (cached for 1 hour) -* "Check inventory levels" (cached briefly to reduce load) - -A cache is the right choice because you want to store API responses temporarily and serve them on subsequent requests without hitting the external API again. - -=== Component selection summary - -The following table summarizes when to use each component type: - -[cols="1,2,2,1", options="header"] -|=== -| Component | Primary Intent | Example Tools | Returns - -| Processor -| Compute, transform, validate, fetch -| Weather lookup, data validation, API calls -| Computed result - -| Output -| Write data with side effects -| Publish events, store records, trigger webhooks -| Write confirmation - -| Output + processors -| Transform then write -| Summarize and store, enrich and publish -| Write confirmation - -| Input -| Read and return data -| Query recent events, search logs -| Retrieved data - -| Cache -| Store and retrieve temporary data -| Session state, API response caching -| Cached value or confirmation -|=== - -ifdef::env-cloud[] -For implementation examples and common patterns, see xref:ai-agents:mcp/remote/tool-patterns.adoc[]. -endif::[] -ifndef::env-cloud[] -For implementation examples and common patterns, see xref:ai-agents:mcp-server/tool-patterns.adoc[]. -endif::[] diff --git a/modules/ai-agents/partials/mcp/concepts/execution-model.adoc b/modules/ai-agents/partials/mcp/concepts/execution-model.adoc deleted file mode 100644 index acb4f361..00000000 --- a/modules/ai-agents/partials/mcp/concepts/execution-model.adoc +++ /dev/null @@ -1,47 +0,0 @@ -// ============================================================================= -// PARTIAL: execution-model.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/concepts.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/concepts.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/concepts/execution-model.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/concepts/execution-model.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches xref targets for Cloud context -// -// DEPENDENCIES: -// None -// ============================================================================= - -[[execution-model]] -== The MCP execution model - -When an AI client calls an MCP tool, the MCP server handles the request in a specific sequence. - -The execution follows these steps: - -. The AI client sends a JSON request to the MCP server with the tool name and parameters. -. The MCP server finds the corresponding component configuration. -. The MCP server executes the component with the input data. -. The component runs to completion and returns a result. -. The MCP server sends the result back to the AI client. -. The component instance is torn down. - -This execution model has several important characteristics: - -ifdef::env-cloud[] -- Stateless execution: Each tool invocation is independent. Tools do not maintain state between calls. If you need state, use an external store such as a xref:develop:connect/components/caches/about.adoc[cache], database, or Redpanda topic. -endif::[] -ifndef::env-cloud[] -- Stateless execution: Each tool invocation is independent. Tools do not maintain state between calls. If you need state, use an external store such as a xref:components:caches/about.adoc[cache], database, or Redpanda topic. -endif::[] - -- Synchronous by default: Tools run synchronously from the AI client's perspective. The client waits for the response before continuing. - -- Timeout boundaries: Tools should complete quickly. Long-running operations should be avoided or handled asynchronously. Set explicit timeouts on external calls. - -- No continuous processing: Unlike a traditional Redpanda Connect pipeline, MCP tools do not poll for messages or maintain connections between invocations. They start, execute, and stop. diff --git a/modules/ai-agents/partials/mcp/create-tool/label-naming-rules.adoc b/modules/ai-agents/partials/mcp/create-tool/label-naming-rules.adoc deleted file mode 100644 index ee76b5f6..00000000 --- a/modules/ai-agents/partials/mcp/create-tool/label-naming-rules.adoc +++ /dev/null @@ -1,29 +0,0 @@ -// ============================================================================= -// PARTIAL: label-naming-rules.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/create-tool.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/create-tool.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/create-tool/label-naming-rules.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/create-tool/label-naming-rules.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// -// VALIDATION SOURCE: -// Tested with rpk connect mcp-server lint (v4.66.1+) on 2025-01-11 -// ============================================================================= - -The `label` field (tool name) must follow these rules: - -* Lowercase letters, numbers, underscores, and hyphens only (`a-z`, `0-9`, `_`, `-`) -* Cannot start with an underscore -* No spaces or special characters - -Valid examples: `get-weather`, `lookup_customer`, `send-notification-v2` diff --git a/modules/ai-agents/partials/mcp/create-tool/mcp-metadata-fields-table.adoc b/modules/ai-agents/partials/mcp/create-tool/mcp-metadata-fields-table.adoc deleted file mode 100644 index 71fa86f6..00000000 --- a/modules/ai-agents/partials/mcp/create-tool/mcp-metadata-fields-table.adoc +++ /dev/null @@ -1,44 +0,0 @@ -// ============================================================================= -// PARTIAL: mcp-metadata-fields-table.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/create-tool.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/create-tool.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/create-tool/mcp-metadata-fields-table.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/create-tool/mcp-metadata-fields-table.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Adjusts tags field description for Cloud context -// -// DEPENDENCIES: -// None -// ============================================================================= - -[cols="1,1,3", options="header"] -|=== -| Field | Required | Description - -| `enabled` -| Yes -| Set to `true` to expose this component as an MCP tool. Set to `false` to disable without deleting the configuration. - -| `description` -| Yes -| Explains what the tool does and what it returns. AI clients use this to decide when to call the tool. - -| `properties` -| No -| Array of input parameters the tool accepts. See <> for the fields in each property. - -| `tags` -| No -ifdef::env-cloud[] -| Array of strings for categorizing tools. -endif::[] -ifndef::env-cloud[] -| Array of strings for categorizing tools. Use with `--tag` flag to filter which tools are exposed. -endif::[] -|=== diff --git a/modules/ai-agents/partials/mcp/create-tool/property-fields-table.adoc b/modules/ai-agents/partials/mcp/create-tool/property-fields-table.adoc deleted file mode 100644 index 7988bdb7..00000000 --- a/modules/ai-agents/partials/mcp/create-tool/property-fields-table.adoc +++ /dev/null @@ -1,42 +0,0 @@ -// ============================================================================= -// PARTIAL: property-fields-table.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/create-tool.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/create-tool.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/create-tool/property-fields-table.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/create-tool/property-fields-table.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -[[property-fields]] -Each entry in the `properties` array defines an input parameter: - -[cols="1,1,3", options="header"] -|=== -| Field | Required | Description - -| `name` -| Yes -| Parameter name. - -| `type` -| Yes -| Data type. Must be one of: `string`, `number`, or `boolean`. - -| `description` -| Yes -| Explains what the parameter is for. Include example values and any constraints. - -| `required` -| Yes -| Set to `true` if the tool cannot function without this parameter. -|=== diff --git a/modules/ai-agents/partials/mcp/create-tool/property-restrictions-table.adoc b/modules/ai-agents/partials/mcp/create-tool/property-restrictions-table.adoc deleted file mode 100644 index ff839d3e..00000000 --- a/modules/ai-agents/partials/mcp/create-tool/property-restrictions-table.adoc +++ /dev/null @@ -1,41 +0,0 @@ -// ============================================================================= -// PARTIAL: property-restrictions-table.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/create-tool.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/create-tool.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/create-tool/property-restrictions-table.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/create-tool/property-restrictions-table.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -Different component types have different property capabilities when exposed as MCP tools: - -[cols="1,2,2"] -|=== -| Component Type | Property Support | Details - -| `input` -| Only supports the `count` property -| AI clients can specify how many messages to read, but you cannot define custom properties. - -| `cache` -| No custom properties -| Properties are hardcoded to `key` and `value` for cache operations. - -| `output` -| Custom properties supported -| AI sees properties as an array for batch operations: `[{prop1, prop2}, {prop1, prop2}]`. - -| `processor` -| Custom properties supported -| You can define any properties needed for data processing operations. -|=== diff --git a/modules/ai-agents/partials/mcp/examples/customer-analytics-example.adoc b/modules/ai-agents/partials/mcp/examples/customer-analytics-example.adoc deleted file mode 100644 index 7af427c2..00000000 --- a/modules/ai-agents/partials/mcp/examples/customer-analytics-example.adoc +++ /dev/null @@ -1,61 +0,0 @@ -// ============================================================================= -// PARTIAL: customer-analytics-example.adoc -// ============================================================================= -// -// INCLUDED BY: -// (currently unused - available for future use) -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/examples/customer-analytics-example.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/examples/customer-analytics-example.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches secrets syntax for Cloud vs environment variables -// -// DEPENDENCIES: -// None -// ============================================================================= - -== Example: Customer analytics tool - -Here's how you might expose a customer analytics pipeline as an MCP tool: - -[source,yaml] ----- -label: analyze_customer_orders -processors: - - label: fetch_customer_data - sql_select: - driver: "postgres" -ifdef::env-cloud[] - dsn: "${secrets.POSTGRES_DSN}" -endif::[] -ifndef::env-cloud[] - dsn: "${POSTGRES_DSN}" -endif::[] - table: "orders" - where: "customer_id = ? AND created_at >= NOW() - INTERVAL '30 days'" - args_mapping: 'root = [this.customer_id]' - - - label: calculate_metrics - mutation: | - root = { - "customer_id": this.0.customer_id, - "total_orders": this.length(), - "total_spent": this.map_each(o -> o.total).sum(), - "avg_order_value": this.map_each(o -> o.total).mean(), - "last_order_date": this.map_each(o -> o.created_at).max() - } - -meta: - mcp: - enabled: true - description: "Analyze a customer's order history and spending patterns over the last 30 days" - properties: - - name: customer_id - type: string - description: "Customer ID to analyze" - required: true ----- - -This tool can be invoked by an AI agent with prompts like "analyze the order history for customer ID 12345" without needing to know the underlying SQL or processing logic. diff --git a/modules/ai-agents/partials/mcp/overview/mcp-vs-pipelines.adoc b/modules/ai-agents/partials/mcp/overview/mcp-vs-pipelines.adoc deleted file mode 100644 index e21e72ca..00000000 --- a/modules/ai-agents/partials/mcp/overview/mcp-vs-pipelines.adoc +++ /dev/null @@ -1,28 +0,0 @@ -// ============================================================================= -// PARTIAL: mcp-vs-pipelines.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/overview.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/overview.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/overview/mcp-vs-pipelines.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/overview/mcp-vs-pipelines.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -== MCP tools are not pipelines - -If you already use Redpanda Connect, you might wonder how MCP tools differ from pipelines. - -A pipeline is a continuous data flow: data streams from an input, through processors, to an output. The pipeline runs indefinitely, processing many messages over time. - -An MCP tool is different. It's a single component that executes on demand when called by an AI client. The tool starts, runs, and completes for each invocation. There is no persistent state between calls. Think of it like calling a function rather than running a service. - -This request/response pattern is what makes MCP tools useful for AI agents: the agent asks a question, the tool runs, and it returns an answer. diff --git a/modules/ai-agents/partials/mcp/overview/specification-support.adoc b/modules/ai-agents/partials/mcp/overview/specification-support.adoc deleted file mode 100644 index 76195aaa..00000000 --- a/modules/ai-agents/partials/mcp/overview/specification-support.adoc +++ /dev/null @@ -1,24 +0,0 @@ -// ============================================================================= -// PARTIAL: specification-support.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/overview.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/overview.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/overview/specification-support.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/overview/specification-support.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -== MCP specification support - -MCP servers implement the open MCP protocol for tool exposure. Only the tool concept from the MCP server specification is supported. Features such as MCP resources and prompts are not yet available. - -For full details, see the link:https://modelcontextprotocol.io/specification/2025-06-18/server[official MCP server specification^]. diff --git a/modules/ai-agents/partials/mcp/overview/use-cases-table.adoc b/modules/ai-agents/partials/mcp/overview/use-cases-table.adoc deleted file mode 100644 index 5bfe466f..00000000 --- a/modules/ai-agents/partials/mcp/overview/use-cases-table.adoc +++ /dev/null @@ -1,45 +0,0 @@ -// ============================================================================= -// PARTIAL: use-cases-table.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/overview.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/overview.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/overview/use-cases-table.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/overview/use-cases-table.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -== Use cases - -[cols="1s,3a"] -|=== -|Category |Example prompts - -|Operational monitoring -|* Check partition lag for customer-events topic -* Show me the top 10 producers by message volume today -* Get schema registry health status - -|Data enrichment and analysis -|* Fetch user profile data and recent orders for customer ID 12345 -* Get real-time stock prices for symbols in my portfolio topic -* Analyze sentiment of latest product reviews - -|Team productivity -|* Deploy my microservice to the staging environment -* Generate load test data for the payments service -* Create a summary dashboard of this week's incident reports - -|Business intelligence -|* What are the trending products in the last 24 hours? -* Show revenue impact of the latest feature deployment -* Get customer satisfaction scores from support tickets -|=== diff --git a/modules/ai-agents/partials/mcp/overview/what-is-mcp.adoc b/modules/ai-agents/partials/mcp/overview/what-is-mcp.adoc deleted file mode 100644 index 5993fbd1..00000000 --- a/modules/ai-agents/partials/mcp/overview/what-is-mcp.adoc +++ /dev/null @@ -1,22 +0,0 @@ -// ============================================================================= -// PARTIAL: what-is-mcp.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/overview.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/overview.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/overview/what-is-mcp.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/overview/what-is-mcp.adoc[] -// -// ATTRIBUTES USED: -// None -// -// DEPENDENCIES: -// None -// ============================================================================= - -MCP (Model Context Protocol) is an open standard that lets AI agents use tools. Think of it like a universal adapter: instead of building custom integrations for every AI system, you define your tools once using MCP, and any MCP-compatible AI client can discover and use them. - -Without MCP, connecting AI to your business systems requires custom API code, authentication handling, and response formatting for each AI platform. With MCP, you describe what a tool does and what inputs it needs, and the protocol handles the rest. diff --git a/modules/ai-agents/partials/mcp/tool-patterns/error-handling.adoc b/modules/ai-agents/partials/mcp/tool-patterns/error-handling.adoc deleted file mode 100644 index d887d69b..00000000 --- a/modules/ai-agents/partials/mcp/tool-patterns/error-handling.adoc +++ /dev/null @@ -1,97 +0,0 @@ -// ============================================================================= -// PARTIAL: error-handling.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/tool-patterns.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/tool-patterns.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/tool-patterns/error-handling.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/error-handling.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches xref targets for Cloud context -// -// DEPENDENCIES: -// - example$best-practices/error-handling/*.yaml -// ============================================================================= - -Wrap operations that can fail in `try`/`catch` blocks. This ensures the tool returns useful errors instead of failing silently. - -[source,yaml] ----- -include::example$best-practices/error-handling/basic-try-catch.yaml[tags=basic-try-catch,indent=0] ----- - -ifdef::env-cloud[] -For full configuration options, see xref:develop:connect/components/processors/try.adoc[`try` processor] and xref:develop:connect/components/processors/catch.adoc[`catch` processor]. -endif::[] -ifndef::env-cloud[] -For full configuration options, see xref:components:processors/try.adoc[`try` processor] and xref:components:processors/catch.adoc[`catch` processor]. -endif::[] - -=== Return error details - -ifdef::env-cloud[] -The xref:develop:connect/guides/bloblang/functions.adoc#error[`error()`] function returns the error message from the most recent failure. Use it in `catch` blocks to capture what went wrong: -endif::[] -ifndef::env-cloud[] -The xref:guides:bloblang/functions.adoc#error[`error()`] function returns the error message from the most recent failure. Use it in `catch` blocks to capture what went wrong: -endif::[] - -[source,yaml] ----- -include::example$best-practices/error-handling/error-with-timestamp.yaml[tags=error-with-timestamp,indent=0] ----- - -=== Set timeouts - -Always set explicit timeouts on external calls to prevent tools from hanging indefinitely: - -[source,yaml] ----- -include::example$best-practices/error-handling-snippets/timeout-config.yaml[tags=timeout-config,indent=0] ----- - -ifdef::env-cloud[] -For all timeout and retry options, see xref:develop:connect/components/processors/http.adoc[`http` processor]. -endif::[] -ifndef::env-cloud[] -For all timeout and retry options, see xref:components:processors/http.adoc[`http` processor]. -endif::[] - -=== Handle specific error types - -Create different responses based on error type: - -[source,yaml] ----- -include::example$best-practices/error-handling/handle-error-types.yaml[tags=handle-error-types,indent=0] ----- - -=== Log errors for debugging - -Add logging inside `catch` blocks to aid troubleshooting: - -[source,yaml] ----- -include::example$best-practices/error-handling/log-errors.yaml[tags=log-errors,indent=0] ----- - -ifdef::env-cloud[] -For log level options, see xref:develop:connect/components/processors/log.adoc[`log` processor]. -endif::[] -ifndef::env-cloud[] -For log level options, see xref:components:processors/log.adoc[`log` processor]. -endif::[] - -=== Preserve input context in errors - -Include original input data in error responses to help AI clients retry with corrections: - -[source,yaml] ----- -include::example$best-practices/error-handling/preserve-input-context.yaml[tags=preserve-input-context,indent=0] ----- - diff --git a/modules/ai-agents/partials/mcp/tool-patterns/input-validation.adoc b/modules/ai-agents/partials/mcp/tool-patterns/input-validation.adoc deleted file mode 100644 index 14e0d887..00000000 --- a/modules/ai-agents/partials/mcp/tool-patterns/input-validation.adoc +++ /dev/null @@ -1,149 +0,0 @@ -// ============================================================================= -// PARTIAL: input-validation.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/tool-patterns.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/tool-patterns.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/tool-patterns/input-validation.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/input-validation.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches xref targets for Cloud context -// -// DEPENDENCIES: -// - example$best-practices/input-validation/*.yaml -// ============================================================================= - -Always validate inputs before processing. This prevents errors and provides clear feedback to the AI client. The following example shows a basic validation pattern: - -[source,yaml] ----- -include::example$best-practices/input-validation/validate-required-field.yaml[tags=validate-input,indent=0] ----- - -This validation does three things: First, `.or("")` provides an empty string default if the `city` field is missing, which prevents null errors. Then, `.trim()` removes whitespace so `" "` doesn't pass as a valid city. Then, the `if` expression returns either an error object or the validated data. The AI client receives clear feedback either way. - -=== Essential validation methods - -ifdef::env-cloud[] -Use these xref:develop:connect/guides/bloblang/methods.adoc[Bloblang methods] for input validation: -endif::[] -ifndef::env-cloud[] -Use these xref:guides:bloblang/methods.adoc[Bloblang methods] for input validation: -endif::[] - -[cols="1,2,2"] -|=== -| Method | Purpose | Example - -| `.or(default)` -| Provide fallback for missing fields -| `this.city.or("unknown")` - -| `.trim()` -| Remove leading/trailing whitespace -| `this.name.trim()` - -| `.exists("field")` -| Check if a field is present -| `this.exists("email")` - -| `.type()` -| Get the type of a value -| `this.count.type() == "number"` - -| `.length()` -| Check string or array length -| `this.items.length() > 0` - -| `.re_match(pattern)` -| Validate against regex -| `this.email.re_match("^[^@]+@[^@]+$")` - -| `.number()` -| Convert and validate as number -| `this.quantity.number()` -|=== - -=== Sanitize string inputs - -Remove potentially dangerous characters from user inputs. This is especially important when inputs will be used in URLs, database queries, or shell commands: - -[source,yaml] ----- -include::example$best-practices/input-validation/sanitize-string-input.yaml[tags=sanitize-input,indent=0] ----- - -The regex `[^a-zA-Z\\s\\-]` matches any character that is not a letter, space, or hyphen, and `re_replace_all` removes all matches. An input like `"New York!@#$"` becomes `"New York"`. The `meta` keyword stores the result in message metadata (using `@sanitized_city`), keeping it separate from the message body until validation passes. - -ifdef::env-cloud[] -For regex replacement syntax, see xref:develop:connect/guides/bloblang/methods.adoc#re_replace_all[`re_replace_all`]. -endif::[] -ifndef::env-cloud[] -For regex replacement syntax, see xref:guides:bloblang/methods.adoc#re_replace_all[`re_replace_all`]. -endif::[] - -=== Validate numeric ranges - -Check that numeric inputs fall within acceptable bounds: - -[source,yaml] ----- -include::example$best-practices/input-validation/validate-numeric-range.yaml[tags=validate-quantity,indent=0] ----- - -This example chains `.or(0)` with `.number()` to handle both missing values and type conversion. The chained `if`/`else if` checks both lower and upper bounds. Including the received value in error responses helps AI clients understand what went wrong and correct their input. - -=== Validate multiple fields - -For forms or complex inputs, collect all errors before returning. This gives AI clients a complete list of problems to fix rather than failing on the first error: - -[source,yaml] ----- -include::example$best-practices/input-validation/validate-multiple-fields.yaml[tags=validate-order,indent=0] ----- - -The pattern uses variable reassignment (`let errors = ...`) to accumulate errors into an array. Each check appends to the array if validation fails, or returns the unchanged array if it passes. At the end, if any errors were collected, the response includes all of them. Notice that the email validation only runs if the field exists. This allows optional fields that, when provided, must be valid. - -=== Validate enum values - -Restrict inputs to a set of allowed values. This prevents invalid states and provides helpful feedback when the input doesn't match: - -[source,yaml] ----- -include::example$best-practices/input-validation/validate-enum-values.yaml[tags=validate-status,indent=0] ----- - -The `lowercase()` call normalizes the input so `"PENDING"`, `"Pending"`, and `"pending"` all match. When validation fails, the error response includes the list of allowed values. This helps AI clients self-correct without needing to look up valid options. - -ifdef::env-cloud[] -For more details, see xref:develop:connect/guides/bloblang/methods.adoc#contains[`contains`] and xref:develop:connect/guides/bloblang/methods.adoc#lowercase[`lowercase`]. -endif::[] -ifndef::env-cloud[] -For more details, see xref:guides:bloblang/methods.adoc#contains[`contains`] and xref:guides:bloblang/methods.adoc#lowercase[`lowercase`]. -endif::[] - -=== Use throw for validation failures - -ifdef::env-cloud[] -Use xref:develop:connect/guides/bloblang/functions.adoc#throw[`throw()`] to stop processing with an error message. This is useful when validation failure should stop the entire tool execution: -endif::[] -ifndef::env-cloud[] -Use xref:guides:bloblang/functions.adoc#throw[`throw()`] to stop processing with an error message. This is useful when validation failure should stop the entire tool execution: -endif::[] - -[source,yaml] ----- -- label: require_auth - mutation: | - root = if !this.exists("api_key") || this.api_key == "" { - throw("API key is required for this operation") - } else { - this - } ----- - -Unlike returning an error object, `throw()` immediately stops the processor chain and triggers any `catch` block that follows. Use `throw()` for critical validation failures where continuing would be pointless or dangerous. The `else` branch returns `this` unchanged, passing all input fields to the next processor. diff --git a/modules/ai-agents/partials/mcp/tool-patterns/production-workflows.adoc b/modules/ai-agents/partials/mcp/tool-patterns/production-workflows.adoc deleted file mode 100644 index fda5d263..00000000 --- a/modules/ai-agents/partials/mcp/tool-patterns/production-workflows.adoc +++ /dev/null @@ -1,153 +0,0 @@ -// ============================================================================= -// PARTIAL: production-workflows.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/tool-patterns.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/tool-patterns.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/tool-patterns/production-workflows.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/production-workflows.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches secrets partial and xref targets for Cloud context -// -// DEPENDENCIES: -// - partials/mcp/tool-patterns/secrets-cloud.adoc (included via ifdef) -// - partials/mcp/tool-patterns/secrets-connect.adoc (included via ifndef) -// - example$best-practices/production-workflows/*.yaml -// - example$resources/processors/*.yaml -// ============================================================================= - -== Advanced workflows - -Build multi-step workflows with dynamic configuration, conditional logic, and observability. - -=== Dynamic configuration - -Build tools that adapt their behavior based on input parameters: - -[source,yaml] ----- -include::example$best-practices/production-workflows/dynamic-config.yaml[tags=dynamic-config] ----- - -=== Conditional processing - -Build tools that branch based on input or data characteristics: - -[source,yaml] ----- -include::example$best-practices/production-workflows/conditional-processing.yaml[tags=conditional-processing] ----- - -// Secrets section - deployment-specific -ifdef::env-cloud[] -include::ai-agents:partial$mcp/tool-patterns/secrets-cloud.adoc[] -endif::[] -ifndef::env-cloud[] -include::ai-agents:partial$mcp/tool-patterns/secrets-connect.adoc[] -endif::[] - -=== Monitoring, debugging, and observability - -Use structured logging, request tracing, and performance metrics to gain insights into tool execution. - -[source,yaml] ----- -include::ai-agents:example$resources/processors/observable-tool.yaml[] ----- - -Observability features: - -* *Correlation IDs*: Use `uuid_v7()` to generate unique request identifiers for tracing -* *Execution timing*: Track how long your tools take to execute using nanosecond precision -* *Structured logging*: Include consistent fields like `request_id`, `duration_ms`, `tool_name` -* *Request/response metadata*: Log input parameters and response characteristics -* *Success tracking*: Monitor whether operations complete successfully - -You can test this pattern by invoking the tool with valid and invalid parameters, and observe the structured logs for tracing execution flow. For example, with a user ID of 1, you might see logs like: - -[source,json] ----- -{ - "metadata": { - "execution_time_ms": 0.158977, - "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", - "success": true, - "timestamp": "2025-09-16T08:37:18.589Z", - "tool": "observable_tool" - }, - "trace": { - "request_id": "019951ab-d07d-703f-aaae-7e1c9a5afa95", - "timestamp": "2025-09-16T08:37:18.589Z", - "tool": "observable_tool", - "version": "1.0.0" - }, - "user_id": "1" -} ----- - -See also: xref:components:processors/log.adoc[`log` processor], xref:components:processors/try.adoc[`try` processor], xref:guides:bloblang/functions.adoc[Bloblang functions] (for timing and ID generation) - -=== Multi-step data enrichment - -Build tools that combine data from multiple sources. - -This workflow fetches customer data from a SQL database, enriches it with recent order history, and computes summary metrics. - -[source,yaml] ----- -include::ai-agents:example$resources/processors/customer-enrichment.yaml[] ----- - -See also: xref:components:processors/sql_select.adoc[`sql_select` processor], xref:guides:bloblang/about.adoc[Bloblang functions] (for data manipulation and aggregations) - -=== Workflow orchestration - -Coordinate complex workflows with multiple steps and conditional logic. - -This workflow simulates a complete order processing pipeline with mock data for inventory and processing tiers. This allows you to test the full logic without needing real external systems. - -[source,yaml] ----- -include::ai-agents:example$resources/processors/order-workflow.yaml[] ----- - -For the input `{"order_id": "ORD001", "product_id": "widget-001", "quantity": 5, "total": 250, "customer_tier": "vip"}`, the workflow produces: - -[source,json] ----- -{ - "assigned_rep": "vip-team@company.com", - "available_quantity": 100, - "customer_tier": "vip", - "estimated_fulfillment": "TBD - calculated based on processing tier", - "inventory_check": "passed", - "order_id": "ORD001", - "order_status": "processed", - "perks": [ - "expedited_shipping", - "white_glove_service" - ], - "priority_score": 90, - "processed_at": "2025-09-16T09:05:29.138Z", - "processing_tier": "vip", - "processing_time_estimate": "1-2 hours", - "processing_time_hours": 2, - "product_id": "widget-001", - "product_name": "Standard Widget", - "quantity": 5, - "total": 250 -} ----- - -Notice how the workflow: - -. Preserves original input: `order_id`, `product_id`, `quantity`, `total`, and `customer_tier` pass through unchanged. -. Adds inventory data: `available_quantity`, `product_name`, and `inventory_check` status from the mock lookup. -. Routes by customer tier: Since `customer_tier` is `vip`, it gets VIP processing with special `perks` and priority. -. Enriches with processing metadata: `assigned_rep`, `priority_score`, `processing_tier`, and time estimates. -. Finalizes with timestamps: `order_status`, `processed_at`, and calculated `processing_time_hours`. - diff --git a/modules/ai-agents/partials/mcp/tool-patterns/response-formatting.adoc b/modules/ai-agents/partials/mcp/tool-patterns/response-formatting.adoc deleted file mode 100644 index c37c3f2c..00000000 --- a/modules/ai-agents/partials/mcp/tool-patterns/response-formatting.adoc +++ /dev/null @@ -1,189 +0,0 @@ -// ============================================================================= -// PARTIAL: response-formatting.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/tool-patterns.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/tool-patterns.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/tool-patterns/response-formatting.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/response-formatting.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches xref targets for Cloud context -// -// DEPENDENCIES: -// - example$best-practices/response-formatting/*.yaml -// ============================================================================= - -Structure responses consistently so AI clients can interpret them reliably. The following example takes a raw weather API response and transforms it into a clean, predictable format: - -[source,yaml] ----- -include::example$best-practices/response-formatting/format-basic-response.yaml[tags=format-response,indent=0] ----- - -This mapping does four things: - -* Extracts the city name from a nested `location.name` field -* Converts `temp_c` to a number type (APIs sometimes return numbers as strings) -* Pulls out the weather description text -* Adds a timestamp so the AI client knows when the data was fetched - -The result is a flat JSON object with predictable field names and types, rather than the raw API response which might have deeply nested structures or inconsistent formatting. - -=== Type coercion methods - -ifdef::env-cloud[] -Use xref:develop:connect/guides/bloblang/methods.adoc[type coercion methods] to ensure fields have the correct data types: -endif::[] -ifndef::env-cloud[] -Use xref:guides:bloblang/methods.adoc[type coercion methods] to ensure fields have the correct data types: -endif::[] - -[cols="1,2,2"] -|=== -| Method | Purpose | Example - -| `.string()` -| Convert to string -| `this.id.string()` becomes `"123"` - -| `.number()` -| Convert to number -| `this.price.number()` becomes `19.99` -| `.bool()` -| Convert to boolean -| `this.active.bool()` becomes `true` - -| `.int64()` -| Convert to 64-bit integer -| `this.count.int64()` becomes `42` -| `.float64()` -| Convert to 64-bit float -| `this.ratio.float64()` becomes `0.75` -|=== - -=== Format timestamps - -ifdef::env-cloud[] -Use xref:develop:connect/guides/bloblang/functions.adoc#now[`now()`] with xref:develop:connect/guides/bloblang/methods.adoc#format_timestamp[`format_timestamp()`] for consistent time formatting: -endif::[] -ifndef::env-cloud[] -Use xref:guides:bloblang/functions.adoc#now[`now()`] with xref:guides:bloblang/methods.adoc#format_timestamp[`format_timestamp()`] for consistent time formatting: -endif::[] - -[source,yaml] ----- -include::example$best-practices/response-formatting/add-timestamps.yaml[tags=add-timestamps,indent=0] ----- - -This example preserves all existing fields (`root = this`) and adds three timestamp fields. The `now()` function returns the current time, and `format_timestamp()` converts it to a string. Each field uses a different format: full ISO 8601 timestamp, date only, and time only. - -The format string uses Go's reference time layout. Common patterns: - -[cols="1,2"] -|=== -| Format | Output example - -| `"2006-01-02T15:04:05Z07:00"` -| `2024-03-15T14:30:00-07:00` (ISO 8601) - -| `"2006-01-02"` -| `2024-03-15` - -| `"15:04:05"` -| `14:30:00` - -| `"Mon, 02 Jan 2006"` -| `Fri, 15 Mar 2024` -|=== - -=== Extract nested fields - -API responses often have deeply nested structures. Extract only the fields your AI client needs and flatten them into a simple object: - -[source,yaml] ----- -include::example$best-practices/response-formatting/extract-nested-fields.yaml[tags=extract-nested,indent=0] ----- - -This mapping navigates a nested structure like `{"data": {"user": {"id": 123, "profile": {"display_name": "..."}}}}` and creates a flat response. The dot notation (`this.data.user.id`) drills down through nested objects. Type coercion (`.string()`, `.bool()`) ensures consistent output types. - -ifdef::env-cloud[] -For navigating nested structures, see xref:develop:connect/guides/bloblang/about.adoc#dot-notation[dot notation]. -endif::[] -ifndef::env-cloud[] -For navigating nested structures, see xref:guides:bloblang/about.adoc#dot-notation[dot notation]. -endif::[] - -=== Handle arrays - -When your data contains arrays, you can transform each element, extract specific items, or compute aggregates: - -[source,yaml] ----- -include::example$best-practices/response-formatting/handle-arrays.yaml[tags=handle-arrays,indent=0] ----- - -This example demonstrates four array operations: - -* `length()` returns the array size for the `total_items` count -* `map_each()` transforms each item into a new object with only the fields you need -* `index(0)` accesses the first element (zero-indexed) to get the first item's name -* A second `map_each()` extracts just the names into a simple string array - -ifdef::env-cloud[] -For array operations, see xref:develop:connect/guides/bloblang/methods.adoc#map_each[`map_each()`] and xref:develop:connect/guides/bloblang/methods.adoc#index[`index()`]. -endif::[] -ifndef::env-cloud[] -For array operations, see xref:guides:bloblang/methods.adoc#map_each[`map_each()`] and xref:guides:bloblang/methods.adoc#index[`index()`]. -endif::[] - -=== Include fields conditionally - -Sometimes you want to include fields only when they have meaningful values. This avoids returning `null` or empty fields that clutter the response: - -[source,yaml] ----- -include::example$best-practices/response-formatting/conditional-fields.yaml[tags=conditional-fields,indent=0] ----- - -This mapping starts with required fields (`id`, `name`), then conditionally adds optional fields. The `exists()` check prevents errors when accessing missing fields. When the condition is false, `deleted()` removes the field entirely from the output. The AI client won't see `"email": null`. The field simply won't exist. - -ifdef::env-cloud[] -The xref:develop:connect/guides/bloblang/functions.adoc#deleted[`deleted()`] function removes the field from the output entirely. -endif::[] -ifndef::env-cloud[] -The xref:guides:bloblang/functions.adoc#deleted[`deleted()`] function removes the field from the output entirely. -endif::[] - -=== Filter sensitive data - -When your data source contains sensitive fields, strip them before returning responses to the AI client: - -[source,yaml] ----- -include::example$best-practices/response-formatting/filter-sensitive-data.yaml[tags=filter-sensitive,indent=0] ----- - -The `without()` method creates a copy of the object with the specified fields removed. This is safer than manually selecting fields because new fields added to the source data are included automatically, so you only need to maintain the exclusion list. Use this when returning database records or API responses that might contain credentials or personal information. - -ifdef::env-cloud[] -For field removal, see xref:develop:connect/guides/bloblang/methods.adoc#without[`without()`]. -endif::[] -ifndef::env-cloud[] -For field removal, see xref:guides:bloblang/methods.adoc#without[`without()`]. -endif::[] - -=== Wrap responses in a success envelope - -When AI clients call multiple tools, they need a predictable way to check if the call succeeded. Wrapping responses in a consistent envelope structure makes this easy: - -[source,yaml] ----- -include::example$best-practices/response-formatting/success-envelope.yaml[tags=success-envelope,indent=0] ----- - -Both success and error responses share the same top-level structure: a `success` boolean, a payload field (`data` or `error`), and a `timestamp`. The AI client can check `success` first, then access the appropriate field. The error response uses the `catch` processor to handle failures and the `error()` function to capture the error message. diff --git a/modules/ai-agents/partials/mcp/tool-patterns/secrets-cloud.adoc b/modules/ai-agents/partials/mcp/tool-patterns/secrets-cloud.adoc deleted file mode 100644 index f16b9b43..00000000 --- a/modules/ai-agents/partials/mcp/tool-patterns/secrets-cloud.adoc +++ /dev/null @@ -1,41 +0,0 @@ -// ============================================================================= -// PARTIAL: secrets-cloud.adoc -// ============================================================================= -// -// INCLUDED BY: -// - This partial: partials/mcp/tool-patterns/production-workflows.adoc (via ifdef::env-cloud[]) -// -// INCLUDE SYNTAX: -// include::ai-agents:partial$mcp/tool-patterns/secrets-cloud.adoc[] -// -// ATTRIBUTES USED: -// None (only included in Cloud context) -// -// DEPENDENCIES: -// None -// ============================================================================= - -[[secrets]] -=== Secrets and credentials - -Securely handle multiple credentials and API keys. - -Here is an example of using an API key secret. - -. Create a secret in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] with name `EXTERNAL_API_KEY` and your API key as the value. - -. Reference the secret in your YAML configuration: -+ -[source,yaml] ----- -processors: - - label: call_external_api - http: - url: "https://api.example.com/data" - verb: GET - headers: - Authorization: "Bearer ${secrets.EXTERNAL_API_KEY}" # <1> - Accept: "application/json" ----- -+ -<1> The secret is injected at runtime. Never store the actual API key in your YAML configuration. The actual secret value never appears in your configuration files or logs. diff --git a/modules/ai-agents/partials/mcp/tool-patterns/secrets-connect.adoc b/modules/ai-agents/partials/mcp/tool-patterns/secrets-connect.adoc deleted file mode 100644 index 7f926927..00000000 --- a/modules/ai-agents/partials/mcp/tool-patterns/secrets-connect.adoc +++ /dev/null @@ -1,46 +0,0 @@ -// ============================================================================= -// PARTIAL: secrets-connect.adoc -// ============================================================================= -// -// INCLUDED BY: -// - This partial: partials/mcp/tool-patterns/production-workflows.adoc (via ifndef::env-cloud[]) -// -// INCLUDE SYNTAX: -// include::ai-agents:partial$mcp/tool-patterns/secrets-connect.adoc[] -// -// ATTRIBUTES USED: -// None (only included in Connect context) -// -// DEPENDENCIES: -// None -// ============================================================================= - -[[secrets]] -=== Secrets and credentials - -Securely handle multiple credentials and API keys using environment variables. - -Here is an example of using an API key from environment variables. - -. Set an environment variable with your API key: -+ -[source,bash] ----- -export EXTERNAL_API_KEY="your-api-key-here" ----- - -. Reference the environment variable in your configuration: -+ -[source,yaml] ----- -processors: - - label: call_external_api - http: - url: "https://api.example.com/data" - verb: GET - headers: - Authorization: "Bearer ${EXTERNAL_API_KEY}" # <1> - Accept: "application/json" ----- -+ -<1> The environment variable is injected at runtime. Never store the actual API key in your YAML. The actual secret value never appears in your configuration files or logs. diff --git a/modules/ai-agents/partials/mcp/troubleshooting/debugging-techniques.adoc b/modules/ai-agents/partials/mcp/troubleshooting/debugging-techniques.adoc deleted file mode 100644 index 4a15ae4b..00000000 --- a/modules/ai-agents/partials/mcp/troubleshooting/debugging-techniques.adoc +++ /dev/null @@ -1,76 +0,0 @@ -// ============================================================================= -// PARTIAL: debugging-techniques.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/troubleshooting.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/troubleshooting.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/troubleshooting/debugging-techniques.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/debugging-techniques.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches xref targets and testing instructions for Cloud context -// -// DEPENDENCIES: -// None -// ============================================================================= - -=== Add temporary logging - -ifdef::env-cloud[] -Insert xref:develop:connect/components/processors/log.adoc[`log` processors] to debug data flow: -endif::[] -ifndef::env-cloud[] -Insert xref:components:processors/log.adoc[`log` processors] to debug data flow: -endif::[] - -[source,yaml] ----- -processors: - - log: - message: "Input received: ${! json() }" - level: DEBUG - - # ... your processing logic ... - - log: - message: "Output produced: ${! json() }" - level: DEBUG ----- - -ifdef::env-cloud[] -The `${! json() }` syntax uses xref:develop:connect/guides/bloblang/functions.adoc#json[Bloblang interpolation] to insert the current message content. -endif::[] -ifndef::env-cloud[] -The `${! json() }` syntax uses xref:guides:bloblang/functions.adoc#json[Bloblang interpolation] to insert the current message content. -endif::[] - -Remove debug processors before deploying to production. - -=== Test your tools - -Build confidence by testing at each stage: - -ifdef::env-cloud[] -. Lint your configuration using the *Lint* button in the Cloud Console. -. Test tool logic using the *MCP Inspector*. -. Connect to your AI client using `rpk cloud mcp proxy`. -. Test end-to-end with realistic prompts. -endif::[] -ifndef::env-cloud[] -. Lint your configuration with `rpk connect mcp-server lint`. -. Start the server and test with cURL or an MCP client. -. Connect to your AI client and verify the tool appears. -. Test end-to-end with realistic prompts. - -For the complete testing workflow with cURL scripts, see xref:ai-agents:mcp-server/create-tool.adoc#_test_the_tool[Test the tool]. -endif::[] - -=== Isolate the problem - -When debugging complex tools: - -. Test each processor individually by commenting out others. -. Use static test data instead of live API calls. -. Check if the issue is in input validation, processing logic, or output formatting. -. Compare working tools with broken ones to identify differences. diff --git a/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-connection.adoc b/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-connection.adoc deleted file mode 100644 index 6ca3b314..00000000 --- a/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-connection.adoc +++ /dev/null @@ -1,98 +0,0 @@ -// ============================================================================= -// PARTIAL: troubleshooting-connection.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/troubleshooting.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/troubleshooting.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/troubleshooting/troubleshooting-connection.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/troubleshooting-connection.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches troubleshooting steps for Cloud vs self-managed -// -// DEPENDENCIES: -// None -// ============================================================================= - -=== MCP client can't connect to server - -ifdef::env-cloud[] -If your MCP client can't connect to your Remote MCP server: - -. Verify authentication: -+ --- -* Run `rpk cloud login` to refresh your authentication token. -* Tokens expire after 1 hour. --- - -. Check the MCP proxy configuration: -+ --- -* Verify the cluster ID and MCP server ID are correct. -* Run `rpk cloud mcp proxy --help` to see available options. --- - -. Verify the MCP server is running: -+ --- -* Check the server status in the Cloud Console. -* Review the *Logs* tab for startup errors. --- -endif::[] - -ifndef::env-cloud[] -If your MCP client can't connect to your local MCP server: - -. Verify the server is running: -+ --- -* Check that the MCP server process is still running in your terminal. -* Look for "Registering processor tool" log messages at startup. -* Confirm the server is listening on the expected address (default: `localhost:4195`). --- - -. Check for port conflicts: -+ --- -If port 4195 is already in use, specify a different port: - -[source,bash] ----- -rpk connect mcp-server --address localhost:4196 --tag my-tag ----- - -Then update your `mcp-remote` connection: - -[source,bash] ----- -claude mcp add local -- npx mcp-remote http://localhost:4196/sse ----- --- - -. Verify the SSE endpoint: -+ --- -* The MCP client connects to `/sse` endpoint (for example, `http://localhost:4195/sse`). -* Test the endpoint directly: `curl http://localhost:4195/sse` --- -endif::[] - -=== Connection drops or times out - -If connections are unstable: - -. Check network connectivity between the client and server. - -. Verify no firewall rules are blocking the connection. - -ifdef::env-cloud[] -. Check if the MCP server is being restarted due to resource limits. Consider scaling up resources. -endif::[] - -ifndef::env-cloud[] -. Ensure the MCP server process isn't being killed by the OS (check system logs). -endif::[] diff --git a/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-lint.adoc b/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-lint.adoc deleted file mode 100644 index db2881fa..00000000 --- a/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-lint.adoc +++ /dev/null @@ -1,106 +0,0 @@ -// ============================================================================= -// PARTIAL: troubleshooting-lint.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/troubleshooting.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/troubleshooting.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/troubleshooting/troubleshooting-lint.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/troubleshooting-lint.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches lint instructions and upgrade guidance for Cloud context -// -// DEPENDENCIES: -// None -// ============================================================================= - -ifdef::env-cloud[] -Always lint your configuration before deploying. The Cloud Console provides a *Lint* button that validates your YAML. -endif::[] -ifndef::env-cloud[] -Always lint your configuration before starting the server: - -[source,bash] ----- -rpk connect mcp-server lint ----- -endif::[] - -=== Common lint errors - -- `unable to infer component type`: Your file contains multiple component types or uses wrapper blocks. Each YAML file must contain only a single component type and should not be wrapped in an `input:` or `output:` block. See <>. - -- `unknown field`: A configuration field is misspelled. Check the field name against the component documentation. - -- `missing required field`: A required field is missing from your configuration. Add the missing field. - -[[fix-unable-to-infer]] -=== Unable to infer component type - -If you see errors like the following, your YAML file contains more than one component type or uses a wrapper: - -[source] ----- -resources/inputs/redpanda-consume.yaml(1,1) unable to infer component type: [input processors cache_resources meta] -resources/outputs/redpanda-publish.yaml(1,1) unable to infer component type: [processors output meta] ----- - -To fix this, split out each component type into its own file. - -.Incorrect: Multiple component types -[source,yaml] ----- -label: incorrect-example -input: - redpanda: { ... } -processors: - - mutation: { ... } -output: - redpanda: { ... } ----- - -.Correct: Single component type -[source,yaml] ----- -label: event-reader -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "events" ] - consumer_group: "mcp-reader" -meta: - mcp: - enabled: true - description: "Consume events from Redpanda" ----- - -=== JSON schema errors - -JSON schema errors indicate that you're using an outdated version of Redpanda Connect with an incompatible JSON schema format: - -[source,json] ----- -{ - "type": "error", - "error": { - "type": "invalid_request_error", - "message": "tools.17.custom.input_schema: JSON schema is invalid..." - } -} ----- - -ifdef::env-cloud[] -Contact Redpanda support if you see this error in Redpanda Cloud. -endif::[] -ifndef::env-cloud[] -Upgrade to at least version 4.66.1 of Redpanda Connect: - -[source,bash] ----- -rpk connect --version ----- - -If you need to upgrade, see xref:install:rpk.adoc#upgrade[Upgrade Redpanda Connect]. -endif::[] diff --git a/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-runtime.adoc b/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-runtime.adoc deleted file mode 100644 index 116d8e7a..00000000 --- a/modules/ai-agents/partials/mcp/troubleshooting/troubleshooting-runtime.adoc +++ /dev/null @@ -1,81 +0,0 @@ -// ============================================================================= -// PARTIAL: troubleshooting-runtime.adoc -// ============================================================================= -// -// INCLUDED BY: -// - rp-connect-docs: modules/ai-agents/pages/mcp-server/troubleshooting.adoc -// - cloud-docs: modules/ai-agents/pages/mcp/remote/troubleshooting.adoc -// -// INCLUDE SYNTAX: -// Connect: include::ai-agents:partial$mcp/troubleshooting/troubleshooting-runtime.adoc[] -// Cloud: include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/troubleshooting-runtime.adoc[] -// -// ATTRIBUTES USED: -// - env-cloud: Switches runtime troubleshooting steps for Cloud context -// -// DEPENDENCIES: -// None -// ============================================================================= - -=== Tool not appearing in MCP client - -If your tool doesn't appear in the MCP client's tool list: - -. Verify that `meta.mcp.enabled: true` is set in your YAML configuration. - -. Check the tool has the correct tag: -+ --- -ifdef::env-cloud[] -* Verify the MCP server status shows *Running* in the Cloud Console. -* Check the *Logs* tab for any startup errors. -endif::[] -ifndef::env-cloud[] -* Ensure the tool's `meta.tags` array includes the tag you specified when starting the server. -* Example: If you started with `--tag bluesky`, your tool needs `tags: [ bluesky ]`. -endif::[] --- - -. Verify correct directory structure: -+ --- -ifndef::env-cloud[] -* Processors: `resources/processors/` -* Inputs: `resources/inputs/` -* Outputs: `resources/outputs/` -* Caches: `resources/caches/` -endif::[] --- - -.Example correct structure -[source,yaml] ----- -label: my-tool -# ... component configuration ... -meta: - tags: [ my-tag ] # Must match --tag argument - mcp: - enabled: true # Required for exposure - description: Tool description ----- - -=== Tool returns unexpected results - -If your tool runs but returns unexpected data: - -. Check input validation. Add logging to see what inputs the tool receives: -+ -[source,yaml] ----- -- log: - message: "Received input: ${! json() }" - level: DEBUG ----- - -. Verify data transformations. Log intermediate results between processors. - -. Check external API responses. The API may return different data than expected. - -ifdef::env-cloud[] -. Review the *Logs* tab in the Cloud Console for error messages. -endif::[] diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index c438e9ba..0e455cb3 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -58,6 +58,17 @@ This release adds support for the following new fields: |=== +=== Deprecations and removals + +The MCP (Model Context Protocol) server feature in Redpanda Connect has been deprecated and removed. This includes: + +- All MCP server documentation +- The following CLI commands: +** `rpk connect mcp-server` +** `rpk connect mcp-server init` +** `rpk connect mcp-server lint` + + == Version 4.87.0 link:https://github.com/redpanda-data/connect/releases/tag/v4.87.0[See the full release notes^]. @@ -1233,7 +1244,6 @@ For instructions, see: === Model Context Protocol (MCP) server You can now build and run an MCP server directly in Redpanda Connect. The MCP server lets you expose your Redpanda Connect configurations as AI-consumable HTTP endpoints, making them discoverable and callable by AI clients such as Claude Code and other compatible agents. -To get started, see xref:ai-agents:mcp-server/quickstart.adoc[]. == Version 4.64.0