diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000000..5c76e02c919 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,77 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: conformance +on: + pull_request: + pull_request_target: + types: [labeled] + +permissions: read-all + +jobs: + conformance: + if: "${{ github.event.action != 'labeled' || github.event.label.name == 'tests: run' }}" + name: conformance + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: + contents: 'read' + issues: 'write' + pull-requests: 'write' + steps: + - name: Remove PR Label + if: "${{ github.event.action == 'labeled' && github.event.label.name == 'tests: run' }}" + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + try { + await github.rest.issues.removeLabel({ + name: 'tests: run', + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number + }); + } catch (e) { + console.log('Failed to remove label. Another job may have already removed it!'); + } + + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23' + + - name: Start MCP Toolbox Server + run: | + go build -o toolbox + ./toolbox --config tests/tools.yaml & + sleep 5 # Give the server a moment to bind to port 5000 + + - name: Run Conformance Tests + uses: modelcontextprotocol/conformance@main + with: + mode: server + url: 'http://localhost:5000/mcp' + suite: active + expected-failures: tests/conformance-baseline.yml diff --git a/.github/workflows/nightly_tier_report.yml b/.github/workflows/nightly_tier_report.yml new file mode 100644 index 00000000000..aa8e43246cc --- /dev/null +++ b/.github/workflows/nightly_tier_report.yml @@ -0,0 +1,110 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Nightly Server Tier Assessment + +on: + schedule: + - cron: '0 6 * * *' # Runs at 6 AM every morning + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + tier-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23' + + - name: Start MCP Toolbox Server + run: | + go build -o toolbox + ./toolbox --config tests/tools.yaml & + sleep 5 # Give the server a moment to bind to port 5000 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Run Server Tier Assessment + id: assessment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npx --yes @modelcontextprotocol/conformance@latest tier-check --repo ${{ github.repository }} --output markdown --conformance-server-url "http://localhost:5000/mcp" > tier_report.txt + continue-on-error: true + + - name: Report Tier Status + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const report = fs.readFileSync('tier_report.txt', 'utf8'); + const title = 'Nightly Server Tier Assessment Failure'; + const periodicLabel = 'periodic-failure'; + + const prevIssues = await github.rest.issues.listForRepo({ + ...context.repo, + state: 'open', + creator: 'github-actions[bot]', + labels: [periodicLabel] + }); + + let existingIssue = prevIssues.data.find(issue => issue.title === title); + + if (report.includes('Tier Assessment: Tier 1')) { + console.log('Tier 1 achieved! Closing open issue if it exists.'); + if (existingIssue) { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: existingIssue.number, + body: "🎉 The nightly Server Tier Assessment confirms we are now **Tier 1**! Closing this issue." + }); + await github.rest.issues.update({ + ...context.repo, + issue_number: existingIssue.number, + state: 'closed' + }); + } + return; + } + + const body = `The nightly Server Tier Assessment found gaps preventing Tier 1 status:\n\n${report}\n\nPlease fix these operational gaps.`; + + if (existingIssue) { + console.log(`Found previous issue ${existingIssue.html_url}, updating body`); + await github.rest.issues.update({ + ...context.repo, + issue_number: existingIssue.number, + body: body + }); + } else { + console.log('No previous issue found, creating one'); + await github.rest.issues.create({ + ...context.repo, + title: title, + body: body, + labels: [periodicLabel] + }); + } diff --git a/tests/conformance-baseline.yml b/tests/conformance-baseline.yml new file mode 100644 index 00000000000..a82e0d74456 --- /dev/null +++ b/tests/conformance-baseline.yml @@ -0,0 +1,36 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server: + - logging-set-level + - completion-complete + - tools-call-image + - tools-call-audio + - tools-call-embedded-resource + - tools-call-mixed-content + - tools-call-with-logging + - tools-call-with-progress + - tools-call-sampling + - tools-call-elicitation + - elicitation-sep1034-defaults + - elicitation-sep1330-enums + - resources-list + - resources-read-text + - resources-read-binary + - resources-templates-read + - resources-subscribe + - resources-unsubscribe + - prompts-get-embedded-resource + - prompts-get-with-image + - dns-rebinding-protection diff --git a/tests/tools.yaml b/tests/tools.yaml new file mode 100644 index 00000000000..e1b5bd9d897 --- /dev/null +++ b/tests/tools.yaml @@ -0,0 +1,51 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: source +name: test-sqlite +type: sqlite +database: tests/test.db +--- +kind: tool +name: test_simple_text +type: sqlite-sql +source: test-sqlite +description: Test simple text +statement: SELECT 'This is a simple text response for testing.' AS text; +--- +kind: tool +name: test_error_handling +type: sqlite-sql +source: test-sqlite +description: Test error handling +statement: SELECT * FROM non_existent_table; +--- +kind: prompt +name: test_simple_prompt +description: Test simple prompt +messages: + - role: user + content: "This is a simple prompt for testing." +--- +kind: prompt +name: test_prompt_with_arguments +description: Test prompt with arguments +messages: + - role: user + content: "Prompt with arguments: arg1='{{.arg1}}', arg2='{{.arg2}}'" +arguments: + - name: "arg1" + description: "First test argument" + - name: "arg2" + description: "Second test argument"