ci(conformance): integrate MCP server conformance test pipeline

.github/workflows/conformance.yml

@@ -0,0 +1,78 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: conformance
+on:
+  pull_request:
+  pull_request_target:
+    types: [labeled]
+
+permissions: read-all
+
+jobs:
+  conformance:
+    if: "${{ github.event.action != 'labeled' || github.event.label.name == 'tests: run' }}"
+    name: conformance
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    permissions:
+      contents: 'read'
+      issues: 'write'
+      pull-requests: 'write'
+    steps:
+      - name: Remove PR Label
+        if: "${{ github.event.action == 'labeled' && github.event.label.name == 'tests: run' }}"
+        uses: actions/github-script@v9
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            try {
+              await github.rest.issues.removeLabel({
+                name: 'tests: run',
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.payload.pull_request.number
+              });
+            } catch (e) {
+              console.log('Failed to remove label. Another job may have already removed it!');
+            }
+
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+
+      - name: Start MCP Toolbox Server
+        run: |
+          echo "sources: {}" > tools.yaml
+          go build -o toolbox
+          ./toolbox &
+          sleep 5 # Give the server a moment to bind to port 5000
+
+      - name: Run Conformance Tests
+        uses: modelcontextprotocol/conformance@main
+        with:
+          mode: server
+          url: 'http://localhost:5000/mcp'
+          suite: active
+          expected-failures: tests/conformance-baseline.yml

.github/workflows/nightly_tier_report.yml

@@ -0,0 +1,111 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Nightly Server Tier Assessment
+
+on:
+  schedule:
+    - cron: '0 6 * * *'  # Runs at 6 AM every morning
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  tier-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+
+      - name: Start MCP Toolbox Server
+        run: |
+          echo "sources: {}" > tools.yaml
+          go build -o toolbox
+          ./toolbox &
+          sleep 5 # Give the server a moment to bind to port 5000
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Run Server Tier Assessment
+        id: assessment
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          npx --yes @modelcontextprotocol/conformance@latest tier-check --repo ${{ github.repository }} --output markdown --conformance-server-url "http://localhost:5000/mcp" > tier_report.txt
+        continue-on-error: true
+
+      - name: Report Tier Status
+        uses: actions/github-script@v9
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const report = fs.readFileSync('tier_report.txt', 'utf8');
+            const title = 'Nightly Server Tier Assessment Failure';
+            const periodicLabel = 'periodic-failure';
+
+            const prevIssues = await github.rest.issues.listForRepo({
+              ...context.repo,
+              state: 'open',
+              creator: 'github-actions[bot]',
+              labels: [periodicLabel]
+            });
+
+            let existingIssue = prevIssues.data.find(issue => issue.title === title);
+
+            if (report.includes('Tier Assessment: Tier 1')) {
+              console.log('Tier 1 achieved! Closing open issue if it exists.');
+              if (existingIssue) {
+                await github.rest.issues.createComment({
+                  ...context.repo,
+                  issue_number: existingIssue.number,
+                  body: "🎉 The nightly Server Tier Assessment confirms we are now **Tier 1**! Closing this issue."
+                });
+                await github.rest.issues.update({
+                  ...context.repo,
+                  issue_number: existingIssue.number,
+                  state: 'closed'
+                });
+              }
+              return;
+            }
+
+            const body = `The nightly Server Tier Assessment found gaps preventing Tier 1 status:\n\n${report}\n\nPlease fix these operational gaps.`;
+
+            if (existingIssue) {
+              console.log(`Found previous issue ${existingIssue.html_url}, updating body`);
+              await github.rest.issues.update({
+                ...context.repo,
+                issue_number: existingIssue.number,
+                body: body
+              });
+            } else {
+              console.log('No previous issue found, creating one');
+              await github.rest.issues.create({
+                ...context.repo,
+                title: title,
+                body: body,
+                labels: [periodicLabel]
+              });
+            }

tests/conformance-baseline.yml

@@ -0,0 +1,42 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+server:
+  - logging-set-level
+  - completion-complete
+  - tools-list
+  - tools-call-simple-text
+  - tools-call-image
+  - tools-call-audio
+  - tools-call-embedded-resource
+  - tools-call-mixed-content
+  - tools-call-with-logging
+  - tools-call-error
+  - tools-call-with-progress
+  - tools-call-sampling
+  - tools-call-elicitation
+  - elicitation-sep1034-defaults
+  - server-sse-multiple-streams
+  - elicitation-sep1330-enums
+  - resources-list
+  - resources-read-text
+  - resources-read-binary
+  - resources-templates-read
+  - resources-subscribe
+  - resources-unsubscribe
+  - prompts-get-simple
+  - prompts-get-with-args
+  - prompts-get-embedded-resource
+  - prompts-get-with-image
+  - dns-rebinding-protection