feat(cli-finding-classifier): Phase a lint-screen evals infrastructure

pr-monitor-config.toml

@@ -36,7 +36,7 @@ push_command = "jj git push"
 # §A-2 P-0 (config opt-in) でこのセクションを追加し、master 上で classifier を
 # 有効化する。dogfood で問題が出た場合は revert PR で `enabled = false` に
 # 戻すのが kill-switch (§10.6 C 簡易版)。
-# 詳細: docs/local-llm-offload-analysis.md §A-2 / §10
+# 詳細: docs/local-llm-offload-history.md §A-2 / §10 (PR #130 で history へ分離)
 # ---------------------------------------------------------------------------
 [classifier]
 enabled = true

src/cli-finding-classifier/evals/files/eval1-unused-import.diff

@@ -0,0 +1,16 @@
+diff --git a/src/foo.rs b/src/foo.rs
+index 1111111..2222222 100644
+--- a/src/foo.rs
++++ b/src/foo.rs
+@@ -1,8 +1,10 @@
++use std::collections::HashMap;
+ use std::fs;
++use std::path::Path;
+
+ pub fn read_text(path: &str) -> std::io::Result<String> {
+     fs::read_to_string(path)
+ }
+
+ pub fn count_lines(s: &str) -> usize {
+     s.lines().count()
+ }

src/cli-finding-classifier/evals/files/eval2-deep-nesting.diff

@@ -0,0 +1,26 @@
+diff --git a/src/permission.rs b/src/permission.rs
+index 3333333..4444444 100644
+--- a/src/permission.rs
++++ b/src/permission.rs
+@@ -1,10 +1,28 @@
+ pub fn check(user: &User, resource: &Resource) -> bool {
+-    user.is_active() && resource.is_visible_to(user)
++    if user.is_active() {
++        if let Some(role) = user.role() {
++            if role.has_permission("read") {
++                if resource.is_published() {
++                    if !resource.is_archived() {
++                        if resource.owner_id() == user.id() {
++                            return true;
++                        } else {
++                            if resource.is_public() {
++                                return true;
++                            }
++                        }
++                    }
++                }
++            }
++        }
++    }
++    false
+ }

src/cli-finding-classifier/evals/files/eval3-magic-number.diff

@@ -0,0 +1,25 @@
+diff --git a/src/retry.rs b/src/retry.rs
+index 5555555..6666666 100644
+--- a/src/retry.rs
++++ b/src/retry.rs
+@@ -1,12 +1,18 @@
+ use std::time::Duration;
+ use std::thread::sleep;
+
+ pub fn retry_with_backoff<F: FnMut() -> bool>(mut op: F) -> bool {
+-    for attempt in 0..MAX_RETRIES {
+-        if op() {
+-            return true;
++    for attempt in 0..5 {
++        if op() {
++            return true;
++        }
++        let delay_ms = 100 * (1 << attempt);
++        if delay_ms > 30000 {
++            return false;
+         }
+-        sleep(BACKOFF.pow(attempt));
++        sleep(Duration::from_millis(delay_ms));
+     }
+     false
+ }

src/cli-finding-classifier/evals/files/eval4-clean.diff

@@ -0,0 +1,28 @@
+diff --git a/src/parser.rs b/src/parser.rs
+index 7777777..8888888 100644
+--- a/src/parser.rs
++++ b/src/parser.rs
+@@ -10,6 +10,15 @@ impl Parser {
+         self.tokens.iter().filter(|t| t.is_keyword()).count()
+     }
+
++    /// Returns the first identifier token, or None if no identifier is present.
++    pub fn first_identifier(&self) -> Option<&Token> {
++        self.tokens.iter().find(|t| t.is_identifier())
++    }
++
+     pub fn token_count(&self) -> usize {
+         self.tokens.len()
+     }
+ }
++
++#[cfg(test)]
++mod tests {
++    use super::*;
++
++    #[test]
++    fn first_identifier_returns_none_for_empty() {
++        let parser = Parser::new(vec![]);
++        assert!(parser.first_identifier().is_none());
++    }
++}

src/cli-finding-classifier/evals/files/eval5-multi-issue.diff

@@ -0,0 +1,32 @@
+diff --git a/src/upload.rs b/src/upload.rs
+index 9999999..aaaaaaa 100644
+--- a/src/upload.rs
++++ b/src/upload.rs
+@@ -1,5 +1,7 @@
++use std::collections::BTreeMap;
+ use std::fs;
+ use std::path::Path;
++use serde_json::Value;
+
+ pub fn upload(file: &Path, dest: &str) -> Result<(), String> {
+     let bytes = fs::read(file).map_err(|e| e.to_string())?;
+@@ -7,11 +9,17 @@ pub fn upload(file: &Path, dest: &str) -> Result<(), String> {
+         return Err("file too large".into());
+     }
+
+-    let payload = build_payload(&bytes);
+-    transmit(payload, dest)
++    if !dest.is_empty() {
++        if dest.starts_with("https://") {
++            if bytes.len() < 1048576 {
++                let payload = build_payload(&bytes);
++                return transmit(payload, dest);
++            }
++        }
++    }
++    Err("invalid destination or file size".into())
+ }
+
+ fn build_payload(bytes: &[u8]) -> Vec<u8> {
+     bytes.to_vec()
+ }

src/cli-finding-classifier/evals/files/eval6-existing-lint-overlap.diff

@@ -0,0 +1,22 @@
+diff --git a/web/src/components/Greeting.tsx b/web/src/components/Greeting.tsx
+index bbbbbbb..ccccccc 100644
+--- a/web/src/components/Greeting.tsx
++++ b/web/src/components/Greeting.tsx
+@@ -1,12 +1,16 @@
+ import React from 'react';
++import { useState } from 'react';
+
+ type Props = {
+   name: string;
+ };
+
+ export function Greeting({ name }: Props) {
+-  return <h1>Hello, {name}!</h1>;
++  var greeting = "Hello, " + name + "!";
++  let unused = 42;
++  return <h1>{greeting}</h1>;
+ }
+
++export function _unused_helper() {}
++
+ export default Greeting;

src/cli-finding-classifier/evals/lint-screen-evals.json

@@ -0,0 +1,225 @@
+{
+  "schema_version": 1,
+  "purpose": "mistral:7b の lint 一次フィルタ判定を Claude Code baseline と突合するための eval セット (Phase a infrastructure / docs/local-llm-offload-analysis.md §11.6)。",
+  "agreement_threshold": 0.8,
+  "evals": [
+    {
+      "id": 1,
+      "name": "unused-import-detection",
+      "input_diff": "evals/files/eval1-unused-import.diff",
+      "claude_code_baseline": {
+        "model": "claude-opus-4-7",
+        "captured_at": "2026-05-08",
+        "captured_by": "claude-code-session-f21fb1a3",
+        "lint_findings": [
+          {
+            "severity": "minor",
+            "rule": "unused-import",
+            "file": "src/foo.rs",
+            "line": 1,
+            "issue": "use std::collections::HashMap; が未使用",
+            "suggestion": "import を削除"
+          },
+          {
+            "severity": "minor",
+            "rule": "unused-import",
+            "file": "src/foo.rs",
+            "line": 3,
+            "issue": "use std::path::Path; が未使用",
+            "suggestion": "import を削除"
+          }
+        ],
+        "screen_decision": "auto_fix"
+      },
+      "expectations": [
+        "lint_findings が 1 件以上含まれる",
+        "rule 名が unused-import 系 (unused_imports / unused-import / dead_code 等) を含む",
+        "screen_decision が 'auto_fix'",
+        "false positive (実在しない issue) の出力なし",
+        "JSON parse 成功 (fallback rate 0)"
+      ]
+    },
+    {
+      "id": 2,
+      "name": "deep-nesting-flagging",
+      "input_diff": "evals/files/eval2-deep-nesting.diff",
+      "claude_code_baseline": {
+        "model": "claude-opus-4-7",
+        "captured_at": "2026-05-08",
+        "captured_by": "claude-code-session-f21fb1a3",
+        "lint_findings": [
+          {
+            "severity": "major",
+            "rule": "deep-nesting",
+            "file": "src/permission.rs",
+            "line": 2,
+            "issue": "if 文のネストが 5 階層 (>4 levels) に達しており、可読性と保守性が低下",
+            "suggestion": "early return / guard clause で平坦化、または small functions に抽出"
+          }
+        ],
+        "screen_decision": "human_review"
+      },
+      "expectations": [
+        "lint_findings が 1 件以上含まれる",
+        "rule または issue が nesting / complexity / readability 系の語彙を含む",
+        "screen_decision が 'human_review' (リファクタは設計判断を伴う)",
+        "JSON parse 成功"
+      ]
+    },
+    {
+      "id": 3,
+      "name": "magic-number-detection",
+      "input_diff": "evals/files/eval3-magic-number.diff",
+      "claude_code_baseline": {
+        "model": "claude-opus-4-7",
+        "captured_at": "2026-05-08",
+        "captured_by": "claude-code-session-f21fb1a3",
+        "lint_findings": [
+          {
+            "severity": "minor",
+            "rule": "magic-number",
+            "file": "src/retry.rs",
+            "line": 6,
+            "issue": "リトライ回数 5 がハードコード",
+            "suggestion": "const MAX_RETRIES: u32 = 5; に切り出す"
+          },
+          {
+            "severity": "minor",
+            "rule": "magic-number",
+            "file": "src/retry.rs",
+            "line": 10,
+            "issue": "バックオフ閾値 30000 がハードコード",
+            "suggestion": "const MAX_BACKOFF_MS: u64 = 30000; に切り出す"
+          }
+        ],
+        "screen_decision": "auto_fix"
+      },
+      "expectations": [
+        "lint_findings が 1 件以上含まれる",
+        "rule または issue が magic-number / hardcoded / constant 系の語彙を含む",
+        "screen_decision が 'auto_fix'",
+        "JSON parse 成功"
+      ]
+    },
+    {
+      "id": 4,
+      "name": "clean-diff-no-false-positive",
+      "input_diff": "evals/files/eval4-clean.diff",
+      "claude_code_baseline": {
+        "model": "claude-opus-4-7",
+        "captured_at": "2026-05-08",
+        "captured_by": "claude-code-session-f21fb1a3",
+        "lint_findings": [],
+        "screen_decision": "informational"
+      },
+      "expectations": [
+        "lint_findings が 0 件 (false positive 検知)",
+        "screen_decision が 'informational' (指摘なしのため lint screen 判定不要)",
+        "JSON parse 成功"
+      ]
+    },
+    {
+      "id": 5,
+      "name": "multi-issue-coverage",
+      "input_diff": "evals/files/eval5-multi-issue.diff",
+      "claude_code_baseline": {
+        "model": "claude-opus-4-7",
+        "captured_at": "2026-05-08",
+        "captured_by": "claude-code-session-f21fb1a3",
+        "lint_findings": [
+          {
+            "severity": "minor",
+            "rule": "unused-import",
+            "file": "src/upload.rs",
+            "line": 1,
+            "issue": "use std::collections::BTreeMap; が未使用",
+            "suggestion": "import を削除"
+          },
+          {
+            "severity": "minor",
+            "rule": "unused-import",
+            "file": "src/upload.rs",
+            "line": 4,
+            "issue": "use serde_json::Value; が未使用",
+            "suggestion": "import を削除"
+          },
+          {
+            "severity": "minor",
+            "rule": "magic-number",
+            "file": "src/upload.rs",
+            "line": 15,
+            "issue": "サイズ閾値 1048576 がハードコード",
+            "suggestion": "const MAX_INLINE_SIZE: usize = 1024 * 1024; に切り出す"
+          },
+          {
+            "severity": "major",
+            "rule": "deep-nesting",
+            "file": "src/upload.rs",
+            "line": 12,
+            "issue": "if 文のネストが 3 階層に達し、guard clause で平坦化可能",
+            "suggestion": "early return で平坦化"
+          }
+        ],
+        "screen_decision": "human_review"
+      },
+      "expectations": [
+        "lint_findings が 2 件以上 (複数 issue を取りこぼさない)",
+        "rule の種類が 2 種以上 (unused-import / magic-number / deep-nesting のいずれかを混在検出)",
+        "screen_decision が 'human_review' (deep-nesting を含むため、リファクタ判断が必要)",
+        "JSON parse 成功"
+      ]
+    },
+    {
+      "id": 6,
+      "name": "existing-lint-overlap-measurement",
+      "input_diff": "evals/files/eval6-existing-lint-overlap.diff",
+      "claude_code_baseline": {
+        "model": "claude-opus-4-7",
+        "captured_at": "2026-05-08",
+        "captured_by": "claude-code-session-f21fb1a3",
+        "lint_findings": [
+          {
+            "severity": "minor",
+            "rule": "no-var",
+            "file": "web/src/components/Greeting.tsx",
+            "line": 9,
+            "issue": "var を let / const に置き換えるべき (oxlint/biome の no-var で既出範囲)",
+            "suggestion": "const greeting = ... に変更"
+          },
+          {
+            "severity": "minor",
+            "rule": "no-unused-vars",
+            "file": "web/src/components/Greeting.tsx",
+            "line": 10,
+            "issue": "ローカル変数 unused が未使用 (oxlint/biome の no-unused-vars で既出範囲)",
+            "suggestion": "削除"
+          },
+          {
+            "severity": "minor",
+            "rule": "no-unused-vars",
+            "file": "web/src/components/Greeting.tsx",
+            "line": 14,
+            "issue": "エクスポート関数 _unused_helper が未使用 (oxlint/biome の no-unused-vars で既出範囲)",
+            "suggestion": "削除"
+          },
+          {
+            "severity": "minor",
+            "rule": "unused-import",
+            "file": "web/src/components/Greeting.tsx",
+            "line": 2,
+            "issue": "import { useState } from 'react'; が未使用 (oxlint/biome の no-unused-imports で既出範囲)",
+            "suggestion": "import を削除"
+          }
+        ],
+        "screen_decision": "informational",
+        "overlap_note": "全 finding が oxlint/biome の既存ルール範囲。lint screen で人間に提示する必要なし (既存 linter で auto-fix 可能)"
+      },
+      "expectations": [
+        "lint_findings が 1 件以上含まれる",
+        "screen_decision が 'informational' (既存 linter 範囲のため lint screen の付加価値なし)",
+        "overlap 率測定用: oxlint/biome ルール名 (no-var / no-unused-vars / no-unused-imports) との一致を計算可能",
+        "JSON parse 成功"
+      ]
+    }
+  ]
+}