Claude API のプロンプトをリグレッションから守る — Golden Dataset で品質を継続検証する仕組み

{"id": "support_001", "category": "shipping_delay", "input": "注文した商品がまだ届きません。", "expected_intent": "shipping_inquiry", "expected_keywords": ["注文番号", "配送状況", "確認"], "must_not_include": ["返金", "キャンセル"]}
{"id": "support_002", "category": "refund", "input": "サイズが合わないので返品したいです。", "expected_intent": "refund_request", "expected_keywords": ["返品ポリシー", "返金", "手続き"], "must_not_include": []}
{"id": "support_003", "category": "usage", "input": "アプリにログインできません。", "expected_intent": "technical_support", "expected_keywords": ["パスワード", "リセット", "アカウント"], "must_not_include": ["返金"]}

# evaluator/rule_based.py
import json
from typing import TypedDict
 
class GoldenItem(TypedDict):
    id: str
    category: str
    input: str
    expected_keywords: list[str]
    must_not_include: list[str]
 
class RuleResult(TypedDict):
    id: str
    keyword_score: float  # 0.0〜1.0
    blocklist_passed: bool
    overall_passed: bool
    failure_reasons: list[str]
 
def evaluate_rule(item: GoldenItem, output: str, min_keyword_ratio: float = 0.5) -> RuleResult:
    """ルールベース評価: 期待キーワード比率 + ブロックリスト判定。"""
    output_lower = output.lower()
 
    # ① 期待キーワードのうち何割含まれているか
    matched = [kw for kw in item["expected_keywords"] if kw.lower() in output_lower]
    keyword_score = len(matched) / max(len(item["expected_keywords"]), 1)
 
    # ② ブロックリストは1つでも含めばアウト
    blocklist_violations = [kw for kw in item["must_not_include"] if kw.lower() in output_lower]
    blocklist_passed = len(blocklist_violations) == 0
 
    failure_reasons = []
    if keyword_score < min_keyword_ratio:
        missing = set(item["expected_keywords"]) - set(matched)
        failure_reasons.append(f"keyword_ratio {keyword_score:.2f} < {min_keyword_ratio} (missing: {missing})")
    if not blocklist_passed:
        failure_reasons.append(f"blocklist_violation: {blocklist_violations}")
 
    return {
        "id": item["id"],
        "keyword_score": keyword_score,
        "blocklist_passed": blocklist_passed,
        "overall_passed": keyword_score >= min_keyword_ratio and blocklist_passed,
        "failure_reasons": failure_reasons,
    }
 
# 期待出力例:
# {'id': 'support_001', 'keyword_score': 1.0, 'blocklist_passed': True, 'overall_passed': True, 'failure_reasons': []}

# evaluator/llm_judge.py
import os
import anthropic
from typing import TypedDict
 
JUDGE_MODEL = "claude-sonnet-4-6"  # 評価ログに必ず記録すること
 
class JudgeResult(TypedDict):
    id: str
    politeness_score: int  # 1〜5
    factual_score: int     # 1〜5
    relevance_score: int   # 1〜5
    judge_reasoning: str
    judge_model: str
 
JUDGE_SYSTEM = """あなたはカスタマーサポート応答の評価者です。
3 つの軸で 1〜5 点で評価し、JSON のみを返してください。
 
- politeness: 1=失礼 / 5=非常に丁寧
- factual: 1=事実誤認あり / 5=事実誤認なし
- relevance: 1=的外れ / 5=ユーザーの意図を完全に捉えている
 
評価は厳格に。3 点を「平均的」の基準点とし、迷ったら下方に振ってください。"""
 
JUDGE_USER_TEMPLATE = """ユーザー入力:
{user_input}
 
評価対象の応答:
{model_output}
 
JSON 形式で {{"politeness": N, "factual": N, "relevance": N, "reasoning": "..."}} のみを返してください。"""
 
def judge(item_id: str, user_input: str, model_output: str) -> JudgeResult:
    client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
    try:
        msg = client.messages.create(
            model=JUDGE_MODEL,
            max_tokens=400,
            system=JUDGE_SYSTEM,
            messages=[{
                "role": "user",
                "content": JUDGE_USER_TEMPLATE.format(
                    user_input=user_input, model_output=model_output
                )
            }],
        )
        text = msg.content[0].text.strip()
        # ```json ... ``` ブロックの除去
        if text.startswith("```"):
            text = text.split("```")[1].lstrip("json").strip()
        import json
        parsed = json.loads(text)
        return {
            "id": item_id,
            "politeness_score": int(parsed["politeness"]),
            "factual_score": int(parsed["factual"]),
            "relevance_score": int(parsed["relevance"]),
            "judge_reasoning": parsed.get("reasoning", ""),
            "judge_model": JUDGE_MODEL,
        }
    except (anthropic.APIError, json.JSONDecodeError, KeyError) as e:
        # Judge 自体が失敗したら N/A 扱い（テスト全体は失敗にしない）
        return {
            "id": item_id, "politeness_score": -1, "factual_score": -1,
            "relevance_score": -1, "judge_reasoning": f"judge_error: {e}",
            "judge_model": JUDGE_MODEL,
        }
 
# 期待出力例:
# {'id': 'support_001', 'politeness_score': 5, 'factual_score': 5, 'relevance_score': 4, 'judge_reasoning': '丁寧で事実誤認なし。注文番号確認の手順も明確。', 'judge_model': 'claude-sonnet-4-6'}

# evaluator/kappa.py
from sklearn.metrics import cohen_kappa_score
from typing import Sequence
 
def compute_kappa(judge_scores: Sequence[int], human_scores: Sequence[int]) -> dict:
    """politeness などの順序尺度に対する一致度を測る。"""
    # 1〜5 の順序尺度なので weights="quadratic" を使う
    kappa = cohen_kappa_score(human_scores, judge_scores, weights="quadratic")
    diffs = [abs(j - h) for j, h in zip(judge_scores, human_scores)]
    return {
        "kappa": kappa,
        "mean_abs_diff": sum(diffs) / len(diffs),
        "exact_agreement_rate": sum(1 for d in diffs if d == 0) / len(diffs),
        "n_samples": len(diffs),
    }
 
# 期待出力例:
# {'kappa': 0.72, 'mean_abs_diff': 0.4, 'exact_agreement_rate': 0.65, 'n_samples': 20}
# kappa >= 0.6 を「実用ライン」、>= 0.8 を「強い一致」と判断する

# .github/workflows/prompt-eval.yml
name: Prompt Regression Eval
 
on:
  push:
    branches: [main]
  pull_request:
    paths:
      - 'prompts/**'
      - 'evaluator/**'
 
jobs:
  eval:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'
      - run: pip install -r evaluator/requirements.txt
 
      # PR は安価なルールベースのみ（高速・無料）
      - name: Rule-based eval (PR)
        if: github.event_name == 'pull_request'
        run: python evaluator/run.py --mode rule_only --golden golden_dataset.jsonl
 
      # main マージ時はフル評価（LLM-as-a-Judge 含む）
      - name: Full eval (main)
        if: github.event_name == 'push'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: python evaluator/run.py --mode full --golden golden_dataset.jsonl --output eval_results.json
 
      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: eval_results.json
 
      # 失敗時に Slack 通知（任意）
      - name: Notify on regression
        if: failure() && github.event_name == 'push'
        run: |
          curl -X POST -H 'Content-type: application/json' \
            --data "{\"text\":\"⚠️ Prompt eval regression detected on main. See: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" \
            ${{ secrets.SLACK_WEBHOOK_URL }}