Claude Agent SDK でプログレッシブデリバリーを実装する — カナリア・Feature Flag・自動ロールバックの本番パターン

// eval/offline.ts — CI で実行する
import { runAgent } from "../agent/versions";
import Anthropic from "@anthropic-ai/sdk";
import testset from "./fixtures/regression.json"; // 過去のインシデント由来のケース集
 
const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY! });
 
async function runEval(version: "v2026.04.15" | "v2026.04.22") {
  const results: Array<{ id: string; ok: boolean; reason?: string }> = [];
  for (const tc of testset) {
    const out = await runAgent(version, tc.input, client);
    const ok = tc.must_contain.every((s) => out.text.includes(s));
    results.push({ id: tc.id, ok, reason: ok ? undefined : "missing_required_phrase" });
  }
  const pass = results.filter((r) => r.ok).length / results.length;
  if (pass < 0.9) throw new Error(`Eval pass rate ${(pass * 100).toFixed(1)}% < 90%`);
  return { pass, results };
}

// agent/versions.ts
import type Anthropic from "@anthropic-ai/sdk";
import { agent } from "@anthropic-ai/agent-sdk";
 
// バージョンごとの設定をコードと分離して持つ
export const AGENT_VERSIONS = {
  "v2026.04.15": {
    model: "claude-sonnet-4-6" as const,
    systemPrompt:
      "あなたは公式ドキュメント内でのみ回答する顧客サポートエージェントです。未確定情報は『確認します』で返します。",
    tools: ["search_docs", "create_ticket"],
  },
  "v2026.04.22": {
    model: "claude-sonnet-4-6" as const,
    systemPrompt:
      "あなたは顧客サポートエージェントです。公式ドキュメントを優先しつつ、顧客との過去のやりとりも参照して共感的に応答します。未確定情報は必ず『確認します』で返します。",
    tools: ["search_docs", "search_history", "create_ticket"],
  },
} as const;
 
export type AgentVersion = keyof typeof AGENT_VERSIONS;
 
export async function runAgent(
  version: AgentVersion,
  input: string,
  client: Anthropic,
) {
  const cfg = AGENT_VERSIONS[version];
  const session = agent({
    client,
    model: cfg.model,
    system: cfg.systemPrompt,
    allowed_tools: cfg.tools,
  });
  // エラーはそのまま投げて上位で観測する（握りつぶさない）
  return await session.run({ input });
}

// flags/resolver.ts
import { createHash } from "crypto";
 
// KV に保存される設定例
// {
//   "agent.customer_support": {
//     "stable": "v2026.04.15",
//     "canary": "v2026.04.22",
//     "canary_percentage": 5,           // 5% へ振り分け
//     "canary_allowlist": ["tenant_a"]  // 強制的に新バージョンに入れるテナント
//   }
// }
 
export type FlagConfig = {
  stable: string;
  canary: string;
  canary_percentage: number;
  canary_allowlist: string[];
};
 
// ユーザーごとに一貫して同じバージョンが返るようにハッシュで決める
function bucketOf(userId: string, flagKey: string): number {
  const h = createHash("sha256").update(`${flagKey}:${userId}`).digest();
  // 先頭4バイトを整数化して 0-99 に落とし込む
  return h.readUInt32BE(0) % 100;
}
 
export function resolveVersion(
  flag: FlagConfig,
  ctx: { userId: string; tenantId: string; flagKey: string },
): { version: string; channel: "stable" | "canary" } {
  if (flag.canary_allowlist.includes(ctx.tenantId)) {
    return { version: flag.canary, channel: "canary" };
  }
  const bucket = bucketOf(ctx.userId, ctx.flagKey);
  if (bucket < flag.canary_percentage) {
    return { version: flag.canary, channel: "canary" };
  }
  return { version: flag.stable, channel: "stable" };
}

// routes/support.ts
import { Hono } from "hono";
import Anthropic from "@anthropic-ai/sdk";
import { trace, metrics } from "@opentelemetry/api";
import { runAgent, type AgentVersion } from "../agent/versions";
import { resolveVersion } from "../flags/resolver";
import { getFlag } from "../flags/store";
 
const app = new Hono();
const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY! });
const meter = metrics.getMeter("agent.support");
 
// version タグ付きのメトリクス
const latency = meter.createHistogram("agent.latency", { unit: "ms" });
const errors = meter.createCounter("agent.errors");
const thumbsDown = meter.createCounter("agent.thumbs_down");
 
app.post("/chat", async (c) => {
  const body = await c.req.json<{ userId: string; tenantId: string; input: string }>();
  const flag = await getFlag("agent.customer_support");
  const { version, channel } = resolveVersion(flag, {
    userId: body.userId,
    tenantId: body.tenantId,
    flagKey: "agent.customer_support",
  });
 
  const span = trace.getTracer("agent").startSpan("agent.run", {
    attributes: { "agent.version": version, "agent.channel": channel },
  });
 
  const start = performance.now();
  try {
    const result = await runAgent(version as AgentVersion, body.input, client);
    latency.record(performance.now() - start, { version, channel, status: "ok" });
    return c.json({ version, channel, result });
  } catch (err) {
    // 本番では canary の失敗を stable にフェイルオーバーする選択肢もあるが、
    // SLO を汚染しないよう、canary の失敗は canary で計上する
    errors.add(1, { version, channel, error_type: (err as Error).name });
    latency.record(performance.now() - start, { version, channel, status: "error" });
    throw err;
  } finally {
    span.end();
  }
});
 
// フィードバックエンドポイント — thumbs_down を SLO の一部に使う
app.post("/feedback", async (c) => {
  const { version, channel, rating } = await c.req.json();
  if (rating === "down") thumbsDown.add(1, { version, channel });
  return c.json({ ok: true });
});
 
export default app;

// slo/watchdog.ts — Cron で 60 秒ごとに走らせる想定
import { queryMetrics } from "./prometheus";
import { setFlag, getFlag } from "../flags/store";
import { notifySlack } from "./slack";
 
type Sample = { version: string; errors: number; total: number; p95_ms: number };
 
const IMMEDIATE_BUDGET = {
  error_rate_max: 0.02, // 2%
  p95_ms_max: 6000,
};
 
export async function checkAndRollback(flagKey: string) {
  const flag = await getFlag(flagKey);
  if (flag.canary_percentage === 0) return; // 既に停止済み
 
  const stable: Sample = await queryMetrics(flag.stable, "60s");
  const canary: Sample = await queryMetrics(flag.canary, "60s");
 
  // サンプル数が少なすぎると誤判定するため、下限を設ける
  if (canary.total < 50) return;
 
  const canaryErrRate = canary.errors / canary.total;
  const stableErrRate = stable.total > 0 ? stable.errors / stable.total : 0;
 
  const tooManyErrors = canaryErrRate > IMMEDIATE_BUDGET.error_rate_max;
  const tooSlow = canary.p95_ms > IMMEDIATE_BUDGET.p95_ms_max;
  const regressedVsStable =
    canaryErrRate > stableErrRate * 2 && canary.total >= 200;
 
  if (tooManyErrors || tooSlow || regressedVsStable) {
    await setFlag(flagKey, { ...flag, canary_percentage: 0 });
    await notifySlack(
      `🚨 ${flagKey} canary=${flag.canary} をロールバックしました: ` +
        `errRate=${(canaryErrRate * 100).toFixed(2)}%, p95=${canary.p95_ms}ms`,
    );
  }
}

// slo/promoter.ts — 15 分ごとに走らせる
import { queryMetrics, querySatisfaction } from "./prometheus";
import { setFlag, getFlag } from "../flags/store";
import { notifySlack } from "./slack";
 
const STAGES = [1, 5, 25, 50, 100]; // %
 
export async function tryPromote(flagKey: string) {
  const flag = await getFlag(flagKey);
  if (flag.canary_percentage === 0 || flag.canary_percentage === 100) return;
 
  const canary = await queryMetrics(flag.canary, "15m");
  const sat = await querySatisfaction(flag.canary, "15m"); // thumbs_down 比率
 
  // 満足度の悪化がないことを確認
  if (sat.thumbs_down_rate > 0.05) {
    await notifySlack(
      `⏸ ${flagKey} 昇格を停止: thumbs_down=${sat.thumbs_down_rate.toFixed(3)}`,
    );
    return;
  }
 
  // 十分なサンプルを観測したか
  if (canary.total < 500) return;
 
  const idx = STAGES.indexOf(flag.canary_percentage);
  const next = STAGES[idx + 1];
  if (!next) return;
  await setFlag(flagKey, { ...flag, canary_percentage: next });
  await notifySlack(`✅ ${flagKey} を ${next}% へ昇格`);
}

// agent/shadow.ts
import { runAgent, type AgentVersion } from "./versions";
import type Anthropic from "@anthropic-ai/sdk";
import { recordDiff } from "./shadow_store";
 
export async function runWithShadow(
  primary: AgentVersion,
  shadow: AgentVersion,
  input: string,
  client: Anthropic,
) {
  const [primaryResult, shadowResult] = await Promise.allSettled([
    runAgent(primary, input, client),
    runAgent(shadow, input, client),
  ]);
 
  // シャドウ結果はユーザーに返さず、差分だけ保存する
  if (
    primaryResult.status === "fulfilled" &&
    shadowResult.status === "fulfilled"
  ) {
    await recordDiff({
      input,
      primary: primaryResult.value,
      shadow: shadowResult.value,
    });
  }
  if (primaryResult.status === "rejected") throw primaryResult.reason;
  return primaryResult.value;
}

// flags/resolver.ts に追加
if (flag.tenant_overrides?.[ctx.tenantId]) {
  const override = flag.tenant_overrides[ctx.tenantId];
  return { version: override.version, channel: override.channel };
}