Claude API のプロンプトをコードとして管理する — レジストリ・バージョニング・A/Bテストの本番運用ガイド

// src/prompts/types.ts
import { z } from "zod";
 
export const PromptVariantSchema = z.object({
  id: z.string().regex(/^v\d+(_[a-z0-9-]+)?$/), // 例: v3, v3_a, v3_b
  weight: z.number().int().min(0).max(100),     // A/Bテストの配分 (合計100)
  status: z.enum(["draft", "canary", "stable", "deprecated"]),
  systemPrompt: z.string().min(1),
  messages: z.array(z.object({
    role: z.enum(["user", "assistant"]),
    content: z.string(),
  })),
  model: z.enum([
    "claude-opus-5",
    "claude-sonnet-5",
    "claude-opus-4-6",
    "claude-sonnet-4-6",
    "claude-haiku-4-5-20251001",
  ]),
  maxTokens: z.number().int().positive().max(64000),
  temperature: z.number().min(0).max(1).default(0.7),
  inputSchema: z.record(z.any()).default({}),
  // ⚠️ 内側に .default() を置いて外側を .default({}) にしてはいけません（後述）
  slo: z.object({
    p95LatencyMs: z.number().positive(),
    minSuccessRate: z.number().min(0).max(1),
    maxCostPerCallUsd: z.number().positive(),
  }).default({ p95LatencyMs: 8000, minSuccessRate: 0.97, maxCostPerCallUsd: 0.05 }),
  createdAt: z.string().datetime(),
  createdBy: z.string(),
});
 
export const PromptEntrySchema = z.object({
  promptId: z.string().regex(/^[a-z][a-z0-9-]*$/),
  description: z.string(),
  variants: z.array(PromptVariantSchema).min(1),
});
 
export type PromptEntry = z.infer<typeof PromptEntrySchema>;
export type PromptVariant = z.infer<typeof PromptVariantSchema>;

// src/prompts/types.ts （末尾に追記）
export const SERVABLE_STATUSES = ["canary", "stable"] as const;
export const isServable = (v: PromptVariant) =>
  (SERVABLE_STATUSES as readonly string[]).includes(v.status);

// prompts/article-summarizer.json
{
  "promptId": "article-summarizer",
  "description": "ブログ記事の冒頭を 3 行に要約する",
  "variants": [
    {
      "id": "v2",
      "weight": 95,
      "status": "stable",
      "model": "claude-haiku-4-5-20251001",
      "maxTokens": 256,
      "temperature": 0.3,
      "systemPrompt": "あなたは熟練の編集者です。読者が記事を開くか判断できる、3 行の要約を作成してください。",
      "messages": [
        { "role": "user", "content": "次の記事を 3 行で要約してください:\n\n{{ article }}" }
      ],
      "inputSchema": { "article": "string" },
      "slo": { "p95LatencyMs": 4000, "minSuccessRate": 0.98, "maxCostPerCallUsd": 0.005 },
      "createdAt": "2026-03-12T09:00:00Z",
      "createdBy": "ops@example.com"
    },
    {
      "id": "v3_canary",
      "weight": 5,
      "status": "canary",
      "model": "claude-haiku-4-5-20251001",
      "maxTokens": 256,
      "temperature": 0.2,
      "systemPrompt": "あなたは熟練の編集者です。読者の興味を引く、最初の 1 行で結論を述べる 3 行要約を作成してください。",
      "messages": [
        { "role": "user", "content": "次の記事を 3 行で要約してください。1 行目に結論を必ず置いてください:\n\n{{ article }}" }
      ],
      "inputSchema": { "article": "string" },
      "slo": { "p95LatencyMs": 4000, "minSuccessRate": 0.98, "maxCostPerCallUsd": 0.005 },
      "createdAt": "2026-04-25T09:00:00Z",
      "createdBy": "ops@example.com"
    }
  ]
}

// src/prompts/registry.ts
import fs from "node:fs";
import path from "node:path";
import { PromptEntrySchema, isServable, type PromptEntry } from "./types";
 
const PROMPTS_DIR = path.join(process.cwd(), "prompts");
 
export function loadRegistry(): Map<string, PromptEntry> {
  const registry = new Map<string, PromptEntry>();
  const files = fs.readdirSync(PROMPTS_DIR).filter((f) => f.endsWith(".json"));
 
  for (const file of files) {
    const raw = JSON.parse(fs.readFileSync(path.join(PROMPTS_DIR, file), "utf8"));
    const parsed = PromptEntrySchema.parse(raw);
    // 配信対象 (canary / stable) だけを 100 に合わせる。draft を数に入れてはいけない
    const totalWeight = parsed.variants
      .filter(isServable)
      .reduce((acc, v) => acc + v.weight, 0);
    if (totalWeight !== 100) {
      throw new Error(`[${parsed.promptId}] 配信対象 weight の合計が 100 ではありません (${totalWeight})`);
    }
    const strays = parsed.variants.filter((v) => !isServable(v) && v.weight !== 0);
    if (strays.length > 0) {
      throw new Error(
        `[${parsed.promptId}] 非配信 variant に weight が残っています: ${strays.map((v) => v.id).join(", ")}`,
      );
    }
    registry.set(parsed.promptId, parsed);
  }
  return registry;
}

// src/prompts/selector.ts
import crypto from "node:crypto";
import { isServable, type PromptEntry, type PromptVariant } from "./types";
 
function stableHash(input: string): number {
  const hash = crypto.createHash("sha256").update(input).digest();
  const num = hash.readUInt32BE(0);
  return num % 100;
}
 
export function selectVariant(
  entry: PromptEntry,
  identityKey: string,
  options?: { forceVariantId?: string; disabled?: ReadonlySet<string> },
): PromptVariant {
  if (options?.forceVariantId) {
    const forced = entry.variants.find((v) => v.id === options.forceVariantId);
    if (forced) return forced;
  }
 
  // 配信対象は canary / stable のみ。さらに Guardian が落としたものを除外する
  const candidates = entry.variants.filter(
    (v) => isServable(v) && !options?.disabled?.has(`${entry.promptId}:${v.id}`),
  );
  if (candidates.length === 0) {
    throw new Error(`[${entry.promptId}] 配信可能な variant がありません`);
  }
 
  // 除外が起きると合計が 100 未満になるため、残った合計へ射影し直す
  const totalWeight = candidates.reduce((acc, v) => acc + v.weight, 0);
  const scaled = (stableHash(`${entry.promptId}::${identityKey}`) / 100) * totalWeight;
  let cumulative = 0;
  for (const v of candidates) {
    cumulative += v.weight;
    if (scaled < cumulative) return v;
  }
  return candidates[candidates.length - 1]!;
}

// src/prompts/renderer.ts
export function render(template: string, vars: Record<string, string>): string {
  return template.replace(/\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/g, (_, key) => {
    if (!(key in vars)) {
      throw new Error(`未代入のテンプレート変数: ${key}`);
    }
    return vars[key]!;
  });
}

// src/prompts/invoke.ts
import Anthropic from "@anthropic-ai/sdk";
import { loadRegistry } from "./registry";
import { selectVariant } from "./selector";
import { render } from "./renderer";
import { recordTelemetry } from "./telemetry";
 
const client = new Anthropic();
const registry = loadRegistry();
 
export async function invokePrompt(params: {
  promptId: string;
  identityKey: string;
  vars: Record<string, string>;
  forceVariantId?: string;
}) {
  const entry = registry.get(params.promptId);
  if (!entry) throw new Error(`Unknown promptId: ${params.promptId}`);
 
  const variant = selectVariant(entry, params.identityKey, {
    forceVariantId: params.forceVariantId,
  });
 
  const renderedMessages = variant.messages.map((m) => ({
    role: m.role,
    content: render(m.content, params.vars),
  }));
 
  const startedAt = Date.now();
  let usage: Anthropic.Messages.Usage | undefined;
  let success = false;
 
  try {
    const res = await client.messages.create({
      model: variant.model,
      max_tokens: variant.maxTokens,
      temperature: variant.temperature,
      system: variant.systemPrompt,
      messages: renderedMessages,
    });
    usage = res.usage;
    success = true;
    return {
      promptId: entry.promptId,
      variantId: variant.id,
      output: res.content,
      usage,
    };
  } catch (err) {
    success = false;
    throw err;
  } finally {
    await recordTelemetry({
      promptId: entry.promptId,
      variantId: variant.id,
      model: variant.model,
      identityKey: params.identityKey,
      latencyMs: Date.now() - startedAt,
      success,
      usage,
    });
  }
}

// src/prompts/telemetry.ts
import { env } from "cloudflare:workers"; // DB / PROMPT_FLAGS は wrangler.toml で bind
import type { Anthropic } from "@anthropic-ai/sdk";
 
interface TelemetryRecord {
  promptId: string;
  variantId: string;
  model: string;        // ← 単価を引くために必須。variant.model をそのまま渡す
  identityKey: string;
  latencyMs: number;
  success: boolean;
  usage?: Anthropic.Messages.Usage;
  feedbackScore?: number;
}
 
export async function recordTelemetry(rec: TelemetryRecord) {
  const inputTokens = rec.usage?.input_tokens ?? 0;
  const outputTokens = rec.usage?.output_tokens ?? 0;
  const costUsd = estimateCostUsd(rec.model, inputTokens, outputTokens);
 
  await env.DB.prepare(
    `INSERT INTO prompt_invocations
     (prompt_id, variant_id, model, identity_key, latency_ms, success, input_tokens, output_tokens, cost_usd, created_at)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
  )
    .bind(
      rec.promptId,
      rec.variantId,
      rec.model,
      rec.identityKey,
      rec.latencyMs,
      rec.success ? 1 : 0,
      inputTokens,
      outputTokens,
      costUsd,
      new Date().toISOString(),
    )
    .run();
}
 
// 公開単価 (USD / 100万トークン)。モデルを増やしたらここだけを更新します
const PRICE_USD_PER_MTOK: Record<string, { input: number; output: number }> = {
  "claude-opus-5": { input: 5, output: 25 },
  "claude-sonnet-5": { input: 2, output: 10 },
  "claude-opus-4-6": { input: 5, output: 25 },
  "claude-sonnet-4-6": { input: 3, output: 15 },
  "claude-haiku-4-5-20251001": { input: 1, output: 5 },
};
 
function estimateCostUsd(model: string, inputTokens: number, outputTokens: number) {
  const price = PRICE_USD_PER_MTOK[model];
  // 黙って 0 を返さない。未知のモデルは即座に気づけるようにします
  if (!price) throw new Error(`単価表に無いモデルです: ${model}`);
  return (inputTokens * price.input + outputTokens * price.output) / 1_000_000;
}

// src/prompts/guardian.ts
import { env } from "cloudflare:workers";
import { loadRegistry } from "./registry";
 
interface VariantStats {
  count: number;
  successRate: number;
  p95LatencyMs: number;
  avgCostUsd: number;
}
 
async function fetchStats(promptId: string, variantId: string, sinceIso: string): Promise<VariantStats> {
  const result = await env.DB.prepare(
    `SELECT
       COUNT(*) AS count,
       AVG(CASE WHEN success = 1 THEN 1.0 ELSE 0.0 END) AS success_rate,
       AVG(cost_usd) AS avg_cost
     FROM prompt_invocations
     WHERE prompt_id = ? AND variant_id = ? AND created_at >= ?`
  )
    .bind(promptId, variantId, sinceIso)
    .first<{ count: number; success_rate: number; avg_cost: number }>();
 
  // p95 は近似 (D1 では PERCENTILE 関数が無いため、ORDER BY + OFFSET で代用)
  const p95Row = await env.DB.prepare(
    `SELECT latency_ms FROM prompt_invocations
     WHERE prompt_id = ? AND variant_id = ? AND created_at >= ?
     ORDER BY latency_ms ASC
     LIMIT 1 OFFSET CAST((SELECT COUNT(*) FROM prompt_invocations
       WHERE prompt_id = ? AND variant_id = ? AND created_at >= ?) * 0.95 AS INTEGER)`
  )
    .bind(promptId, variantId, sinceIso, promptId, variantId, sinceIso)
    .first<{ latency_ms: number }>();
 
  return {
    count: result?.count ?? 0,
    successRate: result?.success_rate ?? 1,
    p95LatencyMs: p95Row?.latency_ms ?? 0,
    avgCostUsd: result?.avg_cost ?? 0,
  };
}
 
export async function evaluateAndRollback() {
  const registry = loadRegistry();
  const sinceIso = new Date(Date.now() - 15 * 60 * 1000).toISOString();
 
  for (const entry of registry.values()) {
    const canary = entry.variants.find((v) => v.status === "canary");
    if (!canary) continue;
 
    const stats = await fetchStats(entry.promptId, canary.id, sinceIso);
    if (stats.count < 50) continue; // サンプル不足なら判定保留
 
    const slo = canary.slo;
    const violations: string[] = [];
    if (stats.successRate < slo.minSuccessRate) {
      violations.push(`successRate ${stats.successRate.toFixed(3)} < ${slo.minSuccessRate}`);
    }
    if (stats.p95LatencyMs > slo.p95LatencyMs) {
      violations.push(`p95 ${stats.p95LatencyMs}ms > ${slo.p95LatencyMs}ms`);
    }
    if (stats.avgCostUsd > slo.maxCostPerCallUsd) {
      violations.push(`avgCost $${stats.avgCostUsd.toFixed(4)} > $${slo.maxCostPerCallUsd}`);
    }
 
    if (violations.length > 0) {
      console.warn(`[GUARDIAN] ${entry.promptId}/${canary.id} SLO 違反: ${violations.join(", ")}`);
      await rollback(entry.promptId, canary.id);
    }
  }
}
 
async function rollback(promptId: string, variantId: string) {
  // KV に「この variant は配信停止」フラグを書く
  await env.PROMPT_FLAGS.put(`disabled:${promptId}:${variantId}`, "1", { expirationTtl: 86400 });
}
 
// --- 読み出し側。これが無いとフラグは誰にも参照されません ---
export async function loadDisabledVariants(): Promise<ReadonlySet<string>> {
  const listed = await env.PROMPT_FLAGS.list({ prefix: "disabled:" });
  return new Set<string>(
    listed.keys.map((k: { name: string }) => k.name.slice("disabled:".length)),
  );
}

// src/prompts/invoke.ts （抜粋）
let cache: { at: number; set: ReadonlySet<string> } | null = null;
 
async function disabledSet(): Promise<ReadonlySet<string>> {
  if (cache && Date.now() - cache.at < 30_000) return cache.set;
  const set = await loadDisabledVariants();
  cache = { at: Date.now(), set };
  return set;
}
 
const variant = selectVariant(entry, identityKey, { disabled: await disabledSet() });

slo: z.object({
  p95LatencyMs: z.number().positive().default(8000),
  minSuccessRate: z.number().min(0).max(1).default(0.97),
  maxCostPerCallUsd: z.number().positive().default(0.05),
}).default({}),