Claude API のプロンプトをコードとして管理する — レジストリ・バージョニング・A/Bテストの本番運用ガイド

// src/prompts/types.ts
import { z } from "zod";
 
export const PromptVariantSchema = z.object({
  id: z.string().regex(/^v\d+(_[a-z0-9-]+)?$/), // 例: v3, v3_a, v3_b
  weight: z.number().int().min(0).max(100),     // A/Bテストの配分 (合計100)
  status: z.enum(["draft", "canary", "stable", "deprecated"]),
  systemPrompt: z.string().min(1),
  messages: z.array(z.object({
    role: z.enum(["user", "assistant"]),
    content: z.string(),
  })),
  model: z.enum(["claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5-20251001"]),
  maxTokens: z.number().int().positive().max(64000),
  temperature: z.number().min(0).max(1).default(0.7),
  inputSchema: z.record(z.any()).default({}),
  slo: z.object({
    p95LatencyMs: z.number().positive().default(8000),
    minSuccessRate: z.number().min(0).max(1).default(0.97),
    maxCostPerCallUsd: z.number().positive().default(0.05),
  }).default({}),
  createdAt: z.string().datetime(),
  createdBy: z.string(),
});
 
export const PromptEntrySchema = z.object({
  promptId: z.string().regex(/^[a-z][a-z0-9-]*$/),
  description: z.string(),
  variants: z.array(PromptVariantSchema).min(1),
});
 
export type PromptEntry = z.infer<typeof PromptEntrySchema>;
export type PromptVariant = z.infer<typeof PromptVariantSchema>;

// prompts/article-summarizer.json
{
  "promptId": "article-summarizer",
  "description": "ブログ記事の冒頭を 3 行に要約する",
  "variants": [
    {
      "id": "v2",
      "weight": 95,
      "status": "stable",
      "model": "claude-haiku-4-5-20251001",
      "maxTokens": 256,
      "temperature": 0.3,
      "systemPrompt": "あなたは熟練の編集者です。読者が記事を開くか判断できる、3 行の要約を作成してください。",
      "messages": [
        { "role": "user", "content": "次の記事を 3 行で要約してください:\n\n{{ article }}" }
      ],
      "inputSchema": { "article": "string" },
      "slo": { "p95LatencyMs": 4000, "minSuccessRate": 0.98, "maxCostPerCallUsd": 0.005 },
      "createdAt": "2026-03-12T09:00:00Z",
      "createdBy": "masaki@dolice.design"
    },
    {
      "id": "v3_canary",
      "weight": 5,
      "status": "canary",
      "model": "claude-haiku-4-5-20251001",
      "maxTokens": 256,
      "temperature": 0.2,
      "systemPrompt": "あなたは熟練の編集者です。読者の興味を引く、最初の 1 行で結論を述べる 3 行要約を作成してください。",
      "messages": [
        { "role": "user", "content": "次の記事を 3 行で要約してください。1 行目に結論を必ず置いてください:\n\n{{ article }}" }
      ],
      "inputSchema": { "article": "string" },
      "slo": { "p95LatencyMs": 4000, "minSuccessRate": 0.98, "maxCostPerCallUsd": 0.005 },
      "createdAt": "2026-04-25T09:00:00Z",
      "createdBy": "masaki@dolice.design"
    }
  ]
}

// src/prompts/registry.ts
import fs from "node:fs";
import path from "node:path";
import { PromptEntrySchema, type PromptEntry } from "./types";
 
const PROMPTS_DIR = path.join(process.cwd(), "prompts");
 
export function loadRegistry(): Map<string, PromptEntry> {
  const registry = new Map<string, PromptEntry>();
  const files = fs.readdirSync(PROMPTS_DIR).filter((f) => f.endsWith(".json"));
 
  for (const file of files) {
    const raw = JSON.parse(fs.readFileSync(path.join(PROMPTS_DIR, file), "utf8"));
    const parsed = PromptEntrySchema.parse(raw);
    const totalWeight = parsed.variants
      .filter((v) => v.status !== "deprecated")
      .reduce((acc, v) => acc + v.weight, 0);
    if (totalWeight !== 100) {
      throw new Error(`[${parsed.promptId}] weight の合計が 100 ではありません (${totalWeight})`);
    }
    registry.set(parsed.promptId, parsed);
  }
  return registry;
}

// src/prompts/selector.ts
import crypto from "node:crypto";
import type { PromptEntry, PromptVariant } from "./types";
 
function stableHash(input: string): number {
  const hash = crypto.createHash("sha256").update(input).digest();
  const num = hash.readUInt32BE(0);
  return num % 100;
}
 
export function selectVariant(
  entry: PromptEntry,
  identityKey: string,
  options?: { forceVariantId?: string },
): PromptVariant {
  if (options?.forceVariantId) {
    const forced = entry.variants.find((v) => v.id === options.forceVariantId);
    if (forced) return forced;
  }
 
  const candidates = entry.variants.filter((v) => v.status !== "deprecated");
  if (candidates.length === 0) {
    throw new Error(`[${entry.promptId}] 配信可能な variant がありません`);
  }
 
  const bucket = stableHash(`${entry.promptId}::${identityKey}`);
  let cumulative = 0;
  for (const v of candidates) {
    cumulative += v.weight;
    if (bucket < cumulative) return v;
  }
  return candidates[candidates.length - 1]!;
}

// src/prompts/renderer.ts
export function render(template: string, vars: Record<string, string>): string {
  return template.replace(/\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/g, (_, key) => {
    if (!(key in vars)) {
      throw new Error(`未代入のテンプレート変数: ${key}`);
    }
    return vars[key]!;
  });
}

// src/prompts/invoke.ts
import Anthropic from "@anthropic-ai/sdk";
import { loadRegistry } from "./registry";
import { selectVariant } from "./selector";
import { render } from "./renderer";
import { recordTelemetry } from "./telemetry";
 
const client = new Anthropic();
const registry = loadRegistry();
 
export async function invokePrompt(params: {
  promptId: string;
  identityKey: string;
  vars: Record<string, string>;
  forceVariantId?: string;
}) {
  const entry = registry.get(params.promptId);
  if (!entry) throw new Error(`Unknown promptId: ${params.promptId}`);
 
  const variant = selectVariant(entry, params.identityKey, {
    forceVariantId: params.forceVariantId,
  });
 
  const renderedMessages = variant.messages.map((m) => ({
    role: m.role,
    content: render(m.content, params.vars),
  }));
 
  const startedAt = Date.now();
  let usage: Anthropic.Messages.Usage | undefined;
  let success = false;
 
  try {
    const res = await client.messages.create({
      model: variant.model,
      max_tokens: variant.maxTokens,
      temperature: variant.temperature,
      system: variant.systemPrompt,
      messages: renderedMessages,
    });
    usage = res.usage;
    success = true;
    return {
      promptId: entry.promptId,
      variantId: variant.id,
      output: res.content,
      usage,
    };
  } catch (err) {
    success = false;
    throw err;
  } finally {
    await recordTelemetry({
      promptId: entry.promptId,
      variantId: variant.id,
      identityKey: params.identityKey,
      latencyMs: Date.now() - startedAt,
      success,
      usage,
    });
  }
}

// src/prompts/telemetry.ts
import type { Anthropic } from "@anthropic-ai/sdk";
 
interface TelemetryRecord {
  promptId: string;
  variantId: string;
  identityKey: string;
  latencyMs: number;
  success: boolean;
  usage?: Anthropic.Messages.Usage;
  feedbackScore?: number;
}
 
export async function recordTelemetry(rec: TelemetryRecord) {
  const inputTokens = rec.usage?.input_tokens ?? 0;
  const outputTokens = rec.usage?.output_tokens ?? 0;
  const costUsd = estimateCostUsd(rec.promptId, rec.variantId, inputTokens, outputTokens);
 
  // env.DB は wrangler.toml で bind した D1
  // @ts-expect-error - グローバルから取得 (Workers 前提)
  await env.DB.prepare(
    `INSERT INTO prompt_invocations
     (prompt_id, variant_id, identity_key, latency_ms, success, input_tokens, output_tokens, cost_usd, created_at)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
  )
    .bind(
      rec.promptId,
      rec.variantId,
      rec.identityKey,
      rec.latencyMs,
      rec.success ? 1 : 0,
      inputTokens,
      outputTokens,
      costUsd,
      new Date().toISOString(),
    )
    .run();
}
 
function estimateCostUsd(_promptId: string, _variantId: string, inputTokens: number, outputTokens: number) {
  // モデル別単価表をここに集約 (例: Haiku 4.5)
  const inputUsdPerMTok = 0.25;
  const outputUsdPerMTok = 1.25;
  return (inputTokens * inputUsdPerMTok + outputTokens * outputUsdPerMTok) / 1_000_000;
}

// src/prompts/guardian.ts
import { loadRegistry } from "./registry";
 
interface VariantStats {
  count: number;
  successRate: number;
  p95LatencyMs: number;
  avgCostUsd: number;
}
 
async function fetchStats(promptId: string, variantId: string, sinceIso: string): Promise<VariantStats> {
  // @ts-expect-error - グローバルから取得 (Workers 前提)
  const result = await env.DB.prepare(
    `SELECT
       COUNT(*) AS count,
       AVG(CASE WHEN success = 1 THEN 1.0 ELSE 0.0 END) AS success_rate,
       AVG(cost_usd) AS avg_cost
     FROM prompt_invocations
     WHERE prompt_id = ? AND variant_id = ? AND created_at >= ?`
  )
    .bind(promptId, variantId, sinceIso)
    .first<{ count: number; success_rate: number; avg_cost: number }>();
 
  // p95 は近似 (D1 では PERCENTILE 関数が無いため、ORDER BY + OFFSET で代用)
  // @ts-expect-error
  const p95Row = await env.DB.prepare(
    `SELECT latency_ms FROM prompt_invocations
     WHERE prompt_id = ? AND variant_id = ? AND created_at >= ?
     ORDER BY latency_ms ASC
     LIMIT 1 OFFSET CAST((SELECT COUNT(*) FROM prompt_invocations
       WHERE prompt_id = ? AND variant_id = ? AND created_at >= ?) * 0.95 AS INTEGER)`
  )
    .bind(promptId, variantId, sinceIso, promptId, variantId, sinceIso)
    .first<{ latency_ms: number }>();
 
  return {
    count: result?.count ?? 0,
    successRate: result?.success_rate ?? 1,
    p95LatencyMs: p95Row?.latency_ms ?? 0,
    avgCostUsd: result?.avg_cost ?? 0,
  };
}
 
export async function evaluateAndRollback() {
  const registry = loadRegistry();
  const sinceIso = new Date(Date.now() - 15 * 60 * 1000).toISOString();
 
  for (const entry of registry.values()) {
    const canary = entry.variants.find((v) => v.status === "canary");
    if (!canary) continue;
 
    const stats = await fetchStats(entry.promptId, canary.id, sinceIso);
    if (stats.count < 50) continue; // サンプル不足なら判定保留
 
    const slo = canary.slo;
    const violations: string[] = [];
    if (stats.successRate < slo.minSuccessRate) {
      violations.push(`successRate ${stats.successRate.toFixed(3)} < ${slo.minSuccessRate}`);
    }
    if (stats.p95LatencyMs > slo.p95LatencyMs) {
      violations.push(`p95 ${stats.p95LatencyMs}ms > ${slo.p95LatencyMs}ms`);
    }
    if (stats.avgCostUsd > slo.maxCostPerCallUsd) {
      violations.push(`avgCost $${stats.avgCostUsd.toFixed(4)} > $${slo.maxCostPerCallUsd}`);
    }
 
    if (violations.length > 0) {
      console.warn(`[GUARDIAN] ${entry.promptId}/${canary.id} SLO 違反: ${violations.join(", ")}`);
      await rollback(entry.promptId, canary.id);
    }
  }
}
 
async function rollback(promptId: string, variantId: string) {
  // KV に「この variant は配信停止」フラグを書き、Selector がそれを参照する設計
  // @ts-expect-error
  await env.PROMPT_FLAGS.put(`disabled:${promptId}:${variantId}`, "1", { expirationTtl: 86400 });
}