Don't Accept an Agent's Numbers and Citations As-Is — A Verification Gate Built on a Dedicated Auditor Subagent

// verify-number.ts
import type { NumericClaim } from "./claims";
 
export type Dataset = Record<string, number[]>;
 
const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);
 
// metric key -> function that determines the value from raw data
export const metricFns: Record<string, (d: Dataset) => number> = {
  "clicks.total": (d) => sum(d.clicks),
  "clicks.avgPerDay": (d) => sum(d.clicks) / d.clicks.length,
  "wowChangePct": (d) => {
    const prev = sum(d.clicksPrevWeek);
    const cur = sum(d.clicksThisWeek);
    return prev === 0 ? NaN : ((cur - prev) / prev) * 100;
  },
};
 
const DEFAULT_TOLERANCE = 0.005; // relative 0.5%
 
export function verifyNumber(
  claim: NumericClaim,
  data: Dataset
): { ok: boolean; reason: string; expected?: number } {
  const fn = metricFns[claim.metric];
  if (!fn) {
    // an unknown metric is "unverifiable" = fail (fail-closed)
    return { ok: false, reason: `unregistered metric: ${claim.metric}` };
  }
  const expected = fn(data);
  if (!Number.isFinite(expected)) {
    return { ok: false, reason: "recomputed value is not finite", expected };
  }
  const tol = claim.tolerance ?? DEFAULT_TOLERANCE;
  const denom = Math.abs(expected) || 1;
  const relErr = Math.abs(claim.value - expected) / denom;
  return relErr <= tol
    ? { ok: true, reason: "match", expected }
    : {
        ok: false,
        reason: `claimed ${claim.value} vs recomputed ${expected.toFixed(2)} differ by ${(relErr * 100).toFixed(2)}%`,
        expected,
      };
}

// verify-citation.ts
import type { CitationClaim } from "./claims";
 
export type Sources = Record<string, string>; // sourceId -> source snippet
 
// absorb width, whitespace, and punctuation variance
function normalize(s: string): string {
  return s
    .normalize("NFKC")
    .replace(/\s+/g, "")
    .replace(/[「」『』()（）,.、。]/g, "")
    .toLowerCase();
}
 
export function verifyCitation(
  claim: CitationClaim,
  sources: Sources
): { ok: boolean; reason: string } {
  const src = sources[claim.sourceId];
  if (!src) {
    return { ok: false, reason: `source not found: ${claim.sourceId}` };
  }
  const q = normalize(claim.quote);
  if (q.length < 8) {
    // a too-short quote matches by chance, so fail it
    return { ok: false, reason: "quote too short to match" };
  }
  return normalize(src).includes(q)
    ? { ok: true, reason: "matches source" }
    : { ok: false, reason: "quote does not exist in source" };
}

<!-- .claude/agents/claim-auditor.md -->
---
name: claim-auditor
description: Judges only whether a generated summary introduced numbers/citations absent from its claim ledger
tools: []
---
You are a dedicated auditor. You receive only the "summary text" and the "claim ledger" (a list of ids and statements).
List every number or specific citation appearing in the summary that maps to no claim in the ledger.
Judge conservatively: if even one number/citation lacks ledger support, report it as unverified.
Output only JSON of shape {"unlisted": string[]}. Do not infer or fill in the generator's intent.

// audit-summary.ts (excerpt)
import Anthropic from "@anthropic-ai/sdk";
const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
 
export async function auditSummary(
  summary: string,
  claimStatements: string[]
): Promise<{ unlisted: string[] }> {
  const msg = await client.messages.create({
    model: "claude-haiku-4-5-20251001", // a light model is enough for auditing
    max_tokens: 512,
    system:
      'List numbers/citations in the summary that map to none of the given claims. Output only JSON {"unlisted": string[]}.',
    messages: [
      {
        role: "user",
        content: `# Summary\n${summary}\n\n# Claims\n${claimStatements.join("\n")}`,
      },
    ],
  });
  const text = msg.content.find((b) => b.type === "text");
  return JSON.parse(text && "text" in text ? text.text : '{"unlisted":[]}');
}

// ❌ Before: accept the artifact's summary and publish it directly
const artifact = await generateReport(data);
await publish(artifact.summary); // no one verified the numbers or citations

// ✅ After: always go through a pre-acceptance gate
import { verifyNumber } from "./verify-number";
import { verifyCitation } from "./verify-citation";
import { auditSummary } from "./audit-summary";
import type { Artifact, Dataset, Sources } from "./types";
 
export async function gate(
  artifact: Artifact,
  data: Dataset,
  sources: Sources
): Promise<{ ok: boolean; failures: string[] }> {
  const failures: string[] = [];
 
  for (const c of artifact.claims) {
    if (c.kind === "number") {
      const r = verifyNumber(c, data);
      if (!r.ok) failures.push(`[${c.id}] ${r.reason}`);
    } else {
      const r = verifyCitation(c, sources);
      if (!r.ok) failures.push(`[${c.id}] ${r.reason}`);
    }
  }
 
  const audit = await auditSummary(
    artifact.summary,
    artifact.claims.map((c) => `${c.id}: ${c.statement}`)
  );
  for (const u of audit.unlisted) {
    failures.push(`[unlisted] not in ledger: ${u}`);
  }
 
  return { ok: failures.length === 0, failures };
}
 
// caller
const artifact = await generateReport(data);
const result = await gate(artifact, data, sources);
if (!result.ok) {
  console.error("rejected:", result.failures);
  // do not publish. log the reasons and route to regeneration or human review
} else {
  await publish(artifact.summary);
}