Dynamic Workflows の規模と effort を、勘ではなく手元の台帳で決める

# ~/.claude/settings.json に置く場合は env ブロックへ。
# まずは1セッションだけ試すならシェルで十分です。
export CLAUDE_CODE_ENABLE_TELEMETRY=1
export OTEL_METRICS_EXPORTER=console
export OTEL_LOGS_EXPORTER=console
export OTEL_METRIC_EXPORT_INTERVAL=10000   # 既定は60秒。短い実行を取りこぼさない
 
# 実行のたびにファイルへ落とす（stdout に混ざるので分離しておく）
mkdir -p ~/.wf-ledger/raw
claude -p "$(cat ~/.wf-ledger/task.md)" 2> ~/.wf-ledger/raw/$(date +%s).otel.jsonl

#!/usr/bin/env bash
# ~/.wf-ledger/sweep.sh — 同一タスクを size × effort で掃き出す
set -euo pipefail
 
TASK_FILE="${1:?usage: sweep.sh <task.md> [repeats]}"
REPEATS="${2:-3}"
LEDGER="$HOME/.wf-ledger"
mkdir -p "$LEDGER/raw"
 
export CLAUDE_CODE_ENABLE_TELEMETRY=1
export OTEL_METRICS_EXPORTER=console
export OTEL_METRIC_EXPORT_INTERVAL=10000
 
for size in small medium large; do
  for effort in high xhigh; do
    for i in $(seq 1 "$REPEATS"); do
      run_id="${size}-${effort}-${i}-$(date +%s)"
      start=$(date +%s%3N)
 
      # size は設定ファイル側を書き換えてから起動する。
      # 対話の /config と同じ値が反映されます。
      python3 - "$size" <<'PY'
import json, pathlib, sys
p = pathlib.Path.home() / ".claude" / "settings.json"
cfg = json.loads(p.read_text()) if p.exists() else {}
cfg["dynamicWorkflowSize"] = sys.argv[1]
p.write_text(json.dumps(cfg, indent=2))
PY
 
      claude -p "$(cat "$TASK_FILE")" \
        --output-format json \
        > "$LEDGER/raw/${run_id}.result.json" \
        2> "$LEDGER/raw/${run_id}.otel.jsonl" || echo "run failed: $run_id" >&2
 
      end=$(date +%s%3N)
      echo "{\"run_id\":\"${run_id}\",\"size\":\"${size}\",\"effort\":\"${effort}\",\"elapsed_ms\":$((end - start))}" \
        >> "$LEDGER/runs.jsonl"
      echo "done ${run_id} ($(( (end - start) / 1000 ))s)" >&2
    done
  done
done

#!/usr/bin/env node
// ~/.wf-ledger/ledger.mjs — raw/*.otel.jsonl を条件別の台帳に畳む
import { readFileSync, readdirSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
 
const DIR = join(homedir(), ".wf-ledger");
const RAW = join(DIR, "raw");
 
// console エクスポータは1メトリクスを複数行の JSON で吐くため、
// 行単位ではなく { ... } のブロック単位で拾う。
function extractMetrics(text) {
  const out = [];
  const re = /"name"\s*:\s*"(claude_code\.[a-z._]+)"[\s\S]*?"value"\s*:\s*([\d.]+)([\s\S]*?"attributes"\s*:\s*\{([\s\S]*?)\})?/g;
  let m;
  while ((m = re.exec(text)) !== null) {
    out.push({ name: m[1], value: Number(m[2]), attrs: m[4] ?? "" });
  }
  return out;
}
 
const runs = readFileSync(join(DIR, "runs.jsonl"), "utf8")
  .trim().split("\n").map((l) => JSON.parse(l));
 
const rows = runs.map((run) => {
  const file = readdirSync(RAW).find((f) => f.startsWith(run.run_id) && f.endsWith(".otel.jsonl"));
  if (!file) return { ...run, tokens: null, cost: null, note: "otel missing" };
 
  const text = readFileSync(join(RAW, file), "utf8");
  const metrics = extractMetrics(text);
 
  const tokens = metrics
    .filter((x) => x.name === "claude_code.token.usage")
    .filter((x) => !/cacheRead/.test(x.attrs))   // キャッシュ読みは費用が別枠
    .reduce((s, x) => s + x.value, 0);
  const cost = metrics
    .filter((x) => x.name === "claude_code.cost.usage")
    .reduce((s, x) => s + x.value, 0);
 
  const result = JSON.parse(readFileSync(join(RAW, `${run.run_id}.result.json`), "utf8"));
  return { ...run, tokens, cost: Number(cost.toFixed(4)), passed: gradeOutput(result) };
});
 
// 合否は自分のタスクの受け入れ条件で判定する。ここは各自の要件に置き換える。
function gradeOutput(result) {
  const text = typeof result === "string" ? result : (result.result ?? JSON.stringify(result));
  const hasAllSections = ["## 前提", "## 検証", "## 結論"].every((h) => text.includes(h));
  const hasCitations = (text.match(/https?:\/\//g) ?? []).length >= 3;
  return hasAllSections && hasCitations;
}
 
// 条件ごとに畳む
const grouped = {};
for (const r of rows) {
  const key = `${r.size}/${r.effort}`;
  (grouped[key] ??= []).push(r);
}
 
const avg = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
console.table(
  Object.entries(grouped).map(([key, rs]) => ({
    condition: key,
    runs: rs.length,
    tokens_avg: Math.round(avg(rs.map((r) => r.tokens ?? 0))),
    cost_avg: Number(avg(rs.map((r) => r.cost ?? 0)).toFixed(4)),
    elapsed_s: Math.round(avg(rs.map((r) => r.elapsed_ms)) / 1000),
    pass_rate: `${Math.round((rs.filter((r) => r.passed).length / rs.length) * 100)}%`,
  })),
);