Claude Code の実行を OpenTelemetry で観測する — エージェント処理を本番品質で可視化する設計ガイド

// otel-init.mjs — エージェント実行時に最初に読み込む
import { NodeSDK } from "@opentelemetry/sdk-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { Resource } from "@opentelemetry/resources";
import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
 
const sdk = new NodeSDK({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: "claude-code-agent",
    [SemanticResourceAttributes.SERVICE_VERSION]: process.env.AGENT_VERSION ?? "dev",
    "deployment.environment": process.env.NODE_ENV ?? "development",
  }),
  traceExporter: new OTLPTraceExporter({
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://localhost:4318/v1/traces",
    headers: process.env.OTEL_AUTH_HEADER
      ? { Authorization: process.env.OTEL_AUTH_HEADER }
      : undefined,
  }),
});
 
try {
  sdk.start();
  console.error("[otel] SDK started");
} catch (err) {
  // 観測性の初期化失敗で本処理を止めない（fail-open が原則）
  console.error("[otel] init failed, continuing without tracing:", err);
}
 
process.on("SIGTERM", async () => {
  try {
    await sdk.shutdown();
  } catch (err) {
    console.error("[otel] shutdown error:", err);
  }
});

// hooks/session-start.mjs
import { trace, context } from "@opentelemetry/api";
import { writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
 
import "./otel-init.mjs"; // SDK 初期化を必ず最初に行う
 
const tracer = trace.getTracer("claude-code-session");
 
const span = tracer.startSpan("claude_code.session", {
  attributes: {
    "session.id": process.env.CLAUDE_SESSION_ID ?? "unknown",
    "session.cwd": process.cwd(),
    "session.user": process.env.USER ?? "unknown",
  },
});
 
// span context を後続フックに渡すため、ファイルに spanId/traceId を書き出す
const ctx = trace.setSpan(context.active(), span);
const spanCtx = span.spanContext();
const handoff = {
  traceId: spanCtx.traceId,
  spanId: spanCtx.spanId,
  startedAt: Date.now(),
};
 
const stateFile = join(tmpdir(), `claude-otel-${process.env.CLAUDE_SESSION_ID}.json`);
writeFileSync(stateFile, JSON.stringify(handoff));
console.error(`[otel] root span started: ${spanCtx.traceId}`);
 
// セッションが続く間スパンを生かしておくため、明示終了は SessionEnd フックで行う

// hooks/pre-tool-use.mjs — ツール実行直前
import { trace, context } from "@opentelemetry/api";
import { readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import "./otel-init.mjs";
 
const tracer = trace.getTracer("claude-code-tool");
 
const stateFile = join(tmpdir(), `claude-otel-${process.env.CLAUDE_SESSION_ID}.json`);
const state = JSON.parse(readFileSync(stateFile, "utf8"));
 
// 親スパンを再構築（trace.setSpanContext で復元）
const parentCtx = trace.setSpanContext(context.active(), {
  traceId: state.traceId,
  spanId: state.spanId,
  traceFlags: 1,
  isRemote: true,
});
 
const toolName = process.env.CLAUDE_TOOL_NAME ?? "unknown";
const toolInput = process.env.CLAUDE_TOOL_INPUT_PREVIEW ?? "";
 
const span = tracer.startSpan(
  `tool.${toolName}`,
  {
    attributes: {
      "tool.name": toolName,
      "tool.input.length": toolInput.length,
      // プロンプトの全文は流さず、長さだけを属性化（PII 漏れと容量爆発を防ぐ）
      "tool.input.preview": toolInput.slice(0, 200),
    },
  },
  parentCtx
);
 
// PostToolUse で復元できるよう、ツール呼び出し用の span を別ファイルで保持
const toolStateFile = join(
  tmpdir(),
  `claude-otel-tool-${process.env.CLAUDE_TOOL_INVOCATION_ID}.json`
);
writeFileSync(
  toolStateFile,
  JSON.stringify({
    traceId: span.spanContext().traceId,
    spanId: span.spanContext().spanId,
    startedAt: Date.now(),
  })
);

// hooks/post-tool-use.mjs — ツール実行直後
import { trace, context, SpanStatusCode } from "@opentelemetry/api";
import { readFileSync, unlinkSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import "./otel-init.mjs";
 
const stateFile = join(
  tmpdir(),
  `claude-otel-tool-${process.env.CLAUDE_TOOL_INVOCATION_ID}.json`
);
const state = JSON.parse(readFileSync(stateFile, "utf8"));
 
// SDK の active context に積めない場合は、span の参照を直接保持して end する設計が安全
const tracer = trace.getTracer("claude-code-tool");
const span = tracer.startSpan(`tool.end.${process.env.CLAUDE_TOOL_NAME}`, {
  startTime: state.startedAt,
  links: [
    {
      context: {
        traceId: state.traceId,
        spanId: state.spanId,
        traceFlags: 1,
        isRemote: true,
      },
    },
  ],
});
 
const exitCode = Number(process.env.CLAUDE_TOOL_EXIT_CODE ?? "0");
const duration = Date.now() - state.startedAt;
 
span.setAttributes({
  "tool.exit_code": exitCode,
  "tool.duration_ms": duration,
});
 
if (exitCode !== 0) {
  span.setStatus({ code: SpanStatusCode.ERROR, message: `exit ${exitCode}` });
}
 
span.end();
unlinkSync(stateFile);

// mcp-tracing-proxy.mjs — MCP サーバーへのリクエストに traceparent を付与
import { trace, context, propagation } from "@opentelemetry/api";
 
export async function callMcpWithTrace(serverUrl, method, params) {
  const tracer = trace.getTracer("mcp-client");
  const span = tracer.startSpan(`mcp.${method}`, {
    attributes: {
      "mcp.server.url": serverUrl,
      "mcp.method": method,
      "mcp.params.keys": Object.keys(params ?? {}).join(","),
    },
  });
 
  const headers = { "content-type": "application/json" };
  // W3C Trace Context (traceparent) を inject
  propagation.inject(trace.setSpan(context.active(), span), headers);
 
  try {
    const res = await fetch(serverUrl, {
      method: "POST",
      headers,
      body: JSON.stringify({ jsonrpc: "2.0", method, params, id: 1 }),
    });
    if (!res.ok) {
      throw new Error(`MCP HTTP ${res.status}`);
    }
    const json = await res.json();
    span.setAttribute("mcp.response.has_error", Boolean(json.error));
    return json;
  } catch (err) {
    span.recordException(err);
    span.setStatus({ code: 2 /* ERROR */, message: String(err) });
    throw err;
  } finally {
    span.end();
  }
}

// hooks/post-tool-use.mjs に追記する属性例
span.setAttributes({
  "claude.model": process.env.CLAUDE_MODEL ?? "unknown",
  "claude.tokens.input": Number(process.env.CLAUDE_INPUT_TOKENS ?? 0),
  "claude.tokens.output": Number(process.env.CLAUDE_OUTPUT_TOKENS ?? 0),
  "claude.cache.read_tokens": Number(process.env.CLAUDE_CACHE_READ_TOKENS ?? 0),
  "claude.cache.creation_tokens": Number(process.env.CLAUDE_CACHE_CREATION_TOKENS ?? 0),
  // コスト計算は事後集計に任せ、生のトークン数だけ記録するのが拡張性が高い
});

// otel-metrics.mjs — otel-init.mjs の sdk.start() 後に読み込む
import { metrics } from "@opentelemetry/api";
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
import { Resource } from "@opentelemetry/resources";
 
const meterProvider = new MeterProvider({
  resource: new Resource({ "service.name": "claude-code-agent" }),
  readers: [
    new PeriodicExportingMetricReader({
      exporter: new OTLPMetricExporter({
        url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.replace("/v1/traces", "/v1/metrics"),
      }),
      exportIntervalMillis: 30_000,
    }),
  ],
});
 
metrics.setGlobalMeterProvider(meterProvider);
const meter = metrics.getMeter("claude-code-agent");
 
export const toolDurationHistogram = meter.createHistogram("claude.tool.duration", {
  unit: "ms",
  description: "1回のツール呼び出しにかかった時間",
});
 
export const tokensCounter = meter.createCounter("claude.tokens.total", {
  unit: "1",
  description: "種類別の累積トークン消費量",
});

// hooks/post-tool-use.mjs に追記
import { toolDurationHistogram, tokensCounter } from "../otel-metrics.mjs";
 
toolDurationHistogram.record(duration, {
  "tool.name": process.env.CLAUDE_TOOL_NAME ?? "unknown",
  "tool.exit_status": exitCode === 0 ? "ok" : "error",
});
tokensCounter.add(Number(process.env.CLAUDE_INPUT_TOKENS ?? 0), {
  "claude.model": process.env.CLAUDE_MODEL ?? "unknown",
  "token.type": "input",
});
tokensCounter.add(Number(process.env.CLAUDE_OUTPUT_TOKENS ?? 0), {
  "claude.model": process.env.CLAUDE_MODEL ?? "unknown",
  "token.type": "output",
});

# otel-collector-config.yaml — Tail-Based サンプリングの例
processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 50000
    expected_new_traces_per_sec: 1000
    policies:
      - name: errors-policy
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow-traces
        type: latency
        latency: { threshold_ms: 5000 }
      - name: probabilistic
        type: probabilistic
        probabilistic: { sampling_percentage: 10 }
exporters:
  otlphttp:
    endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [tail_sampling]
      exporters: [otlphttp]

# otel_init.py — Python ラッパーの先頭で import する
from opentelemetry import trace
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
import os
 
resource = Resource.create({
    "service.name": "claude-code-agent",
    "service.version": os.environ.get("AGENT_VERSION", "dev"),
    "deployment.environment": os.environ.get("ENV", "development"),
})
 
provider = TracerProvider(resource=resource)
exporter = OTLPSpanExporter(
    endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318/v1/traces"),
    headers={"authorization": os.environ["OTEL_AUTH_HEADER"]} if "OTEL_AUTH_HEADER" in os.environ else None,
)
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)
tracer = trace.get_tracer("claude-code-agent")

# hooks/post_tool_use.py
import os, json, time
from otel_init import tracer, provider
from opentelemetry.trace import Status, StatusCode
 
state_file = f"/tmp/claude-otel-tool-{os.environ['CLAUDE_TOOL_INVOCATION_ID']}.json"
with open(state_file) as f:
    state = json.load(f)
 
duration_ms = int((time.time() * 1000) - state["startedAt"])
exit_code = int(os.environ.get("CLAUDE_TOOL_EXIT_CODE", "0"))
 
with tracer.start_as_current_span(f"tool.end.{os.environ['CLAUDE_TOOL_NAME']}") as span:
    span.set_attribute("tool.duration_ms", duration_ms)
    span.set_attribute("tool.exit_code", exit_code)
    if exit_code != 0:
        span.set_status(Status(StatusCode.ERROR, f"exit {exit_code}"))
 
# スパンが送り切れるまで最大 1.5 秒待ってから終了する
try:
    provider.force_flush(timeout_millis=1500)
finally:
    provider.shutdown()

// redact.mjs — プロジェクト固有ルールを足せる最小マスキング
const PATTERNS = [
  { rx: /sk-[A-Za-z0-9]{16,}/g, mask: "[REDACTED:api_key]" },
  { rx: /Bearer\s+[A-Za-z0-9._-]+/gi, mask: "[REDACTED:bearer]" },
  { rx: /[\w.+-]+@[\w-]+\.[\w.-]+/g, mask: "[REDACTED:email]" },
  { rx: /\d{16}/g, mask: "[REDACTED:card]" },
];
 
export function redact(input) {
  let out = input;
  for (const { rx, mask } of PATTERNS) out = out.replace(rx, mask);
  return out;
}

# ツール別 P95 レイテンシ（5分窓・ツール名でファセット）
histogram_quantile(0.95, sum(rate(claude_tool_duration_bucket[5m])) by (tool_name, le))
 
# 直近1時間のセッション失敗率
sum(rate(claude_session_failures_total[1h]))
  / sum(rate(claude_session_total[1h]))
 
# 直近24時間のモデル別トークン消費
sum by (claude_model) (increase(claude_tokens_total[24h]))

# モデル別 P95 レイテンシ — モデル選定を駆動する比較
histogram_quantile(0.95,
  sum(rate(claude_tool_duration_bucket[7d])) by (claude_model, le)
)