Claude API × OpenTelemetry でAIアプリケーションのオブザーバビリティを構築する

// src/telemetry/setup.ts
import { NodeSDK } from "@opentelemetry/sdk-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
import { Resource } from "@opentelemetry/resources";
import {
  ATTR_SERVICE_NAME,
  ATTR_SERVICE_VERSION,
} from "@opentelemetry/semantic-conventions";
 
// アプリケーションのリソース情報を定義
const resource = new Resource({
  [ATTR_SERVICE_NAME]: "claude-ai-app",
  [ATTR_SERVICE_VERSION]: "1.0.0",
  "ai.provider": "anthropic",
  "deployment.environment": process.env.NODE_ENV || "development",
});
 
// トレースエクスポーター（OTLP over HTTP）
const traceExporter = new OTLPTraceExporter({
  url:
    process.env.OTEL_EXPORTER_OTLP_ENDPOINT + "/v1/traces" ||
    "http://localhost:4318/v1/traces",
});
 
// メトリクスエクスポーター
const metricExporter = new OTLPMetricExporter({
  url:
    process.env.OTEL_EXPORTER_OTLP_ENDPOINT + "/v1/metrics" ||
    "http://localhost:4318/v1/metrics",
});
 
// SDKの初期化
const sdk = new NodeSDK({
  resource,
  traceExporter,
  metricReader: new PeriodicExportingMetricReader({
    exporter: metricExporter,
    exportIntervalMillis: 30000, // 30秒間隔でメトリクスを送信
  }),
});
 
export function initTelemetry(): void {
  sdk.start();
  console.log("OpenTelemetry initialized for Claude AI application");
 
  // グレースフルシャットダウン
  process.on("SIGTERM", () => {
    sdk
      .shutdown()
      .then(() => console.log("Telemetry shut down"))
      .catch((err) => console.error("Telemetry shutdown error", err))
      .finally(() => process.exit(0));
  });
}

// src/index.ts
import { initTelemetry } from "./telemetry/setup";
initTelemetry(); // 他のimportより前に実行
 
import { startServer } from "./server";
startServer();

// src/telemetry/claude-instrumentation.ts
import { trace, metrics, SpanStatusCode, context } from "@opentelemetry/api";
import Anthropic from "@anthropic-ai/sdk";
 
// トレーサーとメーターの取得
const tracer = trace.getTracer("claude-api", "1.0.0");
const meter = metrics.getMeter("claude-api", "1.0.0");
 
// カスタムメトリクスの定義
const tokenCounter = meter.createCounter("claude.tokens.total", {
  description: "Total tokens consumed by Claude API calls",
  unit: "tokens",
});
 
const inputTokenCounter = meter.createCounter("claude.tokens.input", {
  description: "Input tokens sent to Claude API",
  unit: "tokens",
});
 
const outputTokenCounter = meter.createCounter("claude.tokens.output", {
  description: "Output tokens received from Claude API",
  unit: "tokens",
});
 
const requestDuration = meter.createHistogram("claude.request.duration", {
  description: "Duration of Claude API requests",
  unit: "ms",
});
 
const requestCounter = meter.createCounter("claude.requests.total", {
  description: "Total number of Claude API requests",
});
 
const errorCounter = meter.createCounter("claude.errors.total", {
  description: "Total number of Claude API errors",
});
 
// コスト計算用のレート（2026年3月時点）
const COST_PER_1K_INPUT: Record<string, number> = {
  "claude-opus-4-6": 0.015,
  "claude-sonnet-4-6": 0.003,
  "claude-haiku-4-5-20251001": 0.0008,
};
 
const COST_PER_1K_OUTPUT: Record<string, number> = {
  "claude-opus-4-6": 0.075,
  "claude-sonnet-4-6": 0.015,
  "claude-haiku-4-5-20251001": 0.004,
};
 
const costCounter = meter.createCounter("claude.cost.usd", {
  description: "Estimated cost of Claude API calls in USD",
  unit: "usd",
});
 
// インストルメンテーション済みのClaude APIクライアント
export class InstrumentedClaudeClient {
  private client: Anthropic;
 
  constructor(apiKey?: string) {
    this.client = new Anthropic({ apiKey });
  }
 
  async createMessage(
    params: Anthropic.MessageCreateParamsNonStreaming
  ): Promise<Anthropic.Message> {
    // スパンを作成してAPIコールをトレース
    return tracer.startActiveSpan(
      "claude.messages.create",
      {
        attributes: {
          "ai.model": params.model,
          "ai.max_tokens": params.max_tokens,
          "ai.message_count": params.messages.length,
          "ai.has_tools": params.tools ? "true" : "false",
          "ai.has_system": params.system ? "true" : "false",
        },
      },
      async (span) => {
        const startTime = Date.now();
 
        try {
          // Claude API コール
          const response = await this.client.messages.create(params);
          const duration = Date.now() - startTime;
 
          // レスポンスのメタデータをスパンに記録
          span.setAttributes({
            "ai.response.id": response.id,
            "ai.response.model": response.model,
            "ai.response.stop_reason": response.stop_reason || "unknown",
            "ai.tokens.input": response.usage.input_tokens,
            "ai.tokens.output": response.usage.output_tokens,
            "ai.tokens.total":
              response.usage.input_tokens + response.usage.output_tokens,
            "ai.duration_ms": duration,
          });
 
          // メトリクスの記録
          const labels = { model: response.model };
          inputTokenCounter.add(response.usage.input_tokens, labels);
          outputTokenCounter.add(response.usage.output_tokens, labels);
          tokenCounter.add(
            response.usage.input_tokens + response.usage.output_tokens,
            labels
          );
          requestDuration.record(duration, labels);
          requestCounter.add(1, { ...labels, status: "success" });
 
          // コスト推定
          const model = response.model;
          const inputCost =
            (response.usage.input_tokens / 1000) *
            (COST_PER_1K_INPUT[model] || 0.003);
          const outputCost =
            (response.usage.output_tokens / 1000) *
            (COST_PER_1K_OUTPUT[model] || 0.015);
          costCounter.add(inputCost + outputCost, labels);
 
          span.setStatus({ code: SpanStatusCode.OK });
          return response;
        } catch (error) {
          const duration = Date.now() - startTime;
 
          // エラー情報をスパンとメトリクスに記録
          span.setStatus({
            code: SpanStatusCode.ERROR,
            message: error instanceof Error ? error.message : "Unknown error",
          });
          span.recordException(error as Error);
 
          errorCounter.add(1, {
            model: params.model,
            error_type:
              error instanceof Anthropic.APIError
                ? `${error.status}`
                : "unknown",
          });
          requestDuration.record(duration, { model: params.model });
          requestCounter.add(1, {
            model: params.model,
            status: "error",
          });
 
          throw error;
        } finally {
          span.end();
        }
      }
    );
  }
}

// src/telemetry/agent-tracing.ts
import { trace, SpanStatusCode, SpanKind } from "@opentelemetry/api";
import { InstrumentedClaudeClient } from "./claude-instrumentation";
import Anthropic from "@anthropic-ai/sdk";
 
const tracer = trace.getTracer("claude-agent", "1.0.0");
 
interface ToolResult {
  tool_use_id: string;
  content: string;
}
 
// ツール実行のインストルメンテーション
async function executeToolWithTracing(
  toolName: string,
  toolInput: Record<string, unknown>,
  toolHandler: (input: Record<string, unknown>) => Promise<string>
): Promise<string> {
  return tracer.startActiveSpan(
    `tool.${toolName}`,
    {
      kind: SpanKind.INTERNAL,
      attributes: {
        "tool.name": toolName,
        "tool.input_keys": Object.keys(toolInput).join(","),
      },
    },
    async (span) => {
      try {
        const result = await toolHandler(toolInput);
        span.setAttributes({
          "tool.result_length": result.length,
          "tool.success": true,
        });
        span.setStatus({ code: SpanStatusCode.OK });
        return result;
      } catch (error) {
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: error instanceof Error ? error.message : "Tool execution failed",
        });
        span.recordException(error as Error);
        throw error;
      } finally {
        span.end();
      }
    }
  );
}
 
// エージェントループのインストルメンテーション
export async function runAgentWithTracing(
  client: InstrumentedClaudeClient,
  systemPrompt: string,
  userMessage: string,
  tools: Anthropic.Tool[],
  toolHandlers: Record<
    string,
    (input: Record<string, unknown>) => Promise<string>
  >,
  maxIterations: number = 10
): Promise<string> {
  return tracer.startActiveSpan(
    "agent.workflow",
    {
      attributes: {
        "agent.max_iterations": maxIterations,
        "agent.tools_available": tools.map((t) => t.name).join(","),
        "agent.system_prompt_length": systemPrompt.length,
      },
    },
    async (rootSpan) => {
      const messages: Anthropic.MessageParam[] = [
        { role: "user", content: userMessage },
      ];
      let iteration = 0;
      let totalInputTokens = 0;
      let totalOutputTokens = 0;
 
      try {
        while (iteration < maxIterations) {
          iteration++;
 
          // 各イテレーションを子スパンとして記録
          const response = await tracer.startActiveSpan(
            `agent.iteration.${iteration}`,
            { attributes: { "agent.iteration": iteration } },
            async (iterSpan) => {
              const resp = await client.createMessage({
                model: "claude-sonnet-4-6",
                max_tokens: 4096,
                system: systemPrompt,
                tools,
                messages,
              });
 
              totalInputTokens += resp.usage.input_tokens;
              totalOutputTokens += resp.usage.output_tokens;
 
              iterSpan.setAttributes({
                "agent.stop_reason": resp.stop_reason || "unknown",
                "agent.content_blocks": resp.content.length,
              });
              iterSpan.end();
              return resp;
            }
          );
 
          // 終了条件: ツール呼び出しがなければ完了
          if (response.stop_reason === "end_turn") {
            const textBlock = response.content.find(
              (b) => b.type === "text"
            );
            const finalAnswer =
              textBlock && "text" in textBlock ? textBlock.text : "";
 
            rootSpan.setAttributes({
              "agent.total_iterations": iteration,
              "agent.total_input_tokens": totalInputTokens,
              "agent.total_output_tokens": totalOutputTokens,
              "agent.final_answer_length": finalAnswer.length,
            });
            rootSpan.setStatus({ code: SpanStatusCode.OK });
            return finalAnswer;
          }
 
          // ツール呼び出しの処理
          const toolUseBlocks = response.content.filter(
            (b) => b.type === "tool_use"
          );
 
          const toolResults: ToolResult[] = [];
          for (const block of toolUseBlocks) {
            if (block.type === "tool_use") {
              const handler = toolHandlers[block.name];
              if (handler) {
                const result = await executeToolWithTracing(
                  block.name,
                  block.input as Record<string, unknown>,
                  handler
                );
                toolResults.push({
                  tool_use_id: block.id,
                  content: result,
                });
              }
            }
          }
 
          // 会話にアシスタント応答とツール結果を追加
          messages.push({ role: "assistant", content: response.content });
          messages.push({
            role: "user",
            content: toolResults.map((r) => ({
              type: "tool_result" as const,
              tool_use_id: r.tool_use_id,
              content: r.content,
            })),
          });
        }
 
        rootSpan.setAttributes({
          "agent.total_iterations": iteration,
          "agent.max_iterations_reached": true,
        });
        rootSpan.setStatus({ code: SpanStatusCode.OK });
        return "Maximum iterations reached";
      } catch (error) {
        rootSpan.setStatus({
          code: SpanStatusCode.ERROR,
          message: error instanceof Error ? error.message : "Agent workflow failed",
        });
        rootSpan.recordException(error as Error);
        throw error;
      } finally {
        rootSpan.end();
      }
    }
  );
}

// src/telemetry/cost-dashboard-metrics.ts
import { metrics } from "@opentelemetry/api";
 
const meter = metrics.getMeter("claude-cost-dashboard", "1.0.0");
 
// 日次コスト追跡（ObservableGauge で外部データソースから取得）
const dailyCostGauge = meter.createObservableGauge(
  "claude.cost.daily_estimate_usd",
  {
    description: "Estimated daily cost based on current usage rate",
    unit: "usd",
  }
);
 
// コストのバジェット使用率
const budgetUtilization = meter.createObservableGauge(
  "claude.budget.utilization_percent",
  {
    description: "Percentage of monthly budget consumed",
    unit: "percent",
  }
);
 
// モデル別のリクエスト分布
const modelDistribution = meter.createHistogram(
  "claude.model.request_distribution",
  {
    description: "Distribution of requests across models",
  }
);
 
// コストアラートの閾値管理
interface CostTracker {
  dailyTotal: number;
  monthlyTotal: number;
  monthlyBudget: number;
  lastResetDate: string;
}
 
const costTracker: CostTracker = {
  dailyTotal: 0,
  monthlyTotal: 0,
  monthlyBudget: parseFloat(process.env.MONTHLY_BUDGET_USD || "500"),
  lastResetDate: new Date().toISOString().split("T")[0],
};
 
// Observable コールバックの登録
dailyCostGauge.addCallback((result) => {
  result.observe(costTracker.dailyTotal);
});
 
budgetUtilization.addCallback((result) => {
  const utilization =
    (costTracker.monthlyTotal / costTracker.monthlyBudget) * 100;
  result.observe(utilization);
});
 
// コスト記録関数（InstrumentedClaudeClient から呼び出し）
export function recordCost(
  model: string,
  inputTokens: number,
  outputTokens: number
): void {
  const today = new Date().toISOString().split("T")[0];
  if (today !== costTracker.lastResetDate) {
    costTracker.dailyTotal = 0;
    costTracker.lastResetDate = today;
  }
 
  const costRates: Record<string, { input: number; output: number }> = {
    "claude-opus-4-6": { input: 0.015, output: 0.075 },
    "claude-sonnet-4-6": { input: 0.003, output: 0.015 },
    "claude-haiku-4-5-20251001": { input: 0.0008, output: 0.004 },
  };
 
  const rates = costRates[model] || { input: 0.003, output: 0.015 };
  const cost =
    (inputTokens / 1000) * rates.input +
    (outputTokens / 1000) * rates.output;
 
  costTracker.dailyTotal += cost;
  costTracker.monthlyTotal += cost;
 
  // バジェット超過アラート
  const utilization =
    (costTracker.monthlyTotal / costTracker.monthlyBudget) * 100;
  if (utilization > 80) {
    console.warn(
      `⚠️ Budget alert: ${utilization.toFixed(1)}% of monthly budget consumed`
    );
  }
}

# 直近1時間のトークン消費量（モデル別）
sum(rate(claude_tokens_total[1h])) by (model)
 
# 1時間あたりの推定コスト
sum(rate(claude_cost_usd[1h])) * 3600
 
# エラー率（5分間のウィンドウ）
sum(rate(claude_errors_total[5m])) / sum(rate(claude_requests_total[5m])) * 100
 
# P95 レイテンシ
histogram_quantile(0.95, rate(claude_request_duration_bucket[5m]))
 
# バジェット消費率の推移
claude_budget_utilization_percent

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      http:
        endpoint: 0.0.0.0:4318
      grpc:
        endpoint: 0.0.0.0:4317
 
processors:
  # バッチ処理でネットワーク効率を向上
  batch:
    timeout: 10s
    send_batch_size: 1024
 
  # AI関連のスパンにメタデータを付加
  attributes:
    actions:
      - key: "service.layer"
        value: "ai-inference"
        action: upsert
 
  # トレーシングのサンプリング（本番環境でのコスト削減）
  probabilistic_sampler:
    sampling_percentage: 25  # 25%のトレースを保持
 
  # 高コストのリクエストは必ず保持
  tail_sampling:
    decision_wait: 10s
    policies:
      # エラーは100%保持
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      # 高レイテンシ（3秒以上）は100%保持
      - name: slow-requests
        type: latency
        latency:
          threshold_ms: 3000
      # Opus モデルの呼び出しは100%保持（高コスト）
      - name: expensive-models
        type: string_attribute
        string_attribute:
          key: ai.model
          values: ["claude-opus-4-6"]
 
exporters:
  otlphttp/tempo:
    endpoint: http://tempo:4318
  prometheus:
    endpoint: 0.0.0.0:8889
    namespace: claude
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, attributes, tail_sampling]
      exporters: [otlphttp/tempo]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

// src/telemetry/cache-metrics.ts
import { metrics } from "@opentelemetry/api";
import Anthropic from "@anthropic-ai/sdk";
 
const meter = metrics.getMeter("claude-cache", "1.0.0");
 
const cacheHitCounter = meter.createCounter("claude.cache.hits", {
  description: "Number of prompt cache hits",
});
 
const cacheMissCounter = meter.createCounter("claude.cache.misses", {
  description: "Number of prompt cache misses",
});
 
const cacheSavingsCounter = meter.createCounter(
  "claude.cache.savings_tokens",
  {
    description: "Tokens saved by prompt caching",
    unit: "tokens",
  }
);
 
// キャッシュ効果をモニタリングする関数
export function recordCacheMetrics(
  response: Anthropic.Message,
  model: string
): void {
  const usage = response.usage as Record<string, number>;
 
  // cache_creation_input_tokens と cache_read_input_tokens を取得
  const cacheCreation = usage.cache_creation_input_tokens || 0;
  const cacheRead = usage.cache_read_input_tokens || 0;
 
  if (cacheRead > 0) {
    cacheHitCounter.add(1, { model });
    // キャッシュ読み取りは通常料金の10%なので90%節約
    cacheSavingsCounter.add(Math.floor(cacheRead * 0.9), { model });
  } else if (cacheCreation > 0) {
    cacheMissCounter.add(1, { model });
  }
}

# Prompt Caching のヒット率
sum(rate(claude_cache_hits[1h])) /
(sum(rate(claude_cache_hits[1h])) + sum(rate(claude_cache_misses[1h]))) * 100
 
# キャッシュによる推定節約トークン数（1日あたり）
sum(increase(claude_cache_savings_tokens[24h]))

# docker-compose.observability.yml
version: "3.9"
services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.96.0
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4317:4317"   # gRPC
      - "4318:4318"   # HTTP
      - "8889:8889"   # Prometheus metrics
 
  tempo:
    image: grafana/tempo:2.4.0
    command: ["-config.file=/etc/tempo.yaml"]
    volumes:
      - ./tempo.yaml:/etc/tempo.yaml
    ports:
      - "3200:3200"
 
  prometheus:
    image: prom/prometheus:v2.50.0
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"
 
  grafana:
    image: grafana/grafana:10.3.0
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
    ports:
      - "3001:3000"
    depends_on:
      - tempo
      - prometheus