Building AI Application Observability with Claude API and OpenTelemetry

// src/telemetry/setup.ts
import { NodeSDK } from "@opentelemetry/sdk-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
import { Resource } from "@opentelemetry/resources";
import {
  ATTR_SERVICE_NAME,
  ATTR_SERVICE_VERSION,
} from "@opentelemetry/semantic-conventions";
 
// Define application resource information
const resource = new Resource({
  [ATTR_SERVICE_NAME]: "claude-ai-app",
  [ATTR_SERVICE_VERSION]: "1.0.0",
  "ai.provider": "anthropic",
  "deployment.environment": process.env.NODE_ENV || "development",
});
 
// Trace exporter (OTLP over HTTP)
const traceExporter = new OTLPTraceExporter({
  url:
    process.env.OTEL_EXPORTER_OTLP_ENDPOINT + "/v1/traces" ||
    "http://localhost:4318/v1/traces",
});
 
// Metrics exporter
const metricExporter = new OTLPMetricExporter({
  url:
    process.env.OTEL_EXPORTER_OTLP_ENDPOINT + "/v1/metrics" ||
    "http://localhost:4318/v1/metrics",
});
 
// Initialize the SDK
const sdk = new NodeSDK({
  resource,
  traceExporter,
  metricReader: new PeriodicExportingMetricReader({
    exporter: metricExporter,
    exportIntervalMillis: 30000, // Export metrics every 30 seconds
  }),
});
 
export function initTelemetry(): void {
  sdk.start();
  console.log("OpenTelemetry initialized for Claude AI application");
 
  // Graceful shutdown handler
  process.on("SIGTERM", () => {
    sdk
      .shutdown()
      .then(() => console.log("Telemetry shut down"))
      .catch((err) => console.error("Telemetry shutdown error", err))
      .finally(() => process.exit(0));
  });
}

// src/index.ts
import { initTelemetry } from "./telemetry/setup";
initTelemetry(); // Must run before other imports
 
import { startServer } from "./server";
startServer();

// src/telemetry/claude-instrumentation.ts
import { trace, metrics, SpanStatusCode, context } from "@opentelemetry/api";
import Anthropic from "@anthropic-ai/sdk";
 
// Get tracer and meter instances
const tracer = trace.getTracer("claude-api", "1.0.0");
const meter = metrics.getMeter("claude-api", "1.0.0");
 
// Define custom metrics
const tokenCounter = meter.createCounter("claude.tokens.total", {
  description: "Total tokens consumed by Claude API calls",
  unit: "tokens",
});
 
const inputTokenCounter = meter.createCounter("claude.tokens.input", {
  description: "Input tokens sent to Claude API",
  unit: "tokens",
});
 
const outputTokenCounter = meter.createCounter("claude.tokens.output", {
  description: "Output tokens received from Claude API",
  unit: "tokens",
});
 
const requestDuration = meter.createHistogram("claude.request.duration", {
  description: "Duration of Claude API requests",
  unit: "ms",
});
 
const requestCounter = meter.createCounter("claude.requests.total", {
  description: "Total number of Claude API requests",
});
 
const errorCounter = meter.createCounter("claude.errors.total", {
  description: "Total number of Claude API errors",
});
 
// Cost rates (as of March 2026)
const COST_PER_1K_INPUT: Record<string, number> = {
  "claude-opus-4-6": 0.015,
  "claude-sonnet-4-6": 0.003,
  "claude-haiku-4-5-20251001": 0.0008,
};
 
const COST_PER_1K_OUTPUT: Record<string, number> = {
  "claude-opus-4-6": 0.075,
  "claude-sonnet-4-6": 0.015,
  "claude-haiku-4-5-20251001": 0.004,
};
 
const costCounter = meter.createCounter("claude.cost.usd", {
  description: "Estimated cost of Claude API calls in USD",
  unit: "usd",
});
 
// Instrumented Claude API client
export class InstrumentedClaudeClient {
  private client: Anthropic;
 
  constructor(apiKey?: string) {
    this.client = new Anthropic({ apiKey });
  }
 
  async createMessage(
    params: Anthropic.MessageCreateParamsNonStreaming
  ): Promise<Anthropic.Message> {
    // Create a span to trace the API call
    return tracer.startActiveSpan(
      "claude.messages.create",
      {
        attributes: {
          "ai.model": params.model,
          "ai.max_tokens": params.max_tokens,
          "ai.message_count": params.messages.length,
          "ai.has_tools": params.tools ? "true" : "false",
          "ai.has_system": params.system ? "true" : "false",
        },
      },
      async (span) => {
        const startTime = Date.now();
 
        try {
          // Make the Claude API call
          const response = await this.client.messages.create(params);
          const duration = Date.now() - startTime;
 
          // Record response metadata on the span
          span.setAttributes({
            "ai.response.id": response.id,
            "ai.response.model": response.model,
            "ai.response.stop_reason": response.stop_reason || "unknown",
            "ai.tokens.input": response.usage.input_tokens,
            "ai.tokens.output": response.usage.output_tokens,
            "ai.tokens.total":
              response.usage.input_tokens + response.usage.output_tokens,
            "ai.duration_ms": duration,
          });
 
          // Record metrics
          const labels = { model: response.model };
          inputTokenCounter.add(response.usage.input_tokens, labels);
          outputTokenCounter.add(response.usage.output_tokens, labels);
          tokenCounter.add(
            response.usage.input_tokens + response.usage.output_tokens,
            labels
          );
          requestDuration.record(duration, labels);
          requestCounter.add(1, { ...labels, status: "success" });
 
          // Estimate cost
          const model = response.model;
          const inputCost =
            (response.usage.input_tokens / 1000) *
            (COST_PER_1K_INPUT[model] || 0.003);
          const outputCost =
            (response.usage.output_tokens / 1000) *
            (COST_PER_1K_OUTPUT[model] || 0.015);
          costCounter.add(inputCost + outputCost, labels);
 
          span.setStatus({ code: SpanStatusCode.OK });
          return response;
        } catch (error) {
          const duration = Date.now() - startTime;
 
          // Record error on span and metrics
          span.setStatus({
            code: SpanStatusCode.ERROR,
            message: error instanceof Error ? error.message : "Unknown error",
          });
          span.recordException(error as Error);
 
          errorCounter.add(1, {
            model: params.model,
            error_type:
              error instanceof Anthropic.APIError
                ? `${error.status}`
                : "unknown",
          });
          requestDuration.record(duration, { model: params.model });
          requestCounter.add(1, {
            model: params.model,
            status: "error",
          });
 
          throw error;
        } finally {
          span.end();
        }
      }
    );
  }
}

// src/telemetry/agent-tracing.ts
import { trace, SpanStatusCode, SpanKind } from "@opentelemetry/api";
import { InstrumentedClaudeClient } from "./claude-instrumentation";
import Anthropic from "@anthropic-ai/sdk";
 
const tracer = trace.getTracer("claude-agent", "1.0.0");
 
interface ToolResult {
  tool_use_id: string;
  content: string;
}
 
// Instrumented tool execution
async function executeToolWithTracing(
  toolName: string,
  toolInput: Record<string, unknown>,
  toolHandler: (input: Record<string, unknown>) => Promise<string>
): Promise<string> {
  return tracer.startActiveSpan(
    `tool.${toolName}`,
    {
      kind: SpanKind.INTERNAL,
      attributes: {
        "tool.name": toolName,
        "tool.input_keys": Object.keys(toolInput).join(","),
      },
    },
    async (span) => {
      try {
        const result = await toolHandler(toolInput);
        span.setAttributes({
          "tool.result_length": result.length,
          "tool.success": true,
        });
        span.setStatus({ code: SpanStatusCode.OK });
        return result;
      } catch (error) {
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: error instanceof Error ? error.message : "Tool execution failed",
        });
        span.recordException(error as Error);
        throw error;
      } finally {
        span.end();
      }
    }
  );
}
 
// Instrumented agent loop
export async function runAgentWithTracing(
  client: InstrumentedClaudeClient,
  systemPrompt: string,
  userMessage: string,
  tools: Anthropic.Tool[],
  toolHandlers: Record<
    string,
    (input: Record<string, unknown>) => Promise<string>
  >,
  maxIterations: number = 10
): Promise<string> {
  return tracer.startActiveSpan(
    "agent.workflow",
    {
      attributes: {
        "agent.max_iterations": maxIterations,
        "agent.tools_available": tools.map((t) => t.name).join(","),
        "agent.system_prompt_length": systemPrompt.length,
      },
    },
    async (rootSpan) => {
      const messages: Anthropic.MessageParam[] = [
        { role: "user", content: userMessage },
      ];
      let iteration = 0;
      let totalInputTokens = 0;
      let totalOutputTokens = 0;
 
      try {
        while (iteration < maxIterations) {
          iteration++;
 
          // Record each iteration as a child span
          const response = await tracer.startActiveSpan(
            `agent.iteration.${iteration}`,
            { attributes: { "agent.iteration": iteration } },
            async (iterSpan) => {
              const resp = await client.createMessage({
                model: "claude-sonnet-4-6",
                max_tokens: 4096,
                system: systemPrompt,
                tools,
                messages,
              });
 
              totalInputTokens += resp.usage.input_tokens;
              totalOutputTokens += resp.usage.output_tokens;
 
              iterSpan.setAttributes({
                "agent.stop_reason": resp.stop_reason || "unknown",
                "agent.content_blocks": resp.content.length,
              });
              iterSpan.end();
              return resp;
            }
          );
 
          // Exit condition: no tool calls means we're done
          if (response.stop_reason === "end_turn") {
            const textBlock = response.content.find(
              (b) => b.type === "text"
            );
            const finalAnswer =
              textBlock && "text" in textBlock ? textBlock.text : "";
 
            rootSpan.setAttributes({
              "agent.total_iterations": iteration,
              "agent.total_input_tokens": totalInputTokens,
              "agent.total_output_tokens": totalOutputTokens,
              "agent.final_answer_length": finalAnswer.length,
            });
            rootSpan.setStatus({ code: SpanStatusCode.OK });
            return finalAnswer;
          }
 
          // Process tool calls
          const toolUseBlocks = response.content.filter(
            (b) => b.type === "tool_use"
          );
 
          const toolResults: ToolResult[] = [];
          for (const block of toolUseBlocks) {
            if (block.type === "tool_use") {
              const handler = toolHandlers[block.name];
              if (handler) {
                const result = await executeToolWithTracing(
                  block.name,
                  block.input as Record<string, unknown>,
                  handler
                );
                toolResults.push({
                  tool_use_id: block.id,
                  content: result,
                });
              }
            }
          }
 
          // Add assistant response and tool results to conversation
          messages.push({ role: "assistant", content: response.content });
          messages.push({
            role: "user",
            content: toolResults.map((r) => ({
              type: "tool_result" as const,
              tool_use_id: r.tool_use_id,
              content: r.content,
            })),
          });
        }
 
        rootSpan.setAttributes({
          "agent.total_iterations": iteration,
          "agent.max_iterations_reached": true,
        });
        rootSpan.setStatus({ code: SpanStatusCode.OK });
        return "Maximum iterations reached";
      } catch (error) {
        rootSpan.setStatus({
          code: SpanStatusCode.ERROR,
          message: error instanceof Error ? error.message : "Agent workflow failed",
        });
        rootSpan.recordException(error as Error);
        throw error;
      } finally {
        rootSpan.end();
      }
    }
  );
}

// src/telemetry/cost-dashboard-metrics.ts
import { metrics } from "@opentelemetry/api";
 
const meter = metrics.getMeter("claude-cost-dashboard", "1.0.0");
 
// Daily cost tracking via ObservableGauge
const dailyCostGauge = meter.createObservableGauge(
  "claude.cost.daily_estimate_usd",
  {
    description: "Estimated daily cost based on current usage rate",
    unit: "usd",
  }
);
 
// Budget utilization percentage
const budgetUtilization = meter.createObservableGauge(
  "claude.budget.utilization_percent",
  {
    description: "Percentage of monthly budget consumed",
    unit: "percent",
  }
);
 
// Cost tracking state
interface CostTracker {
  dailyTotal: number;
  monthlyTotal: number;
  monthlyBudget: number;
  lastResetDate: string;
}
 
const costTracker: CostTracker = {
  dailyTotal: 0,
  monthlyTotal: 0,
  monthlyBudget: parseFloat(process.env.MONTHLY_BUDGET_USD || "500"),
  lastResetDate: new Date().toISOString().split("T")[0],
};
 
// Register observable callbacks
dailyCostGauge.addCallback((result) => {
  result.observe(costTracker.dailyTotal);
});
 
budgetUtilization.addCallback((result) => {
  const utilization =
    (costTracker.monthlyTotal / costTracker.monthlyBudget) * 100;
  result.observe(utilization);
});
 
// Record cost (called from InstrumentedClaudeClient)
export function recordCost(
  model: string,
  inputTokens: number,
  outputTokens: number
): void {
  const today = new Date().toISOString().split("T")[0];
  if (today !== costTracker.lastResetDate) {
    costTracker.dailyTotal = 0;
    costTracker.lastResetDate = today;
  }
 
  const costRates: Record<string, { input: number; output: number }> = {
    "claude-opus-4-6": { input: 0.015, output: 0.075 },
    "claude-sonnet-4-6": { input: 0.003, output: 0.015 },
    "claude-haiku-4-5-20251001": { input: 0.0008, output: 0.004 },
  };
 
  const rates = costRates[model] || { input: 0.003, output: 0.015 };
  const cost =
    (inputTokens / 1000) * rates.input +
    (outputTokens / 1000) * rates.output;
 
  costTracker.dailyTotal += cost;
  costTracker.monthlyTotal += cost;
 
  // Budget alert threshold
  const utilization =
    (costTracker.monthlyTotal / costTracker.monthlyBudget) * 100;
  if (utilization > 80) {
    console.warn(
      `⚠️ Budget alert: ${utilization.toFixed(1)}% of monthly budget consumed`
    );
  }
}

# Token consumption over the last hour (by model)
sum(rate(claude_tokens_total[1h])) by (model)
 
# Estimated hourly cost
sum(rate(claude_cost_usd[1h])) * 3600
 
# Error rate (5-minute window)
sum(rate(claude_errors_total[5m])) / sum(rate(claude_requests_total[5m])) * 100
 
# P95 latency
histogram_quantile(0.95, rate(claude_request_duration_bucket[5m]))
 
# Budget utilization trend
claude_budget_utilization_percent

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      http:
        endpoint: 0.0.0.0:4318
      grpc:
        endpoint: 0.0.0.0:4317
 
processors:
  # Batch for network efficiency
  batch:
    timeout: 10s
    send_batch_size: 1024
 
  # Enrich AI-related spans with metadata
  attributes:
    actions:
      - key: "service.layer"
        value: "ai-inference"
        action: upsert
 
  # Trace sampling (cost reduction in production)
  probabilistic_sampler:
    sampling_percentage: 25  # Keep 25% of traces
 
  # Always retain high-value traces
  tail_sampling:
    decision_wait: 10s
    policies:
      # Keep 100% of errors
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      # Keep high-latency requests (>3s)
      - name: slow-requests
        type: latency
        latency:
          threshold_ms: 3000
      # Keep all Opus model calls (expensive)
      - name: expensive-models
        type: string_attribute
        string_attribute:
          key: ai.model
          values: ["claude-opus-4-6"]
 
exporters:
  otlphttp/tempo:
    endpoint: http://tempo:4318
  prometheus:
    endpoint: 0.0.0.0:8889
    namespace: claude
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, attributes, tail_sampling]
      exporters: [otlphttp/tempo]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

// src/telemetry/cache-metrics.ts
import { metrics } from "@opentelemetry/api";
import Anthropic from "@anthropic-ai/sdk";
 
const meter = metrics.getMeter("claude-cache", "1.0.0");
 
const cacheHitCounter = meter.createCounter("claude.cache.hits", {
  description: "Number of prompt cache hits",
});
 
const cacheMissCounter = meter.createCounter("claude.cache.misses", {
  description: "Number of prompt cache misses",
});
 
const cacheSavingsCounter = meter.createCounter(
  "claude.cache.savings_tokens",
  {
    description: "Tokens saved by prompt caching",
    unit: "tokens",
  }
);
 
// Monitor cache effectiveness
export function recordCacheMetrics(
  response: Anthropic.Message,
  model: string
): void {
  const usage = response.usage as Record<string, number>;
 
  // Get cache_creation_input_tokens and cache_read_input_tokens
  const cacheCreation = usage.cache_creation_input_tokens || 0;
  const cacheRead = usage.cache_read_input_tokens || 0;
 
  if (cacheRead > 0) {
    cacheHitCounter.add(1, { model });
    // Cache reads are charged at 10% of normal rate, saving 90%
    cacheSavingsCounter.add(Math.floor(cacheRead * 0.9), { model });
  } else if (cacheCreation > 0) {
    cacheMissCounter.add(1, { model });
  }
}

# Prompt caching hit rate
sum(rate(claude_cache_hits[1h])) /
(sum(rate(claude_cache_hits[1h])) + sum(rate(claude_cache_misses[1h]))) * 100
 
# Estimated tokens saved by caching (per day)
sum(increase(claude_cache_savings_tokens[24h]))

# docker-compose.observability.yml
version: "3.9"
services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.96.0
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4317:4317"   # gRPC
      - "4318:4318"   # HTTP
      - "8889:8889"   # Prometheus metrics
 
  tempo:
    image: grafana/tempo:2.4.0
    command: ["-config.file=/etc/tempo.yaml"]
    volumes:
      - ./tempo.yaml:/etc/tempo.yaml
    ports:
      - "3200:3200"
 
  prometheus:
    image: prom/prometheus:v2.50.0
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"
 
  grafana:
    image: grafana/grafana:10.3.0
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
    ports:
      - "3001:3000"
    depends_on:
      - tempo
      - prometheus