When Your Claude × Playwright Browser Agent Fails While Reporting Success — Verifying Actions and Catching UI Drift

// The model proposes; a verifier decides. Keep the roles distinct.
interface ActionProposal {
  type: 'click' | 'fill' | 'navigate';
  selector?: string;
  value?: string;
  url?: string;
  // The post-condition the model declares this action should produce
  expectedEffect: string;
}
 
interface VerifiedStep {
  proposal: ActionProposal;
  reasoning: string;
  verified: boolean;       // the verifier's verdict, not the model's self-report
  evidence: string;        // the observed fact behind the verdict
}

// src/agent/verify.ts
import { Page } from 'playwright';
 
interface VerificationResult {
  verified: boolean;
  evidence: string;
}
 
// Verify against a goal-specific, observable fact
export async function verifyCartAdded(
  page: Page,
  expectedCount: number
): Promise<VerificationResult> {
  // 1. Target a stateful element, not UI text
  const badge = page.locator('[data-testid="cart-count"]');
  // 2. Wait for it. Absorb the screenshot-timing gap right here.
  try {
    await badge.waitFor({ state: 'visible', timeout: 5000 });
  } catch {
    return { verified: false, evidence: 'cart-count badge never appeared' };
  }
  const text = (await badge.textContent())?.trim() ?? '';
  const actual = parseInt(text, 10);
  // 3. Verdict comes from matching the expected state
  if (Number.isNaN(actual)) {
    return { verified: false, evidence: `cart-count is not numeric: "${text}"` };
  }
  return actual >= expectedCount
    ? { verified: true, evidence: `cart-count=${actual} (expected >= ${expectedCount})` }
    : { verified: false, evidence: `cart-count=${actual} (below expected ${expectedCount})` };
}

// src/agent/step.ts
import Anthropic from '@anthropic-ai/sdk';
import { Page } from 'playwright';
import { VerificationResult } from './verify';
 
type Verifier = (page: Page) => Promise<VerificationResult>;
 
export async function runVerifiedStep(
  client: Anthropic,
  page: Page,
  proposal: ActionProposal,
  verify: Verifier,
  history: Anthropic.MessageParam[]
): Promise<VerifiedStep> {
  // 1. Execute the proposed action
  await applyAction(page, proposal);
 
  // 2. Goal-specific verification (not the model's self-report)
  const result = await verify(page);
 
  // 3. On failure, hand the observed fact back to the model
  if (!result.verified) {
    history.push({
      role: 'user',
      content:
        `The previous action (${proposal.type} ${proposal.selector ?? ''}) did not ` +
        `produce the expected effect "${proposal.expectedEffect}". ` +
        `Observed fact: ${result.evidence}. Propose a different approach.`,
    });
  }
 
  return {
    proposal,
    reasoning: proposal.expectedEffect,
    verified: result.verified,
    evidence: result.evidence,
  };
}
 
async function applyAction(page: Page, a: ActionProposal): Promise<void> {
  if (a.type === 'click' && a.selector) await page.click(a.selector, { timeout: 8000 });
  else if (a.type === 'fill' && a.selector) await page.fill(a.selector, a.value ?? '');
  else if (a.type === 'navigate' && a.url) await page.goto(a.url, { waitUntil: 'networkidle' });
}

const tools: Anthropic.Tool[] = [
  {
    name: 'execute_action',
    description: 'Run a browser action. Always write a state-measurable post-condition in expectedEffect.',
    input_schema: {
      type: 'object',
      properties: {
        type: { type: 'string', enum: ['click', 'fill', 'navigate'] },
        selector: { type: 'string' },
        value: { type: 'string' },
        expectedEffect: {
          type: 'string',
          description: 'e.g. "cart-count increases by 1" (write it as state, not appearance)',
        },
      },
      required: ['type', 'expectedEffect'],
    },
  },
  {
    name: 'observe_state',
    description: 'Observation used to confirm a post-condition (element count, text, existence, current URL)',
    input_schema: {
      type: 'object',
      properties: {
        selector: { type: 'string' },
        property: { type: 'string', enum: ['count', 'text', 'exists', 'url'] },
      },
      required: ['property'],
    },
  },
];
 
const response = await client.messages.create({
  model: 'claude-sonnet-4-6',
  max_tokens: 1536,
  tools,
  messages: history,
});

const system: Anthropic.TextBlockParam[] = [
  {
    type: 'text',
    text:
      'You are an agent that operates a browser. ' +
      'Attach a state-measurable expectedEffect to every action, ' +
      'and when verification fails, try a different path rather than repeating the same action. ' +
      'Do not assert goal completion by self-report; defer to observed facts.',
    cache_control: { type: 'ephemeral' }, // current SDK takes this directly on messages.create
  },
];
 
const response = await client.messages.create({
  model: 'claude-sonnet-4-6',
  max_tokens: 1536,
  system,
  messages: recentHistory,
});

// src/monitor/drift.ts
import { chromium } from 'playwright';
 
interface Anchor { url: string; selector: string; label: string; }
 
// Register the observation points you verify against as anchors
const ANCHORS: Anchor[] = [
  { url: 'https://example.com/product/1', selector: '[data-testid="add-to-cart"]', label: 'Add button' },
  { url: 'https://example.com/cart', selector: '[data-testid="cart-count"]', label: 'Cart count badge' },
];
 
export async function checkDrift(): Promise<{ ok: boolean; missing: string[] }> {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  const missing: string[] = [];
  for (const a of ANCHORS) {
    await page.goto(a.url, { waitUntil: 'domcontentloaded' });
    const exists = (await page.locator(a.selector).count()) > 0;
    if (!exists) missing.push(`${a.label} (${a.selector})`);
  }
  await browser.close();
  return { ok: missing.length === 0, missing };
}

// src/agent/breaker.ts
export class VerificationBreaker {
  private window: boolean[] = [];
  constructor(private size = 50, private maxFailRate = 0.2) {}
 
  record(verified: boolean): void {
    this.window.push(verified);
    if (this.window.length > this.size) this.window.shift();
  }
 
  shouldHalt(): boolean {
    if (this.window.length < this.size) return false;
    const fails = this.window.filter((v) => !v).length;
    return fails / this.window.length > this.maxFailRate;
  }
}