Contract-Test Every Tool Before You Submit or Automate an MCP Connector

import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 
// Launch the connector under test as a child process and connect to it
export async function connect(): Promise<Client> {
  const transport = new StdioClientTransport({
    command: "node",
    args: ["dist/server.js"],
  });
  const client = new Client(
    { name: "contract-tester", version: "1.0.0" },
    { capabilities: {} },
  );
  await client.connect(transport);
  return client;
}
 
type Issue = { tool: string; rule: string; detail: string };
 
export async function checkSelfDescription(client: Client): Promise<Issue[]> {
  const { tools } = await client.listTools();
  const issues: Issue[] = [];
 
  for (const t of tools) {
    // A missing or too-short description means the model cannot judge intent
    if (!t.description || t.description.trim().length < 12) {
      issues.push({ tool: t.name, rule: "description", detail: "description missing or too short" });
    }
    // inputSchema must be an object type with properties at minimum
    const schema = t.inputSchema as { type?: string; properties?: object } | undefined;
    if (!schema || schema.type !== "object") {
      issues.push({ tool: t.name, rule: "schema", detail: "inputSchema is not an object" });
    } else if (!schema.properties) {
      issues.push({ tool: t.name, rule: "schema", detail: "properties is undefined" });
    }
  }
  return issues;
}

export async function checkReadContract(
  client: Client,
  tool: string,
  args: Record<string, unknown>,
): Promise<Issue[]> {
  const issues: Issue[] = [];
  const res = await client.callTool({ name: tool, arguments: args });
 
  // If a happy-path call returns isError, the args or the server violate the contract
  if (res.isError) {
    issues.push({ tool, rule: "read-error", detail: "isError returned for happy-path args" });
    return issues;
  }
  // An MCP tool response is expected to carry a content array
  if (!Array.isArray(res.content) || res.content.length === 0) {
    issues.push({ tool, rule: "read-shape", detail: "content is empty or not an array" });
  }
  // Tools that promise structured output should pin the presence of structuredContent
  if (res.structuredContent !== undefined && typeof res.structuredContent !== "object") {
    issues.push({ tool, rule: "read-structured", detail: "structuredContent is not an object" });
  }
  return issues;
}

import { randomUUID } from "node:crypto";
 
export async function checkIdempotency(
  client: Client,
  writeTool: string,
  countTool: string,
): Promise<Issue[]> {
  const issues: Issue[] = [];
  const key = `contract-${randomUUID()}`; // unique sentinel
 
  const before = await readCount(client, countTool);
 
  // Write twice with the same idempotency key
  const first = await client.callTool({ name: writeTool, arguments: { idempotencyKey: key, payload: { sentinel: key } } });
  const second = await client.callTool({ name: writeTool, arguments: { idempotencyKey: key, payload: { sentinel: key } } });
 
  const after = await readCount(client, countTool);
 
  if (first.isError) {
    issues.push({ tool: writeTool, rule: "write-first", detail: "first write failed" });
  }
  // The second call returning a success (including "already applied") is the idempotent design
  if (second.isError) {
    issues.push({ tool: writeTool, rule: "idempotent-retry", detail: "second call returned isError" });
  }
  // If the count grew by two, it double-fired
  if (after - before > 1) {
    issues.push({ tool: writeTool, rule: "idempotent-effect", detail: `count grew by ${after - before} (expected 1)` });
  }
  return issues;
}
 
async function readCount(client: Client, countTool: string): Promise<number> {
  const res = await client.callTool({ name: countTool, arguments: {} });
  const text = (res.content?.[0] as { text?: string })?.text ?? "0";
  return Number.parseInt(text, 10) || 0;
}

export async function measureLatency(
  client: Client,
  tool: string,
  args: Record<string, unknown>,
  runs = 20,
): Promise<{ p50: number; p95: number }> {
  const samples: number[] = [];
  for (let i = 0; i < runs; i++) {
    const start = performance.now();
    await client.callTool({ name: tool, arguments: args });
    samples.push(performance.now() - start);
  }
  samples.sort((a, b) => a - b);
  const at = (q: number) => Math.round(samples[Math.floor(q * (samples.length - 1))]);
  return { p50: at(0.5), p95: at(0.95) };
}

async function main() {
  const client = await connect();
  const issues: Issue[] = [];
 
  issues.push(...(await checkSelfDescription(client)));
  issues.push(...(await checkReadContract(client, "list_articles", { limit: 1 })));
  issues.push(...(await checkIdempotency(client, "upsert_article", "count_articles")));
 
  const lat = await measureLatency(client, "list_articles", { limit: 1 });
  console.log(`list_articles p50=${lat.p50}ms p95=${lat.p95}ms`);
 
  await client.close();
 
  if (issues.length > 0) {
    for (const i of issues) console.error(`❌ [${i.tool}] ${i.rule}: ${i.detail}`);
    process.exit(1); // stop the submission or deploy right here
  }
  console.log("✅ contract passed");
}
 
main().catch((e) => { console.error(e); process.exit(1); });