The MCP Server Connects Fine and Still Hands Back Zero Tools — Catching Silent Capability Drift with a Tool Manifest Diff

{
  "mcpServers": {
    "report": {
      "command": "python3",
      "args": ["./report_server.py"],
      "env": { "REPORT_API_TOKEN": "${REPORT_API_TOKEN}" }
    }
  }
}

#!/usr/bin/env python3
"""Fail closed when the effective MCP tool surface differs from the committed lock file."""
import json, os, subprocess, sys, time
 
CONFIG = os.environ.get("MCP_CONFIG", ".mcp.json")
LOCK = os.environ.get("MCP_TOOL_LOCK", "mcp-tools.lock.json")
TIMEOUT_S = float(os.environ.get("MCP_PROBE_TIMEOUT_S", "10"))
 
 
def expand(value: str) -> str:
    # ${VAR} in the config resolves from the *launch* environment.
    if value.startswith("${") and value.endswith("}"):
        return os.environ.get(value[2:-1], "")
    return value
 
 
def probe(name, spec):
    env = {k: v for k, v in os.environ.items()}
    for k, v in (spec.get("env") or {}).items():
        env[k] = expand(v)
    try:
        # Popen must live INSIDE the try: a mistyped command raises FileNotFoundError here,
        # and letting it escape turns a config error into a preflight crash.
        proc = subprocess.Popen([spec["command"], *spec.get("args", [])],
                                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                stderr=subprocess.DEVNULL, text=True, env=env, bufsize=1)
 
        def call(method, params=None, rid=None):
            msg = {"jsonrpc": "2.0", "method": method}
            if params is not None:
                msg["params"] = params
            if rid is not None:
                msg["id"] = rid
            proc.stdin.write(json.dumps(msg) + "\n"); proc.stdin.flush()
            return json.loads(proc.stdout.readline()) if rid is not None else None
 
        call("initialize", {"protocolVersion": "2026-07-28", "capabilities": {},
                            "clientInfo": {"name": "mcp-preflight", "version": "1.0"}}, rid=1)
        call("notifications/initialized")
        listed = call("tools/list", {}, rid=2)
        call("shutdown", None, rid=3)
        proc.wait(timeout=TIMEOUT_S)
        return sorted(t["name"] for t in listed.get("result", {}).get("tools", []))
    except Exception as exc:          # spawn failure, malformed frame, timeout
        if "proc" in dir():
            proc.kill()
        return {"error": f"{type(exc).__name__}: {exc}"}
 
 
def main():
    started = time.perf_counter()
    servers = json.load(open(CONFIG))["mcpServers"]
    observed = {name: probe(name, spec) for name, spec in servers.items()}
 
    if "--update-lock" in sys.argv:
        json.dump(observed, open(LOCK, "w"), indent=2, sort_keys=True)
        print(f"wrote {LOCK}")
        return 0
 
    expected = json.load(open(LOCK))
    failures = []
    for name, want in expected.items():
        got = observed.get(name)
        if isinstance(got, dict):
            failures.append(f"{name}: probe failed ({got['error']})")
            continue
        missing = [t for t in want if t not in (got or [])]
        if missing:
            failures.append(f"{name}: {len(missing)}/{len(want)} tools missing -> {missing}")
 
    took = (time.perf_counter() - started) * 1000
    if failures:
        print(f"MCP preflight FAILED in {took:.0f} ms")
        for f in failures:
            print(f"  - {f}")
        return 78                     # EX_CONFIG: wiring problem, not a runtime fault
    print(f"MCP preflight OK in {took:.0f} ms ({sum(len(v) for v in expected.values())} tools verified)")
    return 0
 
 
if __name__ == "__main__":
    sys.exit(main())

$ REPORT_API_TOKEN=test-token python3 preflight.py --update-lock
wrote mcp-tools.lock.json
 
$ cat mcp-tools.lock.json
{
  "report": [
    "fetch_daily_report",
    "list_report_dates"
  ]
}
 
$ REPORT_API_TOKEN=test-token python3 preflight.py; echo "exit=$?"
MCP preflight OK in 23 ms (2 tools verified)
exit=0
 
$ env -u REPORT_API_TOKEN python3 preflight.py; echo "exit=$?"
MCP preflight FAILED in 23 ms
  - report: 2/2 tools missing -> ['fetch_daily_report', 'list_report_dates']
exit=78

FileNotFoundError: [Errno 2] No such file or directory: './no-such-binary'
exit=1

MCP preflight FAILED in 1 ms
  - ghost: probe failed (FileNotFoundError: [Errno 2] No such file or directory: './no-such-binary')
exit=78

- name: MCP preflight
  env:
    REPORT_API_TOKEN: ${{ secrets.REPORT_API_TOKEN }}
  run: python3 tools/preflight.py
 
- name: Run unattended agent
  run: claude --print --output-format json "$(cat prompts/nightly.md)"