{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/turn-a-verified-ai-failure-into-a-regression-test",
  "id": "MHC-D-RESEARCH-0718",
  "version": "0.1.0",
  "title": "Turn a verified AI failure into a regression test",
  "summary": "A painful failure should pay rent the second time.",
  "kind": "protocol",
  "body": "After fixing the immediate problem, preserve a minimal representative task that reproduces the failure and a grader that detects it. Add it to the relevant eval suite before changing prompts, skills or models again. Tag the failure class so later regressions can be diagnosed rather than hidden inside one aggregate score.",
  "limits": [
    "Known-failure tests do not cover novel failures; keep adding fresh tasks and production monitoring."
  ],
  "topics": [
    "union-ai-eval-learning-workflow"
  ],
  "intents": [],
  "source_ids": [
    "RS-ED3CA2F877BE88CA"
  ],
  "evidence": [
    {
      "claim": "Anthropic recommends sourcing eval tasks from real failures and keeping regression suites so fixes can be tested against previously observed problems.",
      "source_id": "RS-ED3CA2F877BE88CA",
      "role": "supports",
      "note": "A regression suite covers known failures and must be supplemented with new, harder and distribution-shifted tasks.",
      "locator": "Collect tasks and maintain eval suites"
    }
  ],
  "use_when": [
    "You discover an AI output or agent trajectory that failed in a way you care about."
  ],
  "avoid_when": [
    "Known-failure tests do not cover novel failures; keep adding fresh tasks and production monitoring."
  ],
  "example": "If an agent silently edits an unrelated configuration file, add a task whose grader verifies both requested behavior and untouched-file constraints.",
  "check": "The old failure would be caught automatically if it returned tomorrow.",
  "steps": [
    "The old failure would be caught automatically if it returned tomorrow."
  ],
  "sources": [
    {
      "id": "RS-ED3CA2F877BE88CA",
      "title": "Demystifying evals for AI agents",
      "url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0718",
      "to": "MHC-D-RESEARCH-0319",
      "type": "useful_with",
      "url": "/knowledge/keep-an-eval-set-that-can-embarrass-the-agent"
    }
  ],
  "collections": [
    {
      "id": "RC-ABC17A288CB87C85",
      "title": "Make AI work improve from failures instead of accumulating rituals",
      "url": "/collections/make-ai-work-improve-from-failures-instead-of-accumulating-rituals"
    }
  ]
}