{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/mine-traces-for-failure-categories-before-changing-prompts",
  "id": "MHC-D-RESEARCH-1166",
  "version": "0.1.0",
  "title": "Mine traces for failure categories before changing prompts",
  "summary": "A low score tells you that something is wrong. A failure taxonomy tells you where to work.",
  "kind": "protocol",
  "body": "Sample failed and borderline traces, ask concrete diagnostic questions and group recurring causes: retrieval miss, bad tool choice, state loss, weak planning, invalid output, policy failure or model limitation. Change the component that matches the failure instead of accumulating prompt rules for every symptom.",
  "limits": [
    "Trace categories are hypotheses about mechanism. Confirm them with targeted tests before treating them as causes."
  ],
  "topics": [
    "union-ai-runtime-verification-2026"
  ],
  "intents": [],
  "source_ids": [
    "RS-AE31C3BE0BBBA1F6"
  ],
  "evidence": [
    {
      "claim": "Production traces can be mined into recurring failure categories and reusable evidence before teams decide whether prompts, tools, state or the model should change.",
      "source_id": "RS-AE31C3BE0BBBA1F6",
      "role": "supports",
      "note": "Failure categories are observational; they help focus investigation but do not prove which intervention caused an improvement.",
      "locator": "0:00-17:06, execution evidence, trace mining, failure diagnosis and continual learning"
    }
  ],
  "use_when": [
    "An agent is underperforming and the team is tempted to add more instructions without knowing what actually fails."
  ],
  "avoid_when": [
    "Trace categories are hypotheses about mechanism. Confirm them with targeted tests before treating them as causes."
  ],
  "example": "If most failed coding runs selected the wrong file, improve repository retrieval before adding another paragraph telling the model to 'be careful.'",
  "check": "Every proposed intervention names the failure category and trace evidence it is supposed to change.",
  "steps": [
    "Sample failures and a few successful near-neighbours.",
    "Label the first material point where the run diverges.",
    "Group recurring failures by mechanism, not by wording.",
    "Choose an intervention for the largest or most costly supported category.",
    "Rerun the affected eval slice after the change."
  ],
  "sources": [
    {
      "id": "RS-AE31C3BE0BBBA1F6",
      "title": "Improving Agents Is a Data Mining Problem",
      "url": "https://ai.engineer/talks/CvRngaQZQ3Y-improving-agents-is-data-mining-problem"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-1166",
      "to": "MHC-D-RESEARCH-0718",
      "type": "useful_with",
      "url": "/knowledge/turn-a-verified-ai-failure-into-a-regression-test"
    }
  ],
  "collections": [
    {
      "id": "RC-ACD6385FB70207D3",
      "title": "Operate AI agents as systems you can replay, verify and constrain",
      "url": "/collections/operate-ai-agents-as-systems-you-can-replay-verify-and-constrain"
    }
  ]
}