{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/keep-a-list-of-high-confidence-ai-failures",
  "id": "MHC-D-RESEARCH-0495",
  "version": "0.1.0",
  "title": "Keep a list of high-confidence AI failures",
  "summary": "The errors worth memorizing are the ones the model did not know were errors.",
  "kind": "protocol",
  "body": "Maintain a compact failure set of cases where the AI was wrong while expressing high confidence or presenting unusually persuasive evidence. Tag the task shape and likely failure mode. Use these cases in future evals, user warnings or routing rules.",
  "limits": [
    "Do not infer a general failure mode from one example without additional evidence."
  ],
  "topics": [
    "union-ai-reliance-metacognition"
  ],
  "intents": [],
  "source_ids": [
    "RS-A420BE76E329A838"
  ],
  "evidence": [
    {
      "claim": "An AAAI 2026 experiment found well-calibrated AI confidence improved participant decision accuracy more than miscalibrated confidence, while miscalibrated cues increased reliance-related errors.",
      "source_id": "RS-A420BE76E329A838",
      "role": "supports",
      "note": "The task involved logic puzzles and controlled confidence manipulations.",
      "locator": "Results"
    }
  ],
  "use_when": [
    "You are calibrating an AI assistant and ordinary error counts hide the most dangerous mistakes."
  ],
  "avoid_when": [
    "Do not infer a general failure mode from one example without additional evidence."
  ],
  "example": "A coding agent confidently edits the wrong configuration scope because two environments use near-identical names; keep that case in the eval set.",
  "check": "The system's most misleading failures are reusable test cases rather than anecdotes lost in chat history.",
  "steps": [
    "The system's most misleading failures are reusable test cases rather than anecdotes lost in chat history."
  ],
  "sources": [
    {
      "id": "RS-A420BE76E329A838",
      "title": "Too Sure for Our Own Good: A User Study on AI Confidence and Human Reliance",
      "url": "https://ojs.aaai.org/index.php/AAAI/article/view/38798"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-39B535F35706B375",
      "title": "Calibrate when to rely on AI instead of measuring trust as a feeling",
      "url": "/collections/calibrate-when-to-rely-on-ai-instead-of-measuring-trust-as-a-feeling"
    }
  ]
}