{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/do-not-expose-ai-confidence-until-you-have-tested-its-calibration",
  "id": "MHC-D-RESEARCH-0494",
  "version": "0.1.0",
  "title": "Do not expose AI confidence until you have tested its calibration",
  "summary": "A number beside an answer can create trust even when the number is wrong.",
  "kind": "protocol",
  "body": "Before displaying confidence, compare reported probabilities with observed accuracy and inspect high-confidence mistakes. Test whether the display improves appropriate reliance rather than merely increasing agreement. If calibration is unstable, prefer bounded uncertainty language, failure-mode information or no confidence score until the signal is better validated.",
  "limits": [
    "Even calibrated confidence cannot replace source evidence or authorization for high-stakes decisions."
  ],
  "topics": [
    "union-ai-reliance-metacognition"
  ],
  "intents": [],
  "source_ids": [
    "RS-A420BE76E329A838",
    "RS-E5AA539FACACCC98"
  ],
  "evidence": [
    {
      "claim": "An AAAI 2026 experiment found well-calibrated AI confidence improved participant decision accuracy more than miscalibrated confidence, while miscalibrated cues increased reliance-related errors.",
      "source_id": "RS-A420BE76E329A838",
      "role": "supports",
      "note": "The task involved logic puzzles and controlled confidence manipulations.",
      "locator": "Results"
    },
    {
      "claim": "A 2026 experiment found visual uncertainty cues could increase users' subjective confidence-accuracy discrimination while simultaneously increasing behavioral overreliance on incorrect LLM outputs.",
      "source_id": "RS-E5AA539FACACCC98",
      "role": "supports",
      "note": "Interface effects depend on cue design and task; more uncertainty display is not automatically harmful.",
      "locator": "Abstract"
    }
  ],
  "use_when": [
    "A product plans to show users a confidence percentage because it seems transparent."
  ],
  "avoid_when": [
    "Even calibrated confidence cannot replace source evidence or authorization for high-stakes decisions."
  ],
  "example": "Do not show '97% confident' on a compliance recommendation until that confidence has been evaluated on comparable compliance cases.",
  "check": "The confidence display has evidence of both statistical quality and useful behavioral effect.",
  "steps": [
    "Confidence is evaluated against labeled outcomes.",
    "Miscalibration is measured by task segment, not only overall.",
    "High-confidence errors receive explicit review.",
    "User behavior with and without the display is tested where stakes justify it.",
    "The display is versioned with the model/configuration it was validated on."
  ],
  "sources": [
    {
      "id": "RS-A420BE76E329A838",
      "title": "Too Sure for Our Own Good: A User Study on AI Confidence and Human Reliance",
      "url": "https://ojs.aaai.org/index.php/AAAI/article/view/38798"
    },
    {
      "id": "RS-E5AA539FACACCC98",
      "title": "More is not better: Visual uncertainty cues and the fragility of trust calibration in LLM-assisted decision making",
      "url": "https://doi.org/10.1016/j.chbah.2026.100307"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0494",
      "to": "MHC-D-RESEARCH-0502",
      "type": "useful_with",
      "url": "/knowledge/pair-confidence-with-known-failure-modes"
    }
  ],
  "collections": [
    {
      "id": "RC-39B535F35706B375",
      "title": "Calibrate when to rely on AI instead of measuring trust as a feeling",
      "url": "/collections/calibrate-when-to-rely-on-ai-instead-of-measuring-trust-as-a-feeling"
    }
  ]
}