{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/record-confidence-before-feedback-then-score-it-against-the-outcome",
  "id": "MHC-D-RESEARCH-1248",
  "version": "0.1.0",
  "title": "Record confidence before feedback, then score it against the outcome",
  "summary": "Calibration needs two columns: what you believed before the answer and what happened after.",
  "kind": "protocol",
  "body": "For repeated checkable judgments, capture the answer and confidence before seeing the outcome. When the result arrives, score both correctness and confidence. Review the pairs in batches: look for ranges where you are systematically too sure or too hesitant. Forecasting experiments suggest individualized outcome feedback can reduce overconfidence, but the effect is task-dependent, so calibrate on the class of judgments you actually make.",
  "limits": [
    "Calibration feedback has been studied in forecasting tasks and does not automatically transfer to every domain. A confidence percentage is not a scientific probability unless the task and scoring support that interpretation."
  ],
  "topics": [
    "union-metacognitive-calibration-and-task-quality"
  ],
  "intents": [],
  "source_ids": [
    "RS-BBE214612662000E"
  ],
  "evidence": [
    {
      "claim": "In two forecasting studies, individualized calibration feedback reduced confidence among initially overconfident forecasters; overall calibration improved in the more controlled second experiment.",
      "source_id": "RS-BBE214612662000E",
      "role": "supports",
      "note": "The result is task-dependent and comes from forecasting experiments, not a general test of metacognitive training across all work.",
      "locator": "Abstract"
    }
  ],
  "use_when": [
    "You make repeated forecasts, estimates, diagnoses or answerable judgments and want to improve how much trust you place in your own confidence."
  ],
  "avoid_when": [
    "Calibration feedback has been studied in forecasting tasks and does not automatically transfer to every domain. A confidence percentage is not a scientific probability unless the task and scoring support that interpretation."
  ],
  "example": "Before checking a production defect, write your leading cause and confidence. After the trace or fix confirms the cause, add the outcome. Ten such pairs teach more than remembering only the dramatic wins.",
  "check": "You can show a set of pre-outcome confidence–result pairs and name at least one recurring calibration pattern without rewriting old predictions after the fact.",
  "steps": [
    "Write the judgment before feedback or outcome information arrives.",
    "Add a confidence estimate using the same scale each time.",
    "Record the verified outcome separately.",
    "Review several pairs together and adjust future confidence where a pattern is visible."
  ],
  "sources": [
    {
      "id": "RS-BBE214612662000E",
      "title": "Automated calibration training for forecasters",
      "url": "https://doi.org/10.1002/bdm.2334"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-3EBB3946EFB898BD",
      "title": "Calibrate what you know before confidence becomes the plan",
      "url": "/collections/calibrate-what-you-know-before-confidence-becomes-the-plan"
    }
  ]
}