{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/score-your-probability-forecasts-instead-of-remembering-the-wins",
  "id": "MHC-D-RESEARCH-0405",
  "version": "0.1.0",
  "title": "Score your probability forecasts instead of remembering the wins",
  "summary": "A forecaster who remembers only being right has invented a very generous scoring rule.",
  "kind": "protocol",
  "body": "Record each binary forecast as a probability before the event resolves, then calculate a proper score such as the Brier score after resolution. Review a batch of forecasts rather than one dramatic miss. Keep the event definitions and timestamps so hindsight cannot edit the prediction.",
  "limits": [
    "A proper score is not enough to diagnose why forecasts are weak; use calibration and case review as additional diagnostics."
  ],
  "topics": [
    "union-decision-analysis-calibration"
  ],
  "intents": [],
  "source_ids": [
    "RS-58E31CFFFB569015"
  ],
  "evidence": [
    {
      "claim": "The Brier score is a strictly proper scoring rule for binary probabilistic forecasts based on squared distance between the forecast probability and the observed outcome.",
      "source_id": "RS-58E31CFFFB569015",
      "role": "supports",
      "note": "One score summarizes performance and should be supplemented with diagnostics such as calibration when enough forecasts accumulate.",
      "locator": "Brier score example"
    }
  ],
  "use_when": [
    "You make recurring probabilistic judgments and want to improve calibration rather than collect anecdotes."
  ],
  "avoid_when": [
    "A proper score is not enough to diagnose why forecasts are weak; use calibration and case review as additional diagnostics."
  ],
  "example": "Record 60%, 80% and 30% probabilities for project milestones, then score them after outcomes instead of labeling each prediction simply right or wrong.",
  "check": "Forecast performance is inspectable from the recorded probabilities and outcomes rather than memory.",
  "steps": [
    "The probability is recorded before resolution.",
    "The event has a fixed binary resolution rule.",
    "The same scoring rule is used across comparable forecasts.",
    "Resolved and unresolved forecasts remain distinguishable.",
    "Review focuses on a batch, not one lucky or unlucky outcome."
  ],
  "sources": [
    {
      "id": "RS-58E31CFFFB569015",
      "title": "Proper Scoring Rules for Estimation and Forecast Evaluation",
      "url": "https://www.annualreviews.org/content/journals/10.1146/annurev-statistics-042424-050626"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0405",
      "to": "MHC-D-RESEARCH-0406",
      "type": "useful_with",
      "url": "/knowledge/separate-calibration-from-sharpness"
    }
  ],
  "collections": [
    {
      "id": "RC-3EA4A5EF5790DD81",
      "title": "Model the decision before buying more certainty",
      "url": "/collections/model-the-decision-before-buying-more-certainty"
    }
  ]
}