{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/do-not-assume-a-rubric-reduced-noise-measure-it",
  "id": "MHC-D-RESEARCH-0417",
  "version": "0.1.0",
  "title": "Do not assume a rubric reduced noise—measure it",
  "summary": "A rubric can look structured while reviewers still interpret every line differently.",
  "kind": "principle",
  "body": "Compare reliability before and after the structured criteria on repeated or shared cases. If agreement does not improve, inspect criterion definitions, training examples and how criteria are integrated into the final judgment. Keep the null result: structure that does not change reliability may still aid explanation, but it has not earned a noise-reduction claim.",
  "limits": [
    "The IFCN null result is from a clinical expert task; it motivates measurement, not a claim that rubrics generally fail."
  ],
  "topics": [
    "union-decision-hygiene-noise"
  ],
  "intents": [],
  "source_ids": [
    "RS-4DC9763DF7D5A389"
  ],
  "evidence": [
    {
      "claim": "The 2025 IFCN study found that explicitly adding six expert criteria did not materially improve inter-rater reliability, performance or overall calibration in the studied expert judgments.",
      "source_id": "RS-4DC9763DF7D5A389",
      "role": "limits",
      "note": "This null result applies to one clinical judgment task and does not imply that structured criteria are generally useless.",
      "locator": "Abstract results"
    }
  ],
  "use_when": [
    "A team introduces a checklist or criteria set and declares the judgment process standardized."
  ],
  "avoid_when": [
    "The IFCN null result is from a clinical expert task; it motivates measurement, not a claim that rubrics generally fail."
  ],
  "example": "After adding a six-item architecture-risk checklist, compare whether reviewers actually converge more on the same proposals.",
  "check": "The team can distinguish 'we added structure' from 'the structure measurably improved consistency.'",
  "sources": [
    {
      "id": "RS-4DC9763DF7D5A389",
      "title": "Utility of the IFCN criteria for identifying interictal epileptiform discharges by experts: A decision hygiene approach to improve inter-rater reliability",
      "url": "https://www.sciencedirect.com/science/article/pii/S1388245725003268"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-E41621E574ADAE4A",
      "title": "Reduce judgment noise without erasing useful disagreement",
      "url": "/collections/reduce-judgment-noise-without-erasing-useful-disagreement"
    }
  ]
}