{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/measure-agreement-on-the-criteria-not-only-the-verdict",
  "id": "MHC-D-RESEARCH-0418",
  "version": "0.1.0",
  "title": "Measure agreement on the criteria, not only the verdict",
  "summary": "Two people can disagree on the verdict because they never agreed on what the evidence meant one step earlier.",
  "kind": "protocol",
  "body": "Capture reviewer judgments on the important component criteria as well as the final outcome. Calculate or inspect agreement at each layer. Low criterion agreement suggests definitions, examples or evidence interpretation need work; high criterion agreement with different final verdicts points instead to weighting or integration rules.",
  "limits": [
    "Agreement is not accuracy; reviewers can consistently agree on the wrong interpretation."
  ],
  "topics": [
    "union-decision-hygiene-noise"
  ],
  "intents": [],
  "source_ids": [
    "RS-4DC9763DF7D5A389"
  ],
  "evidence": [
    {
      "claim": "In the IFCN study, agreement on individual criteria ranged only from fair to moderate even though the criteria were explicitly defined.",
      "source_id": "RS-4DC9763DF7D5A389",
      "role": "supports",
      "note": "Criterion-level agreement may depend on training, examples and the intrinsic ambiguity of the observed signal.",
      "locator": "Inter-rater agreement on individual criteria"
    }
  ],
  "use_when": [
    "Reviewers disagree on final decisions and the team cannot tell whether the issue is evidence, criteria or synthesis."
  ],
  "avoid_when": [
    "Agreement is not accuracy; reviewers can consistently agree on the wrong interpretation."
  ],
  "example": "Reviewers may agree that a change has high blast radius and weak rollback but disagree on the overall approval because their weighting differs.",
  "check": "The team can locate whether disagreement enters at evidence interpretation, criterion rating or final synthesis.",
  "steps": [
    "The team can locate whether disagreement enters at evidence interpretation, criterion rating or final synthesis."
  ],
  "sources": [
    {
      "id": "RS-4DC9763DF7D5A389",
      "title": "Utility of the IFCN criteria for identifying interictal epileptiform discharges by experts: A decision hygiene approach to improve inter-rater reliability",
      "url": "https://www.sciencedirect.com/science/article/pii/S1388245725003268"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-E41621E574ADAE4A",
      "title": "Reduce judgment noise without erasing useful disagreement",
      "url": "/collections/reduce-judgment-noise-without-erasing-useful-disagreement"
    }
  ]
}