{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/report-how-much-work-an-abstaining-system-actually-covers",
  "id": "MHC-D-RESEARCH-0992",
  "version": "0.1.0",
  "title": "Report how much work an abstaining system actually covers",
  "summary": "Being right on the easiest fifth is not the same as handling the whole queue.",
  "kind": "checklist",
  "body": "Measure both the quality of accepted answers and coverage: the share of incoming cases the system handles. Track the rejected cases and the human work they create. Choose the operating point from observed trade-offs, not from the model's confident tone.",
  "limits": [
    "Raw model confidence is not a validated probability. Evidence from selective classifiers does not provide an automatic safety guarantee for a language-model workflow."
  ],
  "topics": [
    "work-03-ai-assurance"
  ],
  "intents": [],
  "source_ids": [
    "RS-700A3E0A4271B66C"
  ],
  "evidence": [
    {
      "claim": "Selective classification can trade coverage for lower risk on the cases it accepts, as demonstrated in the cited image-classification setting.",
      "source_id": "RS-700A3E0A4271B66C",
      "role": "supports",
      "note": "No guarantee transfers to an uncalibrated language-model confidence statement or a new population.",
      "locator": "Abstract"
    }
  ],
  "use_when": [
    "An AI workflow sends uncertain cases to a human and reports high accuracy on what remains."
  ],
  "avoid_when": [
    "Raw model confidence is not a validated probability. Evidence from selective classifiers does not provide an automatic safety guarantee for a language-model workflow."
  ],
  "example": "An extractor handles 70 of 100 documents and refers 30. Its accuracy on the 70 must not be reported as accuracy on all 100.",
  "check": "The report makes accepted coverage, residual errors and referral load visible together.",
  "checklist": [
    "Define what counts as acceptance, abstention and a correct result.",
    "Report accepted-case errors alongside accepted share and review workload.",
    "Check whether difficult groups are disproportionately left for people."
  ],
  "sources": [
    {
      "id": "RS-700A3E0A4271B66C",
      "title": "Selective Classification for Deep Neural Networks",
      "url": "https://arxiv.org/abs/1705.08500"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0992",
      "to": "MHC-D-RESEARCH-0991",
      "type": "useful_with",
      "url": "/knowledge/judge-the-expensive-error-separately-from-the-common-one"
    }
  ],
  "collections": [
    {
      "id": "RC-620AACFCDEA000D1",
      "title": "Make AI-assisted work earn your trust",
      "url": "/collections/make-ai-assisted-work-earn-your-trust"
    }
  ]
}