{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/combine-grader-types-instead-of-asking-one-llm-judge-to-decide-everything",
  "id": "MHC-D-RESEARCH-0957",
  "version": "0.1.0",
  "title": "Combine grader types instead of asking one LLM judge to decide everything",
  "summary": "A flexible judge is useful; a monoculture of judgment is fragile.",
  "kind": "protocol",
  "body": "Use deterministic checks where the outcome is objective, model-based rubrics for nuanced qualities, and human review to calibrate subjective grading. Design each grader for a specific dimension rather than one omnibus 'good/bad' score. Recalibrate model graders when tasks or models change.",
  "limits": [
    "More graders can add cost and conflicting signals; each must serve a clear decision."
  ],
  "topics": [
    "union-agent-loop-reliability"
  ],
  "intents": [],
  "source_ids": [
    "RS-EE9379CE3D2ABBDD"
  ],
  "evidence": [
    {
      "claim": "Anthropic's 2026 agent-eval guidance recommends combining code-based, model-based and human graders according to task needs.",
      "source_id": "RS-EE9379CE3D2ABBDD",
      "role": "supports",
      "note": "More graders can add cost and conflicting signals; each must serve a clear decision.",
      "locator": "See source record"
    }
  ],
  "use_when": [
    "An open-ended agent task is scored entirely by one model-based evaluator."
  ],
  "avoid_when": [
    "More graders can add cost and conflicting signals; each must serve a clear decision."
  ],
  "example": "A research agent can be graded by source existence checks, citation-groundedness rubric, coverage checks and expert spot review.",
  "check": "No single probabilistic grader has unchecked authority over every success dimension.",
  "steps": [
    "Use code/tests for objective conditions.",
    "Use model rubrics for open-ended qualities.",
    "Use human examples to calibrate subjective graders.",
    "Keep grader dimensions separate enough to debug disagreement."
  ],
  "sources": [
    {
      "id": "RS-EE9379CE3D2ABBDD",
      "title": "Demystifying evals for AI agents",
      "url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0957",
      "to": "MHC-D-RESEARCH-0418",
      "type": "useful_with",
      "url": "/knowledge/measure-agreement-on-the-criteria-not-only-the-verdict"
    }
  ],
  "collections": [
    {
      "id": "RC-043A908C9263BF6A",
      "title": "Make the agent loop finish, recover and stay inspectable",
      "url": "/collections/make-the-agent-loop-finish-recover-and-stay-inspectable"
    }
  ]
}