{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/read-eval-transcripts-before-trusting-the-score",
  "id": "MHC-D-RESEARCH-0956",
  "version": "0.1.0",
  "title": "Read eval transcripts before trusting the score",
  "summary": "A score can fail because the agent failed—or because the eval did.",
  "kind": "protocol",
  "body": "Sample transcripts from passes and failures. Check whether tasks were unambiguous, graders penalized valid solutions, tool constraints caused artificial failures, or the agent genuinely made mistakes. Update the eval when the failure is unfair; update the agent when the failure is real.",
  "limits": [
    "Transcript review is expensive; sample strategically while keeping automated coverage broad."
  ],
  "topics": [
    "union-agent-loop-reliability"
  ],
  "intents": [],
  "source_ids": [
    "RS-EE9379CE3D2ABBDD"
  ],
  "evidence": [
    {
      "claim": "Anthropic explicitly recommends reading eval transcripts to distinguish genuine agent mistakes from unfair grading or harness problems.",
      "source_id": "RS-EE9379CE3D2ABBDD",
      "role": "supports",
      "note": "Transcript review is expensive; sample strategically while keeping automated coverage broad.",
      "locator": "See source record"
    }
  ],
  "use_when": [
    "An agent's eval score moves and the team immediately attributes the change to model quality."
  ],
  "avoid_when": [
    "Transcript review is expensive; sample strategically while keeping automated coverage broad."
  ],
  "example": "A coding agent 'fails' because the grader assumed a filepath the task never specified; that is an eval bug, not an agent regression.",
  "check": "Score changes have a transcript-backed explanation before they drive model or workflow decisions.",
  "steps": [
    "Read multiple failed transcripts.",
    "Read some passes for hidden shortcuts.",
    "Compare grader decision with artifact reality.",
    "Classify agent failure versus eval/harness failure.",
    "Feed real failures back into the eval set."
  ],
  "sources": [
    {
      "id": "RS-EE9379CE3D2ABBDD",
      "title": "Demystifying evals for AI agents",
      "url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0956",
      "to": "MHC-D-RESEARCH-0417",
      "type": "useful_with",
      "url": "/knowledge/do-not-assume-a-rubric-reduced-noise-measure-it"
    }
  ],
  "collections": [
    {
      "id": "RC-043A908C9263BF6A",
      "title": "Make the agent loop finish, recover and stay inspectable",
      "url": "/collections/make-the-agent-loop-finish-recover-and-stay-inspectable"
    }
  ]
}