{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/evaluate-the-same-agent-task-across-multiple-trials",
  "id": "MHC-D-RESEARCH-0955",
  "version": "0.1.0",
  "title": "Evaluate the same agent task across multiple trials",
  "summary": "A nondeterministic system needs more than one audition.",
  "kind": "protocol",
  "body": "Run representative tasks multiple times and report distribution, not only pass@1 from a favorable run. Capture success, failure mode, steps, cost and variance. This distinguishes occasional capability from repeatable reliability, especially on longer tasks.",
  "limits": [
    "Trial count should match cost and risk; small samples still leave uncertainty."
  ],
  "topics": [
    "union-agent-loop-reliability"
  ],
  "intents": [],
  "source_ids": [
    "RS-EE9379CE3D2ABBDD"
  ],
  "evidence": [
    {
      "claim": "Anthropic's agent-eval guidance defines separate trials because model outputs vary and recommends multiple trials for more consistent evaluation.",
      "source_id": "RS-EE9379CE3D2ABBDD",
      "role": "supports",
      "note": "Trial count should match cost and risk; small samples still leave uncertainty.",
      "locator": "See source record"
    }
  ],
  "use_when": [
    "One successful run is being treated as proof that an agent workflow is reliable."
  ],
  "avoid_when": [
    "Trial count should match cost and risk; small samples still leave uncertainty."
  ],
  "example": "Run the same repository change ten times in isolated copies to see whether the agent consistently edits the right files and completes tests.",
  "check": "Reliability claims include repeated trials and failure distribution rather than one anecdote.",
  "steps": [
    "Reliability claims include repeated trials and failure distribution rather than one anecdote."
  ],
  "sources": [
    {
      "id": "RS-EE9379CE3D2ABBDD",
      "title": "Demystifying evals for AI agents",
      "url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0955",
      "to": "MHC-D-RESEARCH-0319",
      "type": "useful_with",
      "url": "/knowledge/keep-an-eval-set-that-can-embarrass-the-agent"
    }
  ],
  "collections": [
    {
      "id": "RC-043A908C9263BF6A",
      "title": "Make the agent loop finish, recover and stay inspectable",
      "url": "/collections/make-the-agent-loop-finish-recover-and-stay-inspectable"
    }
  ]
}