{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/try-to-break-the-evaluator-before-trusting-it",
  "id": "MHC-D-RESEARCH-1163",
  "version": "0.1.0",
  "title": "Try to break the evaluator before trusting it",
  "summary": "If the agent can game the judge, a higher score can mean a worse system.",
  "kind": "protocol",
  "body": "Design counterexamples that should clearly fail and see whether the evaluator catches them. Test shortcuts, superficial completion signals and plausible-but-wrong outputs. Repair the evaluator before optimizing the agent against its score.",
  "limits": [
    "Passing adversarial checks does not prove the evaluator is complete. New shortcuts can appear as the agent changes."
  ],
  "topics": [
    "union-ai-runtime-verification-2026"
  ],
  "intents": [],
  "source_ids": [
    "RS-273F6FA5C48C6C0B"
  ],
  "evidence": [
    {
      "claim": "A task evaluator should itself be challenged with cases designed to expose false passes before its score is used as a trusted optimization target.",
      "source_id": "RS-273F6FA5C48C6C0B",
      "role": "supports",
      "note": "Adversarial evaluator tests improve confidence only for the tested failure modes; evaluator drift and unanticipated shortcuts still require monitoring.",
      "locator": "9:04-10:50, tool changes the result and try to break the evaluator before trusting its score"
    }
  ],
  "use_when": [
    "An automated evaluator will decide whether agent behavior passes, fails or improves."
  ],
  "avoid_when": [
    "Passing adversarial checks does not prove the evaluator is complete. New shortcuts can appear as the agent changes."
  ],
  "example": "A desktop agent's evaluator checks that a file exists. Add a case where the file exists with the wrong contents; if it passes, the evaluator is too shallow.",
  "check": "The evaluator rejects the intentionally broken cases that matter to the task before its score is used as an optimization target.",
  "steps": [
    "Create a clear true-pass case and a clear true-fail case.",
    "Add a shortcut that looks successful but violates the task.",
    "Add a plausible output with the wrong underlying state.",
    "Inspect false passes before using the evaluator for model or prompt selection."
  ],
  "sources": [
    {
      "id": "RS-273F6FA5C48C6C0B",
      "title": "Computer-Use 2.0: Agents Just Got Multi-Cursor",
      "url": "https://ai.engineer/talks/ZSQb5fzRFPw-computer-use-2-0-agents-just-got"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-ACD6385FB70207D3",
      "title": "Operate AI agents as systems you can replay, verify and constrain",
      "url": "/collections/operate-ai-agents-as-systems-you-can-replay-verify-and-constrain"
    }
  ]
}