{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/swap-answer-order-before-trusting-an-ai-judge-s-winner",
  "id": "MHC-D-RESEARCH-0999",
  "version": "0.1.0",
  "title": "Swap answer order before trusting an AI judge's winner",
  "summary": "First place on the screen should not decide first place in the result.",
  "kind": "protocol",
  "body": "Keep the rubric fixed and compare the pair in both orders without revealing which version you favor. Track whether the same answer wins after the swap. Order-sensitive judgments are unresolved evidence, not a reason to keep the more convenient verdict.",
  "limits": [
    "Stochastic variation can also cause disagreement. Passing the order check does not remove verbosity bias, factual mistakes or a poorly chosen rubric."
  ],
  "topics": [
    "work-03-ai-assurance"
  ],
  "intents": [],
  "source_ids": [
    "RS-07323E1B26FBDC0F"
  ],
  "evidence": [
    {
      "claim": "The LLM-as-a-judge study observed order-sensitive preferences and describes comparing both answer orders before declaring a pairwise winner.",
      "source_id": "RS-07323E1B26FBDC0F",
      "role": "supports",
      "note": "Order consistency removes neither all judging biases nor factual errors, and stochastic variation must also be considered.",
      "locator": "Sections 3.3 and 3.4"
    }
  ],
  "use_when": [
    "A model is comparing two drafts, prompts or answers and its verdict will guide a change."
  ],
  "avoid_when": [
    "Stochastic variation can also cause disagreement. Passing the order check does not remove verbosity bias, factual mistakes or a poorly chosen rubric."
  ],
  "example": "A judge favors the new draft when it is shown first but favors the old draft when positions reverse. The comparison has not established a winner.",
  "check": "The report includes order consistency rather than only one pairwise score.",
  "steps": [
    "Remove unnecessary version labels and state the evaluation criteria.",
    "Run the comparison in both orders and map the verdicts back to the original answers.",
    "Inspect disagreement, using a human or task-based test when the distinction matters."
  ],
  "sources": [
    {
      "id": "RS-07323E1B26FBDC0F",
      "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "url": "https://arxiv.org/abs/2306.05685"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0999",
      "to": "MHC-D-RESEARCH-0990",
      "type": "useful_with",
      "url": "/knowledge/keep-acceptance-examples-out-of-the-prompt-workshop"
    }
  ],
  "collections": [
    {
      "id": "RC-620AACFCDEA000D1",
      "title": "Make AI-assisted work earn your trust",
      "url": "/collections/make-ai-assisted-work-earn-your-trust"
    }
  ]
}