{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/test-what-should-stay-unchanged-when-the-input-changes",
  "id": "MHC-D-RESEARCH-0993",
  "version": "0.1.0",
  "title": "Test what should stay unchanged when the input changes",
  "summary": "Sometimes you cannot name the whole answer, but you can name what must not change.",
  "kind": "protocol",
  "body": "Define a justified input transformation and the expected relationship between outputs. Reordering independent records should not change their extracted values; changing a relevant negation should affect the meaning. These behavioral tests expose inconsistencies that a single example can miss.",
  "limits": [
    "Do not assume that paraphrases preserve meaning or that order is irrelevant in every task. Incorrect invariants create incorrect tests."
  ],
  "topics": [
    "work-03-ai-assurance"
  ],
  "intents": [],
  "source_ids": [
    "RS-41B2F225E2662AC3"
  ],
  "evidence": [
    {
      "claim": "CheckList provides perturbation-based invariance and directional tests for expected model behavior under controlled input changes.",
      "source_id": "RS-41B2F225E2662AC3",
      "role": "supports",
      "note": "A change is only an invariant when the task's meaning and expected answer really should remain unchanged.",
      "locator": "Perturbing data for INVs and DIRs"
    }
  ],
  "use_when": [
    "You lack a complete answer key but know a relationship that correct outputs must obey."
  ],
  "avoid_when": [
    "Do not assume that paraphrases preserve meaning or that order is irrelevant in every task. Incorrect invariants create incorrect tests."
  ],
  "example": "An extractor receives the same independent records in a different order. The per-record results should remain the same even if output order changes.",
  "check": "A failure can be explained against a written relationship, not merely a feeling that the answers differ.",
  "steps": [
    "State the task-specific invariant or expected direction before running the test.",
    "Create a controlled pair that changes only the relevant feature.",
    "Compare outputs and inspect failures before expanding the test family."
  ],
  "sources": [
    {
      "id": "RS-41B2F225E2662AC3",
      "title": "CheckList: behavioral testing of NLP models",
      "url": "https://github.com/marcotcr/checklist/blob/master/README.md"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-620AACFCDEA000D1",
      "title": "Make AI-assisted work earn your trust",
      "url": "/collections/make-ai-assisted-work-earn-your-trust"
    }
  ]
}