{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/keep-acceptance-examples-out-of-the-prompt-workshop",
  "id": "MHC-D-RESEARCH-0990",
  "version": "0.1.0",
  "title": "Keep acceptance examples out of the prompt workshop",
  "summary": "The demonstration has had a lot of rehearsal. The next real case has not.",
  "kind": "protocol",
  "body": "Separate development examples from a held-back acceptance set. Use the first group to revise prompts and rules; use the second to evaluate the frozen candidate. Once acceptance failures guide another revision, those cases are no longer untouched evidence of generalization.",
  "limits": [
    "A tiny or unrepresentative holdout remains weak evidence. Repeatedly selecting changes from the same holdout can contaminate it too."
  ],
  "topics": [
    "work-03-ai-assurance"
  ],
  "intents": [],
  "source_ids": [
    "RS-E75F80345E8006C8"
  ],
  "evidence": [
    {
      "claim": "Using evaluation data to make development choices can create optimistic performance estimates; the scikit-learn guidance keeps test data out of model choices.",
      "source_id": "RS-E75F80345E8006C8",
      "role": "supports",
      "note": "A separate acceptance set for prompts is an application of this principle; representative sampling and repeated-use contamination still matter.",
      "locator": "Data leakage"
    }
  ],
  "use_when": [
    "A prompt improves on familiar examples and you need to know whether it improved beyond them."
  ],
  "avoid_when": [
    "A tiny or unrepresentative holdout remains weak evidence. Repeatedly selecting changes from the same holdout can contaminate it too."
  ],
  "example": "A document extractor is tuned on familiar layouts, then tested on previously held-back layouts with known expected fields.",
  "check": "The report identifies which cases influenced development and which first appeared at evaluation.",
  "steps": [
    "Write expected outcomes for representative, permission-safe cases before testing.",
    "Version the prompt, model configuration and evaluation set together.",
    "Report held-back performance separately from results on examples used during development."
  ],
  "sources": [
    {
      "id": "RS-E75F80345E8006C8",
      "title": "Common pitfalls and recommended practices",
      "url": "https://scikit-learn.org/stable/common_pitfalls.html"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0990",
      "to": "MHC-D-RESEARCH-0993",
      "type": "useful_with",
      "url": "/knowledge/test-what-should-stay-unchanged-when-the-input-changes"
    }
  ],
  "collections": [
    {
      "id": "RC-620AACFCDEA000D1",
      "title": "Make AI-assisted work earn your trust",
      "url": "/collections/make-ai-assisted-work-earn-your-trust"
    }
  ]
}