{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/keep-a-fresh-dataset-for-the-discovery-s-next-test",
  "id": "MHC-D-RESEARCH-0270",
  "version": "0.1.0",
  "title": "Keep a fresh dataset for the discovery's next test",
  "summary": "Data that helped invent the rule cannot also be completely surprised by it.",
  "kind": "protocol",
  "body": "Use an untouched holdout or a genuinely new sample for a later check. Fix the rule and evaluation before examining that result. If you repeatedly tune against the holdout, acknowledge that it has become part of development and arrange a fresh evaluation.",
  "limits": [
    "A later sample can differ for real contextual reasons. Separation reduces one problem, not every source of bias."
  ],
  "topics": [
    "union-statistical-judgment"
  ],
  "intents": [],
  "source_ids": [
    "RS-8AFFAB6BB9BA52C9"
  ],
  "evidence": [
    {
      "claim": "Data reserved from exploration can provide a later check of a hypothesis formed on another portion of the data.",
      "source_id": "RS-8AFFAB6BB9BA52C9",
      "role": "supports",
      "note": "Repeatedly examining and tuning against the holdout removes the separation it was meant to provide.",
      "locator": "Holdout data; confirmatory and exploratory research"
    }
  ],
  "use_when": [
    "Exploration has produced a promising rule, segment or predictive pattern."
  ],
  "avoid_when": [
    "A later sample can differ for real contextual reasons. Separation reduces one problem, not every source of bias."
  ],
  "example": "A segment discovered in last quarter's records can be tested on a later eligible period, with time-related differences considered explicitly.",
  "check": "The claimed confirmation uses data that did not guide the final rule's selection.",
  "steps": [
    "Separate exploratory data from a suitable later evaluation source.",
    "Specify the rule, outcome and success criterion before accessing the evaluation results.",
    "Preserve failures and avoid quietly retuning until the reserved data look favorable."
  ],
  "sources": [
    {
      "id": "RS-8AFFAB6BB9BA52C9",
      "title": "Preregistration",
      "url": "https://www.cos.io/initiatives/prereg"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-D4FD0B28897E9517",
      "title": "Check what a study can actually tell you",
      "url": "/collections/check-what-a-study-can-actually-tell-you"
    }
  ]
}