{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/test-an-intervention-across-a-replay-cohort",
  "id": "MHC-D-RESEARCH-1160",
  "version": "0.1.0",
  "title": "Test an intervention across a replay cohort",
  "summary": "One rescued trace is a story. A cohort begins to look like evidence.",
  "kind": "protocol",
  "body": "Collect a bounded set of representative checkpoints from real runs. Apply the same intervention to each, then compare task outcomes, decisions, cost and failure types. This reveals whether a local fix generalizes across the cases that motivated it.",
  "limits": [
    "A cohort is not the future. Rare events and distribution shifts may be absent, so production monitoring still matters."
  ],
  "topics": [
    "union-ai-runtime-verification-2026"
  ],
  "intents": [],
  "source_ids": [
    "RS-EA671DF45F465038"
  ],
  "evidence": [
    {
      "claim": "Comparing an intervention across multiple checkpointed runs can reveal outcome changes that a single replay would not represent reliably.",
      "source_id": "RS-EA671DF45F465038",
      "role": "supports",
      "note": "A replay cohort only represents the production slice it was sampled from; rare cases and future distribution shifts can remain uncovered.",
      "locator": "10:54-15:10, apply the intervention across a cohort and use the cohort to change the release decision"
    }
  ],
  "use_when": [
    "One replay looks better after a model, prompt, tool or routing change and you need to know whether the improvement is stable enough to matter."
  ],
  "avoid_when": [
    "A cohort is not the future. Rare events and distribution shifts may be absent, so production monitoring still matters."
  ],
  "example": "Before moving support triage to a cheaper model, replay a sample of real triage checkpoints and compare correct routing, escalation and cost across the cohort.",
  "check": "The release decision cites cohort-level outcomes and notable regressions rather than one favorable replay.",
  "steps": [
    "Define the production slice the cohort should represent.",
    "Replay each checkpoint with the same intervention.",
    "Measure outcomes and important failure categories, not only trace similarity.",
    "Review cases that improve, regress or remain unchanged before release."
  ],
  "sources": [
    {
      "id": "RS-EA671DF45F465038",
      "title": "Your Agents Need a Save Button",
      "url": "https://ai.engineer/talks/bZISsg7H7DA-your-agents-need-save-button"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-1160",
      "to": "MHC-D-RESEARCH-0955",
      "type": "useful_with",
      "url": "/knowledge/evaluate-the-same-agent-task-across-multiple-trials"
    }
  ],
  "collections": [
    {
      "id": "RC-ACD6385FB70207D3",
      "title": "Operate AI agents as systems you can replay, verify and constrain",
      "url": "/collections/operate-ai-agents-as-systems-you-can-replay-verify-and-constrain"
    }
  ]
}