{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/run-ai-evals-from-a-clean-starting-state",
  "id": "MHC-D-RESEARCH-0722",
  "version": "0.1.0",
  "title": "Run AI evals from a clean starting state",
  "summary": "A hidden file from yesterday can make today's agent look brilliant.",
  "kind": "protocol",
  "body": "Reset the workspace, conversation and generated artifacts to the intended baseline before each comparable trial. Seed only the state the real workflow is supposed to have. Record versions of model, tools, prompt or skill and important dependencies. This prevents leftover outputs, caches or prior messages from leaking answers into the next run.",
  "limits": [
    "If production intentionally carries memory across sessions, evaluate that stateful workflow separately rather than pretending every run is cold."
  ],
  "topics": [
    "union-ai-eval-learning-workflow"
  ],
  "intents": [],
  "source_ids": [
    "RS-FD96DA85C7B10F5A"
  ],
  "evidence": [
    {
      "claim": "AI Engineer skill-eval guidance recommends clean workspaces and repeated trials so hidden artifacts or prior messages do not contaminate comparisons.",
      "source_id": "RS-FD96DA85C7B10F5A",
      "role": "supports",
      "note": "Perfect isolation may be unrealistic for production workflows; the eval should represent intended starting state.",
      "locator": "Test cleanly and repeat"
    }
  ],
  "use_when": [
    "You compare prompts, skills, agents or models across repeated tasks."
  ],
  "avoid_when": [
    "If production intentionally carries memory across sessions, evaluate that stateful workflow separately rather than pretending every run is cold."
  ],
  "example": "When testing a research skill, delete the previous report and memory files unless persistent memory is explicitly part of the product being evaluated.",
  "check": "A trial can be reproduced without depending on residue from a previous run.",
  "steps": [
    "Workspace matches the intended starting state.",
    "Prior generated artifacts are removed unless production would keep them.",
    "Conversation history is controlled.",
    "Model, harness and skill versions are recorded.",
    "Random or external dependencies are noted where relevant."
  ],
  "sources": [
    {
      "id": "RS-FD96DA85C7B10F5A",
      "title": "Don't Ship Skills Without Evals",
      "url": "https://ai.engineer/talks/0vphxNt4wyk-dont-ship-skills-without-evals"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0722",
      "to": "MHC-D-RESEARCH-0709",
      "type": "use_before",
      "url": "/knowledge/ablate-a-skill-before-you-trust-it"
    }
  ],
  "collections": [
    {
      "id": "RC-ABC17A288CB87C85",
      "title": "Make AI work improve from failures instead of accumulating rituals",
      "url": "/collections/make-ai-work-improve-from-failures-instead-of-accumulating-rituals"
    }
  ]
}