{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/ablate-a-skill-before-you-trust-it",
  "id": "MHC-D-RESEARCH-0709",
  "version": "0.1.0",
  "title": "Ablate a skill before you trust it",
  "summary": "If removing the skill changes nothing, the skill may be documentation for humans rather than leverage for the model.",
  "kind": "protocol",
  "body": "Run representative tasks in clean environments with the skill enabled and disabled. Repeat enough trials to see whether the difference survives model variability. Compare outcome quality, trigger accuracy, cost and context use. Prune instructions that do not change useful behavior, and keep the eval so future model or harness upgrades can show when the skill is no longer needed.",
  "limits": [
    "Small eval sets can miss rare benefits; retain safety-critical instructions when removal risk cannot be responsibly tested."
  ],
  "topics": [
    "union-ai-context-memory-skills"
  ],
  "intents": [],
  "source_ids": [
    "RS-0A7653BCD4512BA1"
  ],
  "evidence": [
    {
      "claim": "AI Engineer recommends evaluating skills on repeatable tasks, including clean runs with and without the skill, because a skill that does not change outcomes may only consume context.",
      "source_id": "RS-0A7653BCD4512BA1",
      "role": "supports",
      "note": "Ablation results depend on task set, model and grader quality.",
      "locator": "Test cleanly, repeat, and compare skill behavior"
    }
  ],
  "use_when": [
    "A prompt, skill or instruction bundle is believed to improve an agent."
  ],
  "avoid_when": [
    "Small eval sets can miss rare benefits; retain safety-critical instructions when removal risk cannot be responsibly tested."
  ],
  "example": "A code-review skill that adds 1,500 tokens but catches no additional defects across repeated tasks should be simplified or retired.",
  "check": "The skill has evidence of incremental value over the same workflow without it.",
  "steps": [
    "Choose representative tasks and graders.",
    "Run clean trials with the skill.",
    "Run comparable trials without it.",
    "Compare outcomes, trigger failures and cost.",
    "Prune or revise no-op instructions; keep the regression test."
  ],
  "sources": [
    {
      "id": "RS-0A7653BCD4512BA1",
      "title": "Don't Ship Skills Without Evals",
      "url": "https://ai.engineer/talks/0vphxNt4wyk-dont-ship-skills-without-evals"
    }
  ],
  "relations": [],
  "collections": [
    {
      "id": "RC-74B3DD1F3F9BDCB4",
      "title": "Give AI the right context, memory and reusable skills",
      "url": "/collections/give-ai-the-right-context-memory-and-reusable-skills"
    }
  ]
}