{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/turn-a-production-finding-into-a-reusable-eval-artifact",
  "id": "MHC-D-RESEARCH-1177",
  "version": "0.1.0",
  "title": "Turn a production finding into a reusable eval artifact",
  "summary": "A failure is expensive twice if the system can forget it.",
  "kind": "protocol",
  "body": "Preserve a validated production finding in the form that best prevents recurrence: an issue with evidence, a regression example, an evaluator case or a dataset item. Keep the link to the original trace and the execution boundary where the failure mattered. Then rerun the relevant eval after the fix.",
  "limits": [
    "Do not convert every noisy observation into permanent rules. Validate the finding and avoid leaking production secrets or personal data into repositories or eval sets."
  ],
  "topics": [
    "union-ai-runtime-verification-2026"
  ],
  "intents": [],
  "source_ids": [
    "RS-A4532D2071A27E77"
  ],
  "evidence": [
    {
      "claim": "A production investigation can preserve evidence by turning a validated finding into a reviewable issue, evaluator or dataset example before the repair is automated.",
      "source_id": "RS-A4532D2071A27E77",
      "role": "supports",
      "note": "Runtime evidence can contain noise, secrets or personal data; promotion into a durable artifact requires review, minimization and a clear link to the original failure.",
      "locator": "3:31-18:04, gather evidence, bring it into the repository, create an issue/evaluator/dataset example and preserve the execution boundary"
    }
  ],
  "use_when": [
    "A production incident teaches the team something, but the lesson is likely to disappear after the immediate fix."
  ],
  "avoid_when": [
    "Do not convert every noisy observation into permanent rules. Validate the finding and avoid leaking production secrets or personal data into repositories or eval sets."
  ],
  "example": "A coding agent repeatedly edits generated files instead of their source templates. Preserve one verified incident as a regression case and repository instruction test before merging the fix.",
  "check": "The next version is tested against an artifact derived from the real failure, and the source evidence remains traceable.",
  "steps": [
    "Validate the finding against the original production evidence.",
    "Minimize sensitive data before storing a durable artifact.",
    "Choose the artifact that can catch the failure again.",
    "Link it to the affected component and original evidence.",
    "Rerun the relevant evaluation after the repair."
  ],
  "sources": [
    {
      "id": "RS-A4532D2071A27E77",
      "title": "From Signal to PR: Anatomy of a Self-Improving Agent",
      "url": "https://ai.engineer/talks/9HbzAWnKbo4-from-signal-pr-anatomy-self-improving-agent"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-1177",
      "to": "MHC-D-RESEARCH-0718",
      "type": "useful_with",
      "url": "/knowledge/turn-a-verified-ai-failure-into-a-regression-test"
    }
  ],
  "collections": [
    {
      "id": "RC-ACD6385FB70207D3",
      "title": "Operate AI agents as systems you can replay, verify and constrain",
      "url": "/collections/operate-ai-agents-as-systems-you-can-replay-verify-and-constrain"
    }
  ]
}