{
  "schema": "vedokrok.public-item.v1",
  "release_id": "VEDOKROK-SEL-5d86e1b23401e578",
  "url": "/knowledge/compare-prompt-versions-on-saved-test-cases",
  "id": "MHC-D-RESEARCH-9797",
  "version": "0.1.0",
  "title": "Compare prompt versions on saved test cases",
  "summary": "A better answer on one example can hide a new mistake elsewhere.",
  "kind": "protocol",
  "body": "Save an ordinary case, an ambiguous case and a known failure, with the result each should produce. Compare the old and new prompts using the same inputs and checks. Start each attempt without leftover conversation or test state. Keep the model and tools unchanged where possible; record any change so you do not attribute it to the prompt.",
  "limits": [
    "Three cases are an initial smoke test, not proof of general reliability or safety. Provider changes or hidden settings can limit reproducibility."
  ],
  "topics": [
    "ai-work"
  ],
  "intents": [],
  "source_ids": [
    "ai-evals"
  ],
  "evidence": [
    {
      "claim": "Anthropic describes evaluation tasks with defined inputs and success criteria, records of attempts, repeated trials for variable model outputs, and clean, stable test environments.",
      "source_id": "ai-evals",
      "role": "supports",
      "note": "Claim limit: Provider engineering guidance, not experimental proof that a three-case personal prompt check establishes reliability. Recording visible conditions is an editorial adaptation; hidden provider settings may prevent exact reproduction.\nSource reading history: selected_passages; 2026-10-10; Evaluation structure: task, trial, transcript and harness; Step 4: stable environment. The cited official page was retrieved on 2026-10-10 and the named passages were inspected. Card-specific adaptations are independent editorial reasoning, not validated improvements. Selected definitions and Step 4 reread on 2026-10-12; this does not constitute full scientific review.\nAgent release review: Agent checked the exact card fields and its claim, source, locator, example, check and stated boundary. This bounded projection preserves the source pack’s screened evidence scope and does not claim independent scientific or human editorial review.",
      "locator": "Evaluation structure: task, trial, transcript and harness; Step 4: stable environment"
    }
  ],
  "use_when": [
    "You reuse one prompt each week and assume a fluent answer means success."
  ],
  "avoid_when": [
    "Three cases are an initial smoke test, not proof of general reliability or safety. Provider changes or hidden settings can limit reproducibility."
  ],
  "steps": [
    "Save both prompts, the visible model label, relevant settings and tool access, plus each output and its pass or fail result. Mark unavailable details as unknown.",
    "Inspect failures against the agreed checks. If a decision depends on consistency, repeat cases under the recorded conditions and report failures as well as passes."
  ],
  "example": "An invented inbox test expects a meeting request to be labelled scheduling, a request mixing scheduling and billing to be flagged for review, and an automatic reply to be ignored. A rewrite fixes the mixed request but labels the automatic reply scheduling: keep both results.",
  "check": "The comparison shows which saved cases passed or failed under which conditions. One passing attempt does not establish consistent performance.",
  "published_at": "2026-11-13T05:54:13Z",
  "review": {
    "kind": "agent",
    "label": "Agent editorial and scoped source review",
    "reviewed_at": "2026-11-13T05:54:13Z"
  },
  "knowledge_origin": {
    "release_id": "MHC-RPUB-20261113-0d6697fd",
    "corpus_commit": "0d6697fd15f7fb670cfa465c54a411e91752cae4",
    "manifest_sha256": "038080ffd768d7c1ae21ae65ea2a3b8e36664c28df759e06d19f63f6e059bf29",
    "projection_sha256": "489520d20496b8e53d4bcf37ddaae13d1b66f4405e33ae1666ce681073b24c78",
    "published_at": "2026-11-13T05:54:13Z"
  },
  "editorial_overlay": null,
  "editorial": null,
  "notices": [],
  "sources": [
    {
      "id": "ai-evals",
      "title": "Demystifying evals for AI agents",
      "url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-9797",
      "to": "MHC-D-RESEARCH-9798",
      "type": "use_before",
      "url": "/knowledge/rehearse-an-ai-automation-with-test-data"
    }
  ],
  "collections": [
    {
      "id": "RC-MHC-UNION-EVERYDAY-AI-WORKFLOW-CONTROL-2026",
      "title": "Use an AI assistant without outsourcing verification or authority",
      "url": "/collections/use-an-ai-assistant-without-outsourcing-verification-or-authority"
    }
  ],
  "topic_hubs": [
    {
      "id": "ai-work",
      "title": "Working with AI",
      "url": "/topics/ai-work"
    }
  ],
  "solution_packs": []
}
