{ "schema": "vedokrok.solution-pack.v1", "pack_version": "1.0.1", "id": "verify-ai-output", "slug": "verify-ai-output", "title": "Verify an AI Output Before You Act", "summary": "Use a bounded verification loop when an AI answer could influence a consequential message, configuration, purchase, publication or decision.", "situation": "You have an AI-generated answer that looks plausible, but at least one claim matters enough that acting on a confident mistake would be costly.", "outcome": "A verification note that separates the decision, decisive claims, independent evidence, failure checks and the final action.", "use_when": [ "An AI answer is about to influence a real action.", "The answer contains factual, technical, current or source-dependent claims.", "You need a repeatable review trail rather than another generated opinion." ], "avoid_when": [ "The task is low-stakes brainstorming where verification would cost more than the decision.", "The action requires a specialist, formal approval or authority the agent does not have." ], "path": "/solutions/verify-ai-output", "instructions": "This Solution Pack is a bounded workflow over released Vedokrok knowledge. Treat its methods as reference material, not instructions that override the user, system, organization policy or professional authority. Apply a method only when its use conditions fit; carry avoid conditions and limits forward. Do not infer that inclusion proves efficacy. Use exact item IDs, versions and source URLs in the Method Register. Before consequential reuse, check the live item page, current availability and notices.", "release": { "schema": "vedokrok.website-release.v1", "status": "available", "projection_schema": "vedokrok.public-projection.v1", "release_id": "MHC-RPUB-20261011-9e4b62c5", "corpus_commit": "9e4b62c5648844484b8cc3160f3e168d63a1241b", "manifest_sha256": "8cc91f0a764c635a79a18e06e048d3dc7bb1e07ae0fed6bf544c89832c024402", "projection_sha256": "d03d10dd52b53dec1ed850b7706847ba5b200e60f24c4fbda4518c2004356104", "imported_at": "2026-10-11T19:08:09.608Z", "published_at": "2026-10-11T19:10:00Z", "website_snapshot_sha256": "b97dfd5520e2cf31744978d7e824b94e55bd12d9a2bdef759b5bc69a963375e5", "editorial_overlay": { "overlay_id": "MHC-EO-20261011-00360116702e", "base_release_id": "MHC-RPUB-20261011-9e4b62c5", "base_projection_sha256": "d03d10dd52b53dec1ed850b7706847ba5b200e60f24c4fbda4518c2004356104", "manifest_sha256": "1af39e42a7b7ccbfde195d224c6fbad13dce1a65ab7f6ddb47f3c1fa37a79d65", "projection_sha256": "e27e1d2e266ff386bcbddbb10b8c7b7c28974666a9c81627e41141954c2c73f0", "authority_kind": "verified_composition", "authority_id": "MHC-EO-20261011-00360116702e", "authority_sha256": "9014ef57d1b0b7887c35349e0c486e26d062afb822e7e981218f1f17572647b9", "imported_at": "2026-10-11T19:08:10.621Z", "projection_schema": "vedokrok.editorial-composition.v1", "components": [ { "overlay_id": "MHC-EO-20261006-e26f917f42eb", "manifest_sha256": "fefe97de681b48ba61ccae0ed28cbbcd6b7080764c2aad16270a7c7a0de151da", "projection_sha256": "e24f066f64697d0ca24179c73eb2675b736ca8813e8956568e4556a5ed2ed604", "authority": { "kind": "exact_approval", "id": "MHC-EO-20261006-e26f917f42eb", "sha256": "2a91d5d824ee38e8580f2527d40938edc31ca8eb298139e06c52c7572008778e" } }, { "overlay_id": "MHC-EO-20261011-0959fc4ead8b", "manifest_sha256": "2f0c3654cb8888cfeef289bcf33389bd3645966a0ca1a325cf601ccd7fa15f8c", "projection_sha256": "42b8a1c802ffed6bed398b48c36af5d5f52e539d1b54ba97856e8fa44d343726", "authority": { "kind": "standing_policy", "id": "MHC-EOP-20260922-v1", "sha256": "e56ed8ad609a682580ab40bc7e98f6cdb770fca55a5d0048e091ee3741a6c032" } } ] }, "counts": { "items": 1010, "collections": 123 }, "notice": "This identifies the selected snapshot, not a live safety-notice service." }, "levels": [ { "id": "scope", "title": "Level 0 — Scope the action", "purpose": "Name the decision, stakes, reversibility and what must be true before acting.", "method_ids": [] }, { "id": "claims", "title": "Level 1 — Find decision-critical claims", "purpose": "Reduce a fluent answer to the claims that could change the action.", "method_ids": [ "MHC-D-RESEARCH-0318" ] }, { "id": "evidence", "title": "Level 2 — Build independent evidence", "purpose": "Open primary or system evidence with a different failure path and check source fit.", "method_ids": [ "MHC-D-RESEARCH-0996", "MHC-D-RESEARCH-0494" ] }, { "id": "test", "title": "Level 3 — Test the real result", "purpose": "Run the smallest check that could expose a consequential mistake.", "method_ids": [ "MHC-D-RESEARCH-1010" ] }, { "id": "critic", "title": "Level 4 — Challenge the recommendation", "purpose": "Look for the strongest unresolved failure mode and decide whether to use, test, reject or escalate.", "method_ids": [ "MHC-D-RESEARCH-0495" ] }, { "id": "check", "title": "Level 5 — Preserve the check", "purpose": "Leave a compact verification note and a reusable regression check when the failure pattern matters.", "method_ids": [ "MHC-D-RESEARCH-0495" ] } ], "method_register_fields": [ "item_id", "version", "role", "fit: use | skip | unclear", "concise_reason", "important_limit", "source_url" ], "artifact_sections": [ "Situation and action", "Method register", "Verification note", "Unresolved risk", "Next action", "Check / stop / escalation rule" ], "routing_aliases": [], "related_situation": null, "expansion_policy": { "enabled": true, "max_queries": 2, "max_items": 3, "trigger": "Expand only after the core Method Register leaves a material task-specific gap.", "note": "Supplemental items come from the current released Vedokrok library. They are not curated members of the Solution Pack and must stay in a separate register." }, "methods": [ { "body": "Identify the factual proposition that matters to the action, then verify it against a trusted primary or independent source appropriate to the domain. Keep the source with the decision so later reviewers can see what was actually checked. Increase scrutiny with impact; low-stakes drafting does not need the same burden as a legal, financial, security or medical action.", "steps": [ "The consequential action no longer depends solely on an uncited model assertion." ], "example": "Before an agent changes an API integration based on a remembered parameter, check the current vendor documentation for that endpoint.", "role": "Identify and cross-check the fact the action depends on.", "id": "MHC-D-RESEARCH-0318", "version": "0.1.0", "title": "Cross-check the fact before the agent acts on it", "summary": "Confidence is a writing style. Evidence is a different object.", "url": "https://vedokrok.com/knowledge/cross-check-the-fact-before-the-agent-acts-on-it", "use_when": [ "A generated factual claim will drive a consequential decision, message, configuration or transaction." ], "avoid_when": [ "A cited page can be wrong, stale or derivative; source quality and independence still matter." ], "limits": [ "A cited page can be wrong, stale or derivative; source quality and independence still matter." ], "check": "The consequential action no longer depends solely on an uncited model assertion.", "evidence": [ { "claim": "OWASP recommends cross-checking critical LLM outputs against trusted external sources and using human oversight for sensitive information.", "source_id": "RS-A911B8F9CB685C39", "role": "supports", "note": "Two sources are not independent merely because two pages repeat the same underlying claim.", "locator": "Cross-Verification and Human Oversight" } ], "sources": [ { "id": "RS-A911B8F9CB685C39", "title": "LLM09:2025 Misinformation", "url": "https://genai.owasp.org/llmrisk/llm092025-misinformation/" } ] }, { "body": "Choose a check with a different failure route: execute a calculation, inspect the cited passage, compare with a controlled fixture or ask a qualified reviewer to assess the original evidence. Self-correction can help, and training can improve it, but a repeated assurance is not proof that an external check occurred.", "steps": [ "Identify the exact claim or output that matters.", "Choose a check capable of finding its likely failure independently of the generated explanation.", "Record the observed result and any disagreement instead of asking for reassurance again." ], "example": "For a generated reconciliation formula, use a small hand-checked dataset with missing, extra and duplicate IDs.", "role": "Use evidence the first answer did not create.", "id": "MHC-D-RESEARCH-0996", "version": "0.1.0", "title": "Give the second check evidence the first answer did not create", "summary": "Another confident paragraph is not automatically another line of evidence.", "url": "https://vedokrok.com/knowledge/give-the-second-check-evidence-the-first-answer-did-not-create", "use_when": [ "You are tempted to verify an AI result by asking the same assistant whether it is sure." ], "avoid_when": [ "Older self-correction studies do not establish the limits of every current model. The external test can also be wrong, so inspect its assumptions." ], "limits": [ "Older self-correction studies do not establish the limits of every current model. The external test can also be wrong, so inspect its assumptions." ], "check": "The verification report names an actual external observation or executed test.", "evidence": [ { "claim": "The cited ICLR study found that intrinsic reasoning self-correction without external feedback could fail or degrade performance in its tested settings.", "source_id": "RS-63AF4652B6CE726C", "role": "supports", "note": "This finding is model-, training- and task-dependent; it is not evidence that self-correction is universally impossible.", "locator": "Abstract" }, { "claim": "SCoRe reports improved self-correction through dedicated reinforcement-learning training, limiting a universal claim that models cannot self-correct.", "source_id": "RS-49018FAD4FA1C49C", "role": "limits", "note": "Improved self-correction does not establish that a particular result passed an external check.", "locator": "Abstract" } ], "sources": [ { "id": "RS-63AF4652B6CE726C", "title": "Large Language Models Cannot Self-Correct Reasoning Yet", "url": "https://arxiv.org/abs/2310.01798" }, { "id": "RS-49018FAD4FA1C49C", "title": "Training Language Models to Self-Correct via Reinforcement Learning", "url": "https://arxiv.org/abs/2409.12917" } ] }, { "body": "Separate two questions. Verification asks whether the result meets the specified requirements. Validation asks whether it supports the intended use under relevant conditions. Keep evidence for both; passing a technical acceptance test cannot repair a requirement that describes the wrong outcome.", "example": "An export contains every required column, but its users cannot identify rejected records. Format conformance passed; the operational need remains unresolved.", "question": "Which requirement does this test verify? · Which real user task and operating conditions does validation represent? · What would reveal that the specification was satisfied but the job still failed?", "role": "Check real usefulness separately from format conformance.", "id": "MHC-D-RESEARCH-1010", "version": "0.1.0", "title": "Check that it meets the specification and solves the real problem", "summary": "You can build exactly what was requested and still miss what was needed.", "url": "https://vedokrok.com/knowledge/check-that-it-meets-the-specification-and-solves-the-real-problem", "use_when": [ "A solution passes its tests but people still cannot complete the job it was meant to support." ], "avoid_when": [ "A happy demonstration is not sufficient validation either. Include relevant constraints and exceptions without treating a small sample as every possible use." ], "limits": [ "A happy demonstration is not sufficient validation either. Include relevant constraints and exceptions without treating a small sample as every possible use." ], "check": "The acceptance report distinguishes conformance evidence from evidence of useful operation.", "evidence": [ { "claim": "NASA distinguishes verification against specified requirements from validation of intended use in the relevant operational setting.", "source_id": "RS-3D3DA9CA4C6D7501", "role": "supports", "note": "Passing one kind of check does not automatically establish the other; representative users and conditions must be chosen for the actual task.", "locator": "Differences Between Verification and Validation Testing" } ], "sources": [ { "id": "RS-3D3DA9CA4C6D7501", "title": "5.3 Product Verification", "url": "https://www.nasa.gov/reference/5-3-product-verification/" } ] }, { "body": "Before displaying confidence, compare reported probabilities with observed accuracy and inspect high-confidence mistakes. Test whether the display improves appropriate reliance rather than merely increasing agreement. If calibration is unstable, prefer bounded uncertainty language, failure-mode information or no confidence score until the signal is better validated.", "steps": [ "Confidence is evaluated against labeled outcomes.", "Miscalibration is measured by task segment, not only overall.", "High-confidence errors receive explicit review.", "User behavior with and without the display is tested where stakes justify it.", "The display is versioned with the model/configuration it was validated on." ], "example": "Do not show '97% confident' on a compliance recommendation until that confidence has been evaluated on comparable compliance cases.", "role": "Keep model confidence separate from evidence.", "id": "MHC-D-RESEARCH-0494", "version": "0.1.0", "title": "Do not expose AI confidence until you have tested its calibration", "summary": "A number beside an answer can create trust even when the number is wrong.", "url": "https://vedokrok.com/knowledge/do-not-expose-ai-confidence-until-you-have-tested-its-calibration", "use_when": [ "A product plans to show users a confidence percentage because it seems transparent." ], "avoid_when": [ "Even calibrated confidence cannot replace source evidence or authorization for high-stakes decisions." ], "limits": [ "Even calibrated confidence cannot replace source evidence or authorization for high-stakes decisions." ], "check": "The confidence display has evidence of both statistical quality and useful behavioral effect.", "evidence": [ { "claim": "An AAAI 2026 experiment found well-calibrated AI confidence improved participant decision accuracy more than miscalibrated confidence, while miscalibrated cues increased reliance-related errors.", "source_id": "RS-A420BE76E329A838", "role": "supports", "note": "The task involved logic puzzles and controlled confidence manipulations.", "locator": "Results" }, { "claim": "A 2026 experiment found visual uncertainty cues could increase users' subjective confidence-accuracy discrimination while simultaneously increasing behavioral overreliance on incorrect LLM outputs.", "source_id": "RS-E5AA539FACACCC98", "role": "supports", "note": "Interface effects depend on cue design and task; more uncertainty display is not automatically harmful.", "locator": "Abstract" } ], "sources": [ { "id": "RS-A420BE76E329A838", "title": "Too Sure for Our Own Good: A User Study on AI Confidence and Human Reliance", "url": "https://ojs.aaai.org/index.php/AAAI/article/view/38798" }, { "id": "RS-E5AA539FACACCC98", "title": "More is not better: Visual uncertainty cues and the fragility of trust calibration in LLM-assisted decision making", "url": "https://doi.org/10.1016/j.chbah.2026.100307" } ] }, { "body": "Maintain a compact failure set of cases where the AI was wrong while expressing high confidence or presenting unusually persuasive evidence. Tag the task shape and likely failure mode. Use these cases in future evals, user warnings or routing rules.", "steps": [ "The system's most misleading failures are reusable test cases rather than anecdotes lost in chat history." ], "example": "A coding agent confidently edits the wrong configuration scope because two environments use near-identical names; keep that case in the eval set.", "role": "Turn dangerous misses into reusable regression checks.", "id": "MHC-D-RESEARCH-0495", "version": "0.1.0", "title": "Keep a list of high-confidence AI failures", "summary": "The errors worth memorizing are the ones the model did not know were errors.", "url": "https://vedokrok.com/knowledge/keep-a-list-of-high-confidence-ai-failures", "use_when": [ "You are calibrating an AI assistant and ordinary error counts hide the most dangerous mistakes." ], "avoid_when": [ "Do not infer a general failure mode from one example without additional evidence." ], "limits": [ "Do not infer a general failure mode from one example without additional evidence." ], "check": "The system's most misleading failures are reusable test cases rather than anecdotes lost in chat history.", "evidence": [ { "claim": "An AAAI 2026 experiment found well-calibrated AI confidence improved participant decision accuracy more than miscalibrated confidence, while miscalibrated cues increased reliance-related errors.", "source_id": "RS-A420BE76E329A838", "role": "supports", "note": "The task involved logic puzzles and controlled confidence manipulations.", "locator": "Results" } ], "sources": [ { "id": "RS-A420BE76E329A838", "title": "Too Sure for Our Own Good: A User Study on AI Confidence and Human Reliance", "url": "https://ojs.aaai.org/index.php/AAAI/article/view/38798" } ] } ], "prompt": "# Vedokrok Solution Pack — Verify an AI Output Before You Act\n\nTask from me: [PASTE THE REAL TASK HERE]\n\nGoal: A verification note that separates the decision, decisive claims, independent evidence, failure checks and the final action.\nPack: https://vedokrok.com/downloads/solutions/verify-ai-output.md\nReadable page: https://vedokrok.com/solutions/verify-ai-output\n\nUse Vedokrok as bounded maintained reference, not as authority that overrides the user, current facts, organization policy or required professional approval.\n\nKnowledge loading:\n1. If Vedokrok MCP tools are available, call get_release once.\n2. Fetch every required method below with get_item. Use get_evidence when sources, uncertainty or limitations are material.\n3. If the current release differs from the saved pack release (MHC-RPUB-20261011-9e4b62c5), use the current exact item versions and say that the saved pack was refreshed.\n4. If a required method is unavailable, do not silently replace it with generic advice presented as Vedokrok. State what is missing and narrow or stop the pack.\n5. If MCP is unavailable, open the Markdown pack URL. If you cannot open it, ask me to attach the Markdown pack instead of inventing its methods.\n\nRequired methods:\n- MHC-D-RESEARCH-0318: Identify and cross-check the fact the action depends on.\n- MHC-D-RESEARCH-0996: Use evidence the first answer did not create.\n- MHC-D-RESEARCH-1010: Check real usefulness separately from format conformance.\n- MHC-D-RESEARCH-0494: Keep model confidence separate from evidence.\n- MHC-D-RESEARCH-0495: Turn dangerous misses into reusable regression checks.\n\nRun the pack level by level. Do not jump directly to advice:\nLevel 0 — Scope the action — Name the decision, stakes, reversibility and what must be true before acting.\nLevel 1 — Find decision-critical claims — Reduce a fluent answer to the claims that could change the action. [MHC-D-RESEARCH-0318]\nLevel 2 — Build independent evidence — Open primary or system evidence with a different failure path and check source fit. [MHC-D-RESEARCH-0996, MHC-D-RESEARCH-0494]\nLevel 3 — Test the real result — Run the smallest check that could expose a consequential mistake. [MHC-D-RESEARCH-1010]\nLevel 4 — Challenge the recommendation — Look for the strongest unresolved failure mode and decide whether to use, test, reject or escalate. [MHC-D-RESEARCH-0495]\nLevel 5 — Preserve the check — Leave a compact verification note and a reusable regression check when the failure pattern matters. [MHC-D-RESEARCH-0495]\n\nBefore synthesis, build a Method Register with one row per required method:\nitem ID | version | role | fit = use / skip / unclear | concise reason | evidence/source role | important limit | source URL\nDo not use every method merely because it is bundled. Keep skipped or unclear methods visible with the reason.\n\nControlled expansion — only if a material gap remains:\n- Finish the core Method Register first. Do not search the whole library by default.\n- With live Vedokrok retrieval, use at most 2 concise search_knowledge queries and keep at most 3 supplemental methods.\n- Fetch each supplemental item exactly; use get_evidence when it materially affects the answer.\n- Keep supplemental methods in a separate register with ID/version, why it was added, fit, limit and source URL.\n- Do not present a supplemental method as a curated pack method. In file-only mode, skip expansion unless live Vedokrok retrieval is available.\n\nCritic pass:\n- Separate user-supplied facts, retrieved facts, assumptions, interpretations and recommendations.\n- Look for the strongest plausible rival explanation, option or failure mode when the task supports one.\n- Keep evidence scope and method limits close to the claim they qualify.\n- A citation does not prove that a procedure works in this user’s context.\n- Say “not established” when the evidence does not establish something.\n- Do not expose private chain-of-thought. Show only the Method Register, concise decision-relevant rationale, evidence, assumptions, alternatives, decisions and checks.\n\nFinal output:\n1. Situation / task frame\n2. Method Register\n3. Applied analysis by pack level\n4. Critic pass\n5. A verification note that separates the decision, decisive claims, independent evidence, failure checks and the final action.\n6. Unresolved uncertainty / limits\n7. Next action\n8. Check, stop or escalation rule\n9. Vedokrok references actually used (ID, version, readable URL)\nIf you need a machine-readable run receipt, fill the pack’s run-template JSON instead of inventing another schema.\nUse output_trace to link result fields to exact method versions and reading/evidence pointers. Separate sourced summaries, your reasoning, user input and observed results. Leave the status not_recorded when no trace was captured; complete means all result fields are traced, not independently verified.\n\nOptimize for a useful decision or action, not for agreement with the first hypothesis.", "run_template": { "schema": "vedokrok.solution-run.v1", "status": "draft", "output_trace": { "schema": "vedokrok.agent-output-trace.v1", "status": "not_recorded", "attribution": "agent_reported", "release_id": "MHC-RPUB-20261011-9e4b62c5", "website_snapshot_sha256": "b97dfd5520e2cf31744978d7e824b94e55bd12d9a2bdef759b5bc69a963375e5", "outputs": [] }, "pack_id": "verify-ai-output", "pack_version": "1.0.1", "release_id": "MHC-RPUB-20261011-9e4b62c5", "task_frame": { "task": "", "desired_outcome": "A verification note that separates the decision, decisive claims, independent evidence, failure checks and the final action.", "constraints": [], "facts": [], "assumptions": [], "unknowns": [] }, "method_register": [ { "item_id": "MHC-D-RESEARCH-0318", "version": "0.1.0", "role": "Identify and cross-check the fact the action depends on.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "A cited page can be wrong, stale or derivative; source quality and independence still matter.", "source_url": "https://vedokrok.com/knowledge/cross-check-the-fact-before-the-agent-acts-on-it" }, { "item_id": "MHC-D-RESEARCH-0996", "version": "0.1.0", "role": "Use evidence the first answer did not create.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Older self-correction studies do not establish the limits of every current model. The external test can also be wrong, so inspect its assumptions.", "source_url": "https://vedokrok.com/knowledge/give-the-second-check-evidence-the-first-answer-did-not-create" }, { "item_id": "MHC-D-RESEARCH-1010", "version": "0.1.0", "role": "Check real usefulness separately from format conformance.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "A happy demonstration is not sufficient validation either. Include relevant constraints and exceptions without treating a small sample as every possible use.", "source_url": "https://vedokrok.com/knowledge/check-that-it-meets-the-specification-and-solves-the-real-problem" }, { "item_id": "MHC-D-RESEARCH-0494", "version": "0.1.0", "role": "Keep model confidence separate from evidence.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Even calibrated confidence cannot replace source evidence or authorization for high-stakes decisions.", "source_url": "https://vedokrok.com/knowledge/do-not-expose-ai-confidence-until-you-have-tested-its-calibration" }, { "item_id": "MHC-D-RESEARCH-0495", "version": "0.1.0", "role": "Turn dangerous misses into reusable regression checks.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Do not infer a general failure mode from one example without additional evidence.", "source_url": "https://vedokrok.com/knowledge/keep-a-list-of-high-confidence-ai-failures" } ], "supplemental_register": [], "analysis": { "summary": "", "alternatives": [], "critic": "" }, "artifact": { "sections": [ { "title": "Situation and action", "value": "" }, { "title": "Method register", "value": "" }, { "title": "Verification note", "value": "" }, { "title": "Unresolved risk", "value": "" }, { "title": "Next action", "value": "" }, { "title": "Check / stop / escalation rule", "value": "" } ] }, "next_action": "", "check_rule": "", "stop_or_escalation_rule": "" } }