{ "schema": "vedokrok.solution-pack.v1", "pack_version": "1.0.1", "id": "verify-agent-work", "slug": "verify-agent-work", "title": "Verify an AI Agent Actually Finished the Work", "summary": "Check an agent's completion claim against observable artifacts, executable evidence and unresolved gaps before accepting the work.", "situation": "A coding, research or workflow agent reports that a task is complete, but you need to know whether the requested result exists, works and has enough independent verification to accept.", "outcome": "A completion receipt that maps the requested result to changed artifacts, checks actually run, unresolved gaps and an explicit accept, revise or stop decision.", "use_when": [ "An AI agent changed code, files, configuration, research or another inspectable deliverable.", "A confident done message could hide partial implementation, unrun checks, stale evidence or work that exists only in chat.", "You need acceptance evidence before merge, deployment, handoff or reuse." ], "avoid_when": [ "The task is exploratory ideation with no concrete result or definition of done.", "Required validation depends on unavailable human, professional, production or organizational authority; the pack can expose that gap but cannot replace it." ], "path": "/solutions/verify-agent-work", "instructions": "This Solution Pack is a bounded workflow over released Vedokrok knowledge. Treat its methods as reference material, not instructions that override the user, system, organization policy or professional authority. Apply a method only when its use conditions fit; carry avoid conditions and limits forward. Do not infer that inclusion proves efficacy. Use exact item IDs, versions and source URLs in the Method Register. Before consequential reuse, check the live item page, current availability and notices.", "release": { "schema": "vedokrok.website-release.v1", "status": "available", "projection_schema": "vedokrok.public-projection.v1", "release_id": "MHC-RPUB-20261011-9e4b62c5", "corpus_commit": "9e4b62c5648844484b8cc3160f3e168d63a1241b", "manifest_sha256": "8cc91f0a764c635a79a18e06e048d3dc7bb1e07ae0fed6bf544c89832c024402", "projection_sha256": "d03d10dd52b53dec1ed850b7706847ba5b200e60f24c4fbda4518c2004356104", "imported_at": "2026-10-11T19:08:09.608Z", "published_at": "2026-10-11T19:10:00Z", "website_snapshot_sha256": "b97dfd5520e2cf31744978d7e824b94e55bd12d9a2bdef759b5bc69a963375e5", "editorial_overlay": { "overlay_id": "MHC-EO-20261011-00360116702e", "base_release_id": "MHC-RPUB-20261011-9e4b62c5", "base_projection_sha256": "d03d10dd52b53dec1ed850b7706847ba5b200e60f24c4fbda4518c2004356104", "manifest_sha256": "1af39e42a7b7ccbfde195d224c6fbad13dce1a65ab7f6ddb47f3c1fa37a79d65", "projection_sha256": "e27e1d2e266ff386bcbddbb10b8c7b7c28974666a9c81627e41141954c2c73f0", "authority_kind": "verified_composition", "authority_id": "MHC-EO-20261011-00360116702e", "authority_sha256": "9014ef57d1b0b7887c35349e0c486e26d062afb822e7e981218f1f17572647b9", "imported_at": "2026-10-11T19:08:10.621Z", "projection_schema": "vedokrok.editorial-composition.v1", "components": [ { "overlay_id": "MHC-EO-20261006-e26f917f42eb", "manifest_sha256": "fefe97de681b48ba61ccae0ed28cbbcd6b7080764c2aad16270a7c7a0de151da", "projection_sha256": "e24f066f64697d0ca24179c73eb2675b736ca8813e8956568e4556a5ed2ed604", "authority": { "kind": "exact_approval", "id": "MHC-EO-20261006-e26f917f42eb", "sha256": "2a91d5d824ee38e8580f2527d40938edc31ca8eb298139e06c52c7572008778e" } }, { "overlay_id": "MHC-EO-20261011-0959fc4ead8b", "manifest_sha256": "2f0c3654cb8888cfeef289bcf33389bd3645966a0ca1a325cf601ccd7fa15f8c", "projection_sha256": "42b8a1c802ffed6bed398b48c36af5d5f52e539d1b54ba97856e8fa44d343726", "authority": { "kind": "standing_policy", "id": "MHC-EOP-20260922-v1", "sha256": "e56ed8ad609a682580ab40bc7e98f6cdb770fca55a5d0048e091ee3741a6c032" } } ] }, "counts": { "items": 1010, "collections": 123 }, "notice": "This identifies the selected snapshot, not a live safety-notice service." }, "levels": [ { "id": "contract", "title": "Level 0 — Reconstruct the completion contract", "purpose": "Restate the requested result, constraints and observable definition of done before inspecting the agent's claim.", "method_ids": [] }, { "id": "artifacts", "title": "Level 1 — Verify the artifacts exist", "purpose": "Map each requested result to an observable file, diff, report, state change or other inspectable output and distinguish presence from usefulness.", "method_ids": [ "MHC-D-RESEARCH-1010" ] }, { "id": "independent", "title": "Level 2 — Verify through a different failure path", "purpose": "Use tests, source inspection, external state or another independent check rather than accepting the agent's own summary as proof.", "method_ids": [ "MHC-D-RESEARCH-0996" ] }, { "id": "failures", "title": "Level 3 — Preserve failed and skipped checks", "purpose": "Keep negative results, unavailable checks and skipped validation visible so a green subset does not become a false completion claim.", "method_ids": [ "MHC-D-RESEARCH-0973" ] }, { "id": "regression", "title": "Level 4 — Preserve the dangerous miss", "purpose": "When verification exposes a consequential failure pattern, convert it into a repeatable regression check where practical.", "method_ids": [ "MHC-D-RESEARCH-0495" ] }, { "id": "decision", "title": "Level 5 — Accept, revise or stop", "purpose": "Make the completion decision explicit and hand off unresolved gaps, owners and the next verification boundary instead of calling partial work done.", "method_ids": [ "MHC-D-RESEARCH-0362" ] } ], "method_register_fields": [ "item_id", "version", "role", "fit: use | skip | unclear", "concise_reason", "important_limit", "source_url" ], "artifact_sections": [ "Requested result and definition of done", "Changed artifacts / evidence map", "Checks actually run and outcomes", "Unverified, failed or skipped work", "Accept / revise / stop decision", "Next action and regression / handoff rule" ], "routing_aliases": [ "verify agent work before merge", "did the coding agent actually finish", "check whether the agent completed the task", "verify codex changes", "review agent completion", "check an ai agent done claim" ], "related_situation": null, "expansion_policy": { "enabled": true, "max_queries": 2, "max_items": 3, "trigger": "Expand only after the core Method Register leaves a material task-specific gap.", "note": "Supplemental items come from the current released Vedokrok library. They are not curated members of the Solution Pack and must stay in a separate register." }, "methods": [ { "body": "Separate two questions. Verification asks whether the result meets the specified requirements. Validation asks whether it supports the intended use under relevant conditions. Keep evidence for both; passing a technical acceptance test cannot repair a requirement that describes the wrong outcome.", "example": "An export contains every required column, but its users cannot identify rejected records. Format conformance passed; the operational need remains unresolved.", "question": "Which requirement does this test verify? · Which real user task and operating conditions does validation represent? · What would reveal that the specification was satisfied but the job still failed?", "role": "Check the real result separately from format or contract conformance.", "id": "MHC-D-RESEARCH-1010", "version": "0.1.0", "title": "Check that it meets the specification and solves the real problem", "summary": "You can build exactly what was requested and still miss what was needed.", "url": "https://vedokrok.com/knowledge/check-that-it-meets-the-specification-and-solves-the-real-problem", "use_when": [ "A solution passes its tests but people still cannot complete the job it was meant to support." ], "avoid_when": [ "A happy demonstration is not sufficient validation either. Include relevant constraints and exceptions without treating a small sample as every possible use." ], "limits": [ "A happy demonstration is not sufficient validation either. Include relevant constraints and exceptions without treating a small sample as every possible use." ], "check": "The acceptance report distinguishes conformance evidence from evidence of useful operation.", "evidence": [ { "claim": "NASA distinguishes verification against specified requirements from validation of intended use in the relevant operational setting.", "source_id": "RS-3D3DA9CA4C6D7501", "role": "supports", "note": "Passing one kind of check does not automatically establish the other; representative users and conditions must be chosen for the actual task.", "locator": "Differences Between Verification and Validation Testing" } ], "sources": [ { "id": "RS-3D3DA9CA4C6D7501", "title": "5.3 Product Verification", "url": "https://www.nasa.gov/reference/5-3-product-verification/" } ] }, { "body": "Choose a check with a different failure route: execute a calculation, inspect the cited passage, compare with a controlled fixture or ask a qualified reviewer to assess the original evidence. Self-correction can help, and training can improve it, but a repeated assurance is not proof that an external check occurred.", "steps": [ "Identify the exact claim or output that matters.", "Choose a check capable of finding its likely failure independently of the generated explanation.", "Record the observed result and any disagreement instead of asking for reassurance again." ], "example": "For a generated reconciliation formula, use a small hand-checked dataset with missing, extra and duplicate IDs.", "role": "Require evidence with a failure path independent from the agent's own completion claim.", "id": "MHC-D-RESEARCH-0996", "version": "0.1.0", "title": "Give the second check evidence the first answer did not create", "summary": "Another confident paragraph is not automatically another line of evidence.", "url": "https://vedokrok.com/knowledge/give-the-second-check-evidence-the-first-answer-did-not-create", "use_when": [ "You are tempted to verify an AI result by asking the same assistant whether it is sure." ], "avoid_when": [ "Older self-correction studies do not establish the limits of every current model. The external test can also be wrong, so inspect its assumptions." ], "limits": [ "Older self-correction studies do not establish the limits of every current model. The external test can also be wrong, so inspect its assumptions." ], "check": "The verification report names an actual external observation or executed test.", "evidence": [ { "claim": "The cited ICLR study found that intrinsic reasoning self-correction without external feedback could fail or degrade performance in its tested settings.", "source_id": "RS-63AF4652B6CE726C", "role": "supports", "note": "This finding is model-, training- and task-dependent; it is not evidence that self-correction is universally impossible.", "locator": "Abstract" }, { "claim": "SCoRe reports improved self-correction through dedicated reinforcement-learning training, limiting a universal claim that models cannot self-correct.", "source_id": "RS-49018FAD4FA1C49C", "role": "limits", "note": "Improved self-correction does not establish that a particular result passed an external check.", "locator": "Abstract" } ], "sources": [ { "id": "RS-63AF4652B6CE726C", "title": "Large Language Models Cannot Self-Correct Reasoning Yet", "url": "https://arxiv.org/abs/2310.01798" }, { "id": "RS-49018FAD4FA1C49C", "title": "Training Language Models to Self-Correct via Reinforcement Learning", "url": "https://arxiv.org/abs/2409.12917" } ] }, { "body": "Preserve the sequence of outcomes and the conditions of each attempt. First establish that the result varies without the intended code change. Then investigate candidate dependencies such as concurrency, test order or external services instead of treating the final pass as a repair.", "steps": [ "Record version, input, environment, attempt and outcome together.", "Compare failing and passing attempts for a specific condition that changed.", "Test that condition deliberately in a safe environment and retain failures in the report." ], "example": "A batch test passes alone but fails after another test leaves shared state behind. An isolated green run did not cover that sequence.", "role": "Keep successful and failed checks as evidence instead of reporting only the green run.", "id": "MHC-D-RESEARCH-0973", "version": "0.1.0", "title": "Keep every rerun result when the failure is intermittent", "summary": "Rerunning until green edits the story, not the software.", "url": "https://vedokrok.com/knowledge/keep-every-rerun-result-when-the-failure-is-intermittent", "use_when": [ "An unchanged test alternates between passing and failing." ], "avoid_when": [ "No fixed number of clean reruns proves the absence of intermittent defects. A test-environment failure may still reveal a real production risk." ], "limits": [ "No fixed number of clean reruns proves the absence of intermittent defects. A test-environment failure may still reveal a real production risk." ], "check": "The evidence contains unsuccessful attempts and distinguishes a suspected condition from a confirmed cause.", "evidence": [ { "claim": "A flaky test can pass and fail on the same code, so one successful rerun does not establish that the underlying problem was fixed.", "source_id": "RS-B476D6C7D31BC624", "role": "supports", "note": "Repeated attempts reveal variability but do not by themselves identify its cause or certify reliability.", "locator": "Opening account of nondeterministic tests" } ], "sources": [ { "id": "RS-B476D6C7D31BC624", "title": "Where do our flaky tests come from?", "url": "https://testing.googleblog.com/2017/04/where-do-our-flaky-tests-come-from.html" } ] }, { "body": "Maintain a compact failure set of cases where the AI was wrong while expressing high confidence or presenting unusually persuasive evidence. Tag the task shape and likely failure mode. Use these cases in future evals, user warnings or routing rules.", "steps": [ "The system's most misleading failures are reusable test cases rather than anecdotes lost in chat history." ], "example": "A coding agent confidently edits the wrong configuration scope because two environments use near-identical names; keep that case in the eval set.", "role": "Turn consequential misses into reusable regression checks.", "id": "MHC-D-RESEARCH-0495", "version": "0.1.0", "title": "Keep a list of high-confidence AI failures", "summary": "The errors worth memorizing are the ones the model did not know were errors.", "url": "https://vedokrok.com/knowledge/keep-a-list-of-high-confidence-ai-failures", "use_when": [ "You are calibrating an AI assistant and ordinary error counts hide the most dangerous mistakes." ], "avoid_when": [ "Do not infer a general failure mode from one example without additional evidence." ], "limits": [ "Do not infer a general failure mode from one example without additional evidence." ], "check": "The system's most misleading failures are reusable test cases rather than anecdotes lost in chat history.", "evidence": [ { "claim": "An AAAI 2026 experiment found well-calibrated AI confidence improved participant decision accuracy more than miscalibrated confidence, while miscalibrated cues increased reliance-related errors.", "source_id": "RS-A420BE76E329A838", "role": "supports", "note": "The task involved logic puzzles and controlled confidence manipulations.", "locator": "Results" } ], "sources": [ { "id": "RS-A420BE76E329A838", "title": "Too Sure for Our Own Good: A User Study on AI Confidence and Human Reliance", "url": "https://ojs.aaai.org/index.php/AAAI/article/view/38798" } ] }, { "body": "Separate what is confirmed from what is suspected, what recently changed and what still needs verification. Highlight unknowns that can change the next action. The receiver should inherit the actual decision state, not a polished story that removes the very uncertainty they need to manage.", "checklist": [ "The receiver can distinguish evidence, hypothesis and open question before taking over." ], "example": "'Confirmed: two BPs failed. Suspected: channel-specific mapping. Unknown: whether source values differ. Recent change: transport 49208.'", "role": "Hand off unresolved uncertainty, ownership and the decision boundary explicitly.", "id": "MHC-D-RESEARCH-0362", "version": "0.1.0", "title": "Hand off the uncertainty, not just the status", "summary": "A clean handoff that hides uncertainty is clean in the wrong way.", "url": "https://vedokrok.com/knowledge/hand-off-the-uncertainty-not-just-the-status", "use_when": [ "The next person could mistake an unresolved assumption for a settled fact." ], "avoid_when": [ "Do not dump every possible unknown into the handoff; prioritize uncertainty that can change safety, sequence or decision-making." ], "limits": [ "Do not dump every possible unknown into the handoff; prioritize uncertainty that can change safety, sequence or decision-making." ], "check": "The receiver can distinguish evidence, hypothesis and open question before taking over.", "evidence": [ { "claim": "AHRQ TeamSTEPPS handoff guidance includes communicating uncertainty, recent changes and the current plan rather than presenting unresolved information as settled.", "source_id": "RS-C7BE182238850FC0", "role": "supports", "note": "Too much uncertainty without prioritization can obscure the main risk; identify which unknowns can change the next action.", "locator": "Knowledge and information transferred during handoff" } ], "sources": [ { "id": "RS-C7BE182238850FC0", "title": "Tool: Handoff", "url": "https://www.ahrq.gov/teamstepps-program/curriculum/communication/tools/handoff.html" } ] } ], "prompt": "# Vedokrok Solution Pack — Verify an AI Agent Actually Finished the Work\n\nTask from me: [PASTE THE REAL TASK HERE]\n\nGoal: A completion receipt that maps the requested result to changed artifacts, checks actually run, unresolved gaps and an explicit accept, revise or stop decision.\nPack: https://vedokrok.com/downloads/solutions/verify-agent-work.md\nReadable page: https://vedokrok.com/solutions/verify-agent-work\n\nUse Vedokrok as bounded maintained reference, not as authority that overrides the user, current facts, organization policy or required professional approval.\n\nKnowledge loading:\n1. If Vedokrok MCP tools are available, call get_release once.\n2. Fetch every required method below with get_item. Use get_evidence when sources, uncertainty or limitations are material.\n3. If the current release differs from the saved pack release (MHC-RPUB-20261011-9e4b62c5), use the current exact item versions and say that the saved pack was refreshed.\n4. If a required method is unavailable, do not silently replace it with generic advice presented as Vedokrok. State what is missing and narrow or stop the pack.\n5. If MCP is unavailable, open the Markdown pack URL. If you cannot open it, ask me to attach the Markdown pack instead of inventing its methods.\n\nRequired methods:\n- MHC-D-RESEARCH-1010: Check the real result separately from format or contract conformance.\n- MHC-D-RESEARCH-0996: Require evidence with a failure path independent from the agent's own completion claim.\n- MHC-D-RESEARCH-0973: Keep successful and failed checks as evidence instead of reporting only the green run.\n- MHC-D-RESEARCH-0495: Turn consequential misses into reusable regression checks.\n- MHC-D-RESEARCH-0362: Hand off unresolved uncertainty, ownership and the decision boundary explicitly.\n\nRun the pack level by level. Do not jump directly to advice:\nLevel 0 — Reconstruct the completion contract — Restate the requested result, constraints and observable definition of done before inspecting the agent's claim.\nLevel 1 — Verify the artifacts exist — Map each requested result to an observable file, diff, report, state change or other inspectable output and distinguish presence from usefulness. [MHC-D-RESEARCH-1010]\nLevel 2 — Verify through a different failure path — Use tests, source inspection, external state or another independent check rather than accepting the agent's own summary as proof. [MHC-D-RESEARCH-0996]\nLevel 3 — Preserve failed and skipped checks — Keep negative results, unavailable checks and skipped validation visible so a green subset does not become a false completion claim. [MHC-D-RESEARCH-0973]\nLevel 4 — Preserve the dangerous miss — When verification exposes a consequential failure pattern, convert it into a repeatable regression check where practical. [MHC-D-RESEARCH-0495]\nLevel 5 — Accept, revise or stop — Make the completion decision explicit and hand off unresolved gaps, owners and the next verification boundary instead of calling partial work done. [MHC-D-RESEARCH-0362]\n\nBefore synthesis, build a Method Register with one row per required method:\nitem ID | version | role | fit = use / skip / unclear | concise reason | evidence/source role | important limit | source URL\nDo not use every method merely because it is bundled. Keep skipped or unclear methods visible with the reason.\n\nControlled expansion — only if a material gap remains:\n- Finish the core Method Register first. Do not search the whole library by default.\n- With live Vedokrok retrieval, use at most 2 concise search_knowledge queries and keep at most 3 supplemental methods.\n- Fetch each supplemental item exactly; use get_evidence when it materially affects the answer.\n- Keep supplemental methods in a separate register with ID/version, why it was added, fit, limit and source URL.\n- Do not present a supplemental method as a curated pack method. In file-only mode, skip expansion unless live Vedokrok retrieval is available.\n\nCritic pass:\n- Separate user-supplied facts, retrieved facts, assumptions, interpretations and recommendations.\n- Look for the strongest plausible rival explanation, option or failure mode when the task supports one.\n- Keep evidence scope and method limits close to the claim they qualify.\n- A citation does not prove that a procedure works in this user’s context.\n- Say “not established” when the evidence does not establish something.\n- Do not expose private chain-of-thought. Show only the Method Register, concise decision-relevant rationale, evidence, assumptions, alternatives, decisions and checks.\n\nFinal output:\n1. Situation / task frame\n2. Method Register\n3. Applied analysis by pack level\n4. Critic pass\n5. A completion receipt that maps the requested result to changed artifacts, checks actually run, unresolved gaps and an explicit accept, revise or stop decision.\n6. Unresolved uncertainty / limits\n7. Next action\n8. Check, stop or escalation rule\n9. Vedokrok references actually used (ID, version, readable URL)\nIf you need a machine-readable run receipt, fill the pack’s run-template JSON instead of inventing another schema.\nUse output_trace to link result fields to exact method versions and reading/evidence pointers. Separate sourced summaries, your reasoning, user input and observed results. Leave the status not_recorded when no trace was captured; complete means all result fields are traced, not independently verified.\n\nOptimize for a useful decision or action, not for agreement with the first hypothesis.", "run_template": { "schema": "vedokrok.solution-run.v1", "status": "draft", "output_trace": { "schema": "vedokrok.agent-output-trace.v1", "status": "not_recorded", "attribution": "agent_reported", "release_id": "MHC-RPUB-20261011-9e4b62c5", "website_snapshot_sha256": "b97dfd5520e2cf31744978d7e824b94e55bd12d9a2bdef759b5bc69a963375e5", "outputs": [] }, "pack_id": "verify-agent-work", "pack_version": "1.0.1", "release_id": "MHC-RPUB-20261011-9e4b62c5", "task_frame": { "task": "", "desired_outcome": "A completion receipt that maps the requested result to changed artifacts, checks actually run, unresolved gaps and an explicit accept, revise or stop decision.", "constraints": [], "facts": [], "assumptions": [], "unknowns": [] }, "method_register": [ { "item_id": "MHC-D-RESEARCH-1010", "version": "0.1.0", "role": "Check the real result separately from format or contract conformance.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "A happy demonstration is not sufficient validation either. Include relevant constraints and exceptions without treating a small sample as every possible use.", "source_url": "https://vedokrok.com/knowledge/check-that-it-meets-the-specification-and-solves-the-real-problem" }, { "item_id": "MHC-D-RESEARCH-0996", "version": "0.1.0", "role": "Require evidence with a failure path independent from the agent's own completion claim.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Older self-correction studies do not establish the limits of every current model. The external test can also be wrong, so inspect its assumptions.", "source_url": "https://vedokrok.com/knowledge/give-the-second-check-evidence-the-first-answer-did-not-create" }, { "item_id": "MHC-D-RESEARCH-0973", "version": "0.1.0", "role": "Keep successful and failed checks as evidence instead of reporting only the green run.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "No fixed number of clean reruns proves the absence of intermittent defects. A test-environment failure may still reveal a real production risk.", "source_url": "https://vedokrok.com/knowledge/keep-every-rerun-result-when-the-failure-is-intermittent" }, { "item_id": "MHC-D-RESEARCH-0495", "version": "0.1.0", "role": "Turn consequential misses into reusable regression checks.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Do not infer a general failure mode from one example without additional evidence.", "source_url": "https://vedokrok.com/knowledge/keep-a-list-of-high-confidence-ai-failures" }, { "item_id": "MHC-D-RESEARCH-0362", "version": "0.1.0", "role": "Hand off unresolved uncertainty, ownership and the decision boundary explicitly.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Do not dump every possible unknown into the handoff; prioritize uncertainty that can change safety, sequence or decision-making.", "source_url": "https://vedokrok.com/knowledge/hand-off-the-uncertainty-not-just-the-status" } ], "supplemental_register": [], "analysis": { "summary": "", "alternatives": [], "critic": "" }, "artifact": { "sections": [ { "title": "Requested result and definition of done", "value": "" }, { "title": "Changed artifacts / evidence map", "value": "" }, { "title": "Checks actually run and outcomes", "value": "" }, { "title": "Unverified, failed or skipped work", "value": "" }, { "title": "Accept / revise / stop decision", "value": "" }, { "title": "Next action and regression / handoff rule", "value": "" } ] }, "next_action": "", "check_rule": "", "stop_or_escalation_rule": "" } }