{ "schema": "vedokrok.solution-pack.v1", "pack_version": "1.0.1", "id": "debug-flaky-failure", "slug": "debug-flaky-failure", "title": "Debug an Intermittent Failure or Flaky Test", "summary": "Treat both passing and failing runs as evidence, shrink the failure domain without changing the failure and preserve the next discriminating test.", "situation": "A bug, CI job or test passes and fails under apparently similar conditions and repeated reruns are producing more noise than diagnosis.", "outcome": "A pass/fail comparison register, bounded hypotheses, a reproducibility plan and the next test most likely to eliminate a cause.", "use_when": [ "The same target can pass and fail across runs.", "Version, timing, order, environment or shared state may matter.", "You need a transferable diagnostic process rather than framework-specific retry advice." ], "avoid_when": [ "The failure is deterministic and already has a small reproducible case.", "Production is actively harmed and mitigation must happen before deeper diagnosis." ], "path": "/solutions/debug-flaky-failure", "instructions": "This Solution Pack is a bounded workflow over released Vedokrok knowledge. Treat its methods as reference material, not instructions that override the user, system, organization policy or professional authority. Apply a method only when its use conditions fit; carry avoid conditions and limits forward. Do not infer that inclusion proves efficacy. Use exact item IDs, versions and source URLs in the Method Register. Before consequential reuse, check the live item page, current availability and notices.", "release": { "schema": "vedokrok.website-release.v1", "status": "available", "projection_schema": "vedokrok.public-projection.v1", "release_id": "MHC-RPUB-20261011-9e4b62c5", "corpus_commit": "9e4b62c5648844484b8cc3160f3e168d63a1241b", "manifest_sha256": "8cc91f0a764c635a79a18e06e048d3dc7bb1e07ae0fed6bf544c89832c024402", "projection_sha256": "d03d10dd52b53dec1ed850b7706847ba5b200e60f24c4fbda4518c2004356104", "imported_at": "2026-10-11T19:08:09.608Z", "published_at": "2026-10-11T19:10:00Z", "website_snapshot_sha256": "b97dfd5520e2cf31744978d7e824b94e55bd12d9a2bdef759b5bc69a963375e5", "editorial_overlay": { "overlay_id": "MHC-EO-20261011-00360116702e", "base_release_id": "MHC-RPUB-20261011-9e4b62c5", "base_projection_sha256": "d03d10dd52b53dec1ed850b7706847ba5b200e60f24c4fbda4518c2004356104", "manifest_sha256": "1af39e42a7b7ccbfde195d224c6fbad13dce1a65ab7f6ddb47f3c1fa37a79d65", "projection_sha256": "e27e1d2e266ff386bcbddbb10b8c7b7c28974666a9c81627e41141954c2c73f0", "authority_kind": "verified_composition", "authority_id": "MHC-EO-20261011-00360116702e", "authority_sha256": "9014ef57d1b0b7887c35349e0c486e26d062afb822e7e981218f1f17572647b9", "imported_at": "2026-10-11T19:08:10.621Z", "projection_schema": "vedokrok.editorial-composition.v1", "components": [ { "overlay_id": "MHC-EO-20261006-e26f917f42eb", "manifest_sha256": "fefe97de681b48ba61ccae0ed28cbbcd6b7080764c2aad16270a7c7a0de151da", "projection_sha256": "e24f066f64697d0ca24179c73eb2675b736ca8813e8956568e4556a5ed2ed604", "authority": { "kind": "exact_approval", "id": "MHC-EO-20261006-e26f917f42eb", "sha256": "2a91d5d824ee38e8580f2527d40938edc31ca8eb298139e06c52c7572008778e" } }, { "overlay_id": "MHC-EO-20261011-0959fc4ead8b", "manifest_sha256": "2f0c3654cb8888cfeef289bcf33389bd3645966a0ca1a325cf601ccd7fa15f8c", "projection_sha256": "42b8a1c802ffed6bed398b48c36af5d5f52e539d1b54ba97856e8fa44d343726", "authority": { "kind": "standing_policy", "id": "MHC-EOP-20260922-v1", "sha256": "e56ed8ad609a682580ab40bc7e98f6cdb770fca55a5d0048e091ee3741a6c032" } } ] }, "counts": { "items": 1010, "collections": 123 }, "notice": "This identifies the selected snapshot, not a live safety-notice service." }, "levels": [ { "id": "signature", "title": "Level 0 — Freeze the failure signature", "purpose": "Define what counts as the same failure before changing the system.", "method_ids": [ "MHC-D-RESEARCH-0973" ] }, { "id": "comparison", "title": "Level 1 — Compare red and green runs", "purpose": "Record observable conditions that differ across the nearest comparable pass and fail.", "method_ids": [ "MHC-D-RESEARCH-0973" ] }, { "id": "hypotheses", "title": "Level 2 — Keep competing causes", "purpose": "Maintain more than one plausible explanation and identify predictions that differ.", "method_ids": [ "MHC-D-RESEARCH-0295" ] }, { "id": "reduce", "title": "Level 3 — Reduce the search space", "purpose": "Use bisection only when the candidate space has a valid testable order.", "method_ids": [ "MHC-D-RESEARCH-0295", "MHC-D-RESEARCH-0971" ] }, { "id": "boundaries", "title": "Level 4 — Inspect semantic boundaries", "purpose": "Capture the point where two components may disagree about the same data.", "method_ids": [ "MHC-D-RESEARCH-1159" ] }, { "id": "action", "title": "Level 5 — Mitigate, test or continue", "purpose": "If impact is active, mitigate without destroying evidence; otherwise run the next discriminating test.", "method_ids": [ "MHC-D-RESEARCH-0300" ] } ], "method_register_fields": [ "item_id", "version", "role", "fit: use | skip | unclear", "concise_reason", "important_limit", "source_url" ], "artifact_sections": [ "Failure signature", "Pass/fail comparison", "Method register", "Current and rival hypotheses", "Next discriminating test", "Mitigation / stop condition" ], "routing_aliases": [], "related_situation": null, "expansion_policy": { "enabled": true, "max_queries": 2, "max_items": 3, "trigger": "Expand only after the core Method Register leaves a material task-specific gap.", "note": "Supplemental items come from the current released Vedokrok library. They are not curated members of the Solution Pack and must stay in a separate register." }, "methods": [ { "body": "Preserve the sequence of outcomes and the conditions of each attempt. First establish that the result varies without the intended code change. Then investigate candidate dependencies such as concurrency, test order or external services instead of treating the final pass as a repair.", "steps": [ "Record version, input, environment, attempt and outcome together.", "Compare failing and passing attempts for a specific condition that changed.", "Test that condition deliberately in a safe environment and retain failures in the report." ], "example": "A batch test passes alone but fails after another test leaves shared state behind. An isolated green run did not cover that sequence.", "role": "Keep every rerun result and compare failures with successes.", "id": "MHC-D-RESEARCH-0973", "version": "0.1.0", "title": "Keep every rerun result when the failure is intermittent", "summary": "Rerunning until green edits the story, not the software.", "url": "https://vedokrok.com/knowledge/keep-every-rerun-result-when-the-failure-is-intermittent", "use_when": [ "An unchanged test alternates between passing and failing." ], "avoid_when": [ "No fixed number of clean reruns proves the absence of intermittent defects. A test-environment failure may still reveal a real production risk." ], "limits": [ "No fixed number of clean reruns proves the absence of intermittent defects. A test-environment failure may still reveal a real production risk." ], "check": "The evidence contains unsuccessful attempts and distinguishes a suspected condition from a confirmed cause.", "evidence": [ { "claim": "A flaky test can pass and fail on the same code, so one successful rerun does not establish that the underlying problem was fixed.", "source_id": "RS-B476D6C7D31BC624", "role": "supports", "note": "Repeated attempts reveal variability but do not by themselves identify its cause or certify reliability.", "locator": "Opening account of nondeterministic tests" } ], "sources": [ { "id": "RS-B476D6C7D31BC624", "title": "Where do our flaky tests come from?", "url": "https://testing.googleblog.com/2017/04/where-do-our-flaky-tests-come-from.html" } ] }, { "body": "Split the search space into two meaningful parts and run a test that tells you which side still contains the changed behavior. Keep the implicated half and split again. This works especially well for revision history, layered systems and ordered pipelines when the state can be classified consistently.", "example": "If a regression appeared somewhere between two releases, binary-search the revision history instead of reading every commit.", "role": "Bisect a valid ordered failure domain.", "id": "MHC-D-RESEARCH-0295", "version": "0.1.0", "title": "Bisect a large failure domain", "summary": "When the haystack has structure, stop inspecting every piece of hay.", "url": "https://vedokrok.com/knowledge/bisect-a-large-failure-domain", "use_when": [ "The searchable space is ordered or divisible and checking every candidate one by one would be expensive." ], "avoid_when": [ "Do not use bisection when candidates interact non-monotonically, the midpoint cannot be tested or the failure classification is unstable." ], "limits": [ "Do not use bisection when candidates interact non-monotonically, the midpoint cannot be tested or the failure classification is unstable." ], "check": "Each round removes a substantial part of the search space without changing the property you are testing.", "evidence": [ { "claim": "Google SRE describes bisection as repeatedly splitting a large system in half to narrow a possibly faulty component; Git bisect applies binary search to ordered revision history.", "source_id": "RS-A4A16C807A94C9D8", "role": "supports", "note": "Bisection needs a meaningful ordered search space and a reasonably testable distinction between the two states.", "locator": "Simplify and reduce; bisection" } ], "sources": [ { "id": "RS-A4A16C807A94C9D8", "title": "Effective Troubleshooting", "url": "https://sre.google/sre-book/effective-troubleshooting/" } ] }, { "body": "Choose a reproducibly passing version and a failing version. Test a version between them, then keep the half containing the transition. Continue until the boundary is narrow enough to inspect. Git bisect automates this search for code history.", "steps": [ "Use the same input, test and relevant environment across versions.", "Record untestable versions as unknown, not as failures.", "Confirm the candidate boundary with a fresh run and inspect the actual change." ], "example": "A mapping worked in release A and breaks in H. Testing D and then F narrows the investigation without reading every transport first.", "role": "Bisect version history when a known-good and known-bad boundary exists.", "id": "MHC-D-RESEARCH-0971", "version": "0.1.0", "title": "Bisect the version history instead of inspecting every change", "summary": "The last edit is a suspect, not automatically the culprit.", "url": "https://vedokrok.com/knowledge/bisect-the-version-history-instead-of-inspecting-every-change", "use_when": [ "The same test passed in an earlier version and fails now." ], "avoid_when": [ "Repeated introduction and removal of a bug, changing dependencies or flaky tests can invalidate a simple one-transition story." ], "limits": [ "Repeated introduction and removal of a bug, changing dependencies or flaky tests can invalidate a simple one-transition story." ], "check": "The report names the tested boundary and any skipped versions that prevent a precise conclusion.", "evidence": [ { "claim": "Git bisect narrows a known-good to known-bad version interval through repeated tests; skipped versions can leave an ambiguous boundary.", "source_id": "RS-BB2489EEC3B17FAA", "role": "supports", "note": "A first failing version is an investigation lead, not proof of a complete root cause.", "locator": "Description; Bisect skip" } ], "sources": [ { "id": "RS-BB2489EEC3B17FAA", "title": "git-bisect: Use binary search to find the commit that introduced a bug", "url": "https://git-scm.com/docs/git-bisect" } ] }, { "body": "Record the inputs and outputs at the model and tool boundaries that matter to the task: model request and response identifiers, parsed arguments, tool results, policy decisions and the state carried into the next step. This creates an execution envelope that can replay the bad decision against repaired code.", "checklist": [ "Capture the model decision that selected the action.", "Capture the exact parsed tool arguments and returned result.", "Record policy or validation decisions around the call.", "Keep enough state to replay the boundary without storing unrelated sensitive data." ], "example": "A trading assistant interpreted dollars as shares. The useful trace preserves the model's action, parsed amount, tool contract and validation result at the order boundary.", "role": "Capture semantic boundaries where representation or meaning changes.", "id": "MHC-D-RESEARCH-1159", "version": "0.1.0", "title": "Capture semantic boundaries, not only raw logs", "summary": "A log can be complete and still omit the boundary where meaning changed.", "url": "https://vedokrok.com/knowledge/capture-semantic-boundaries-not-only-raw-logs", "use_when": [ "A production agent failure disappears when developers try to reproduce it locally." ], "avoid_when": [ "Do not turn observability into indiscriminate data retention. Minimize personal data, secrets and irrelevant payloads." ], "limits": [ "Do not turn observability into indiscriminate data retention. Minimize personal data, secrets and irrelevant payloads." ], "check": "A developer can replay the consequential boundary without inventing missing inputs or re-prompting the model for a similar decision.", "evidence": [ { "claim": "Recording semantic model and tool boundaries can preserve the decision context needed to replay a production failure against repaired enforcement code.", "source_id": "RS-F0FB27D3E1B6C4A1", "role": "supports", "note": "Capturing more data is not automatically useful or safe; retain only the boundaries needed for diagnosis while respecting privacy and secret-handling requirements.", "locator": "4:51-13:14, recover the run, record semantic boundaries, replay the bad decision and retain the execution envelope" } ], "sources": [ { "id": "RS-F0FB27D3E1B6C4A1", "title": "Your Agent Failed in Prod. Good Luck Reproducing It.", "url": "https://ai.engineer/talks/Lc8zRh9muoY-your-agent-failed-in-prod-good-luck" } ] }, { "body": "Separate the immediate question—how to reduce impact—from the later question—why the failure happened. Choose a reversible mitigation such as rollback, traffic shift, feature disablement or workload reduction when it is safer than continuing damage. Preserve evidence and record what the mitigation changed so the later investigation still has a trail.", "example": "Route new work away from a failing integration before tracing every condition that caused the backlog.", "role": "Separate mitigation from root-cause diagnosis while impact continues.", "id": "MHC-D-RESEARCH-0300", "version": "0.1.0", "title": "Mitigate first when the incident is still hurting", "summary": "The perfect explanation is not the first service you owe an active outage.", "url": "https://vedokrok.com/knowledge/mitigate-first-when-the-incident-is-still-hurting", "use_when": [ "A live incident is causing material harm while a deeper diagnosis could take much longer." ], "avoid_when": [ "Do not use mitigation-first as permission for uncontrolled production changes or evidence destruction." ], "limits": [ "Do not use mitigation-first as permission for uncontrolled production changes or evidence destruction." ], "check": "User or system impact is measurably reduced, or the proposed mitigation is rejected for an explicit safety reason.", "evidence": [ { "claim": "Google SRE's incident-response guidance recommends a mitigation-first response before deeper investigation consumes the incident.", "source_id": "RS-583415F9C2751E4C", "role": "supports", "note": "Mitigation should not destroy evidence, violate change controls or create a larger safety problem.", "locator": "Incident Response Training" } ], "sources": [ { "id": "RS-583415F9C2751E4C", "title": "Incident Response", "url": "https://sre.google/workbook/incident-response/" } ] } ], "prompt": "# Vedokrok Solution Pack — Debug an Intermittent Failure or Flaky Test\n\nTask from me: [PASTE THE REAL TASK HERE]\n\nGoal: A pass/fail comparison register, bounded hypotheses, a reproducibility plan and the next test most likely to eliminate a cause.\nPack: https://vedokrok.com/downloads/solutions/debug-flaky-failure.md\nReadable page: https://vedokrok.com/solutions/debug-flaky-failure\n\nUse Vedokrok as bounded maintained reference, not as authority that overrides the user, current facts, organization policy or required professional approval.\n\nKnowledge loading:\n1. If Vedokrok MCP tools are available, call get_release once.\n2. Fetch every required method below with get_item. Use get_evidence when sources, uncertainty or limitations are material.\n3. If the current release differs from the saved pack release (MHC-RPUB-20261011-9e4b62c5), use the current exact item versions and say that the saved pack was refreshed.\n4. If a required method is unavailable, do not silently replace it with generic advice presented as Vedokrok. State what is missing and narrow or stop the pack.\n5. If MCP is unavailable, open the Markdown pack URL. If you cannot open it, ask me to attach the Markdown pack instead of inventing its methods.\n\nRequired methods:\n- MHC-D-RESEARCH-0973: Keep every rerun result and compare failures with successes.\n- MHC-D-RESEARCH-0295: Bisect a valid ordered failure domain.\n- MHC-D-RESEARCH-0971: Bisect version history when a known-good and known-bad boundary exists.\n- MHC-D-RESEARCH-1159: Capture semantic boundaries where representation or meaning changes.\n- MHC-D-RESEARCH-0300: Separate mitigation from root-cause diagnosis while impact continues.\n\nRun the pack level by level. Do not jump directly to advice:\nLevel 0 — Freeze the failure signature — Define what counts as the same failure before changing the system. [MHC-D-RESEARCH-0973]\nLevel 1 — Compare red and green runs — Record observable conditions that differ across the nearest comparable pass and fail. [MHC-D-RESEARCH-0973]\nLevel 2 — Keep competing causes — Maintain more than one plausible explanation and identify predictions that differ. [MHC-D-RESEARCH-0295]\nLevel 3 — Reduce the search space — Use bisection only when the candidate space has a valid testable order. [MHC-D-RESEARCH-0295, MHC-D-RESEARCH-0971]\nLevel 4 — Inspect semantic boundaries — Capture the point where two components may disagree about the same data. [MHC-D-RESEARCH-1159]\nLevel 5 — Mitigate, test or continue — If impact is active, mitigate without destroying evidence; otherwise run the next discriminating test. [MHC-D-RESEARCH-0300]\n\nBefore synthesis, build a Method Register with one row per required method:\nitem ID | version | role | fit = use / skip / unclear | concise reason | evidence/source role | important limit | source URL\nDo not use every method merely because it is bundled. Keep skipped or unclear methods visible with the reason.\n\nControlled expansion — only if a material gap remains:\n- Finish the core Method Register first. Do not search the whole library by default.\n- With live Vedokrok retrieval, use at most 2 concise search_knowledge queries and keep at most 3 supplemental methods.\n- Fetch each supplemental item exactly; use get_evidence when it materially affects the answer.\n- Keep supplemental methods in a separate register with ID/version, why it was added, fit, limit and source URL.\n- Do not present a supplemental method as a curated pack method. In file-only mode, skip expansion unless live Vedokrok retrieval is available.\n\nCritic pass:\n- Separate user-supplied facts, retrieved facts, assumptions, interpretations and recommendations.\n- Look for the strongest plausible rival explanation, option or failure mode when the task supports one.\n- Keep evidence scope and method limits close to the claim they qualify.\n- A citation does not prove that a procedure works in this user’s context.\n- Say “not established” when the evidence does not establish something.\n- Do not expose private chain-of-thought. Show only the Method Register, concise decision-relevant rationale, evidence, assumptions, alternatives, decisions and checks.\n\nFinal output:\n1. Situation / task frame\n2. Method Register\n3. Applied analysis by pack level\n4. Critic pass\n5. A pass/fail comparison register, bounded hypotheses, a reproducibility plan and the next test most likely to eliminate a cause.\n6. Unresolved uncertainty / limits\n7. Next action\n8. Check, stop or escalation rule\n9. Vedokrok references actually used (ID, version, readable URL)\nIf you need a machine-readable run receipt, fill the pack’s run-template JSON instead of inventing another schema.\nUse output_trace to link result fields to exact method versions and reading/evidence pointers. Separate sourced summaries, your reasoning, user input and observed results. Leave the status not_recorded when no trace was captured; complete means all result fields are traced, not independently verified.\n\nOptimize for a useful decision or action, not for agreement with the first hypothesis.", "run_template": { "schema": "vedokrok.solution-run.v1", "status": "draft", "output_trace": { "schema": "vedokrok.agent-output-trace.v1", "status": "not_recorded", "attribution": "agent_reported", "release_id": "MHC-RPUB-20261011-9e4b62c5", "website_snapshot_sha256": "b97dfd5520e2cf31744978d7e824b94e55bd12d9a2bdef759b5bc69a963375e5", "outputs": [] }, "pack_id": "debug-flaky-failure", "pack_version": "1.0.1", "release_id": "MHC-RPUB-20261011-9e4b62c5", "task_frame": { "task": "", "desired_outcome": "A pass/fail comparison register, bounded hypotheses, a reproducibility plan and the next test most likely to eliminate a cause.", "constraints": [], "facts": [], "assumptions": [], "unknowns": [] }, "method_register": [ { "item_id": "MHC-D-RESEARCH-0973", "version": "0.1.0", "role": "Keep every rerun result and compare failures with successes.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "No fixed number of clean reruns proves the absence of intermittent defects. A test-environment failure may still reveal a real production risk.", "source_url": "https://vedokrok.com/knowledge/keep-every-rerun-result-when-the-failure-is-intermittent" }, { "item_id": "MHC-D-RESEARCH-0295", "version": "0.1.0", "role": "Bisect a valid ordered failure domain.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Do not use bisection when candidates interact non-monotonically, the midpoint cannot be tested or the failure classification is unstable.", "source_url": "https://vedokrok.com/knowledge/bisect-a-large-failure-domain" }, { "item_id": "MHC-D-RESEARCH-0971", "version": "0.1.0", "role": "Bisect version history when a known-good and known-bad boundary exists.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Repeated introduction and removal of a bug, changing dependencies or flaky tests can invalidate a simple one-transition story.", "source_url": "https://vedokrok.com/knowledge/bisect-the-version-history-instead-of-inspecting-every-change" }, { "item_id": "MHC-D-RESEARCH-1159", "version": "0.1.0", "role": "Capture semantic boundaries where representation or meaning changes.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Do not turn observability into indiscriminate data retention. Minimize personal data, secrets and irrelevant payloads.", "source_url": "https://vedokrok.com/knowledge/capture-semantic-boundaries-not-only-raw-logs" }, { "item_id": "MHC-D-RESEARCH-0300", "version": "0.1.0", "role": "Separate mitigation from root-cause diagnosis while impact continues.", "fit": "unclear", "reason": "", "evidence_or_source_role": "", "important_limit": "Do not use mitigation-first as permission for uncontrolled production changes or evidence destruction.", "source_url": "https://vedokrok.com/knowledge/mitigate-first-when-the-incident-is-still-hurting" } ], "supplemental_register": [], "analysis": { "summary": "", "alternatives": [], "critic": "" }, "artifact": { "sections": [ { "title": "Failure signature", "value": "" }, { "title": "Pass/fail comparison", "value": "" }, { "title": "Method register", "value": "" }, { "title": "Current and rival hypotheses", "value": "" }, { "title": "Next discriminating test", "value": "" }, { "title": "Mitigation / stop condition", "value": "" } ] }, "next_action": "", "check_rule": "", "stop_or_escalation_rule": "" } }