{ "id": "site-how-to-debug-an-intermittent-failure", "slug": "how-to-debug-an-intermittent-failure", "status": "published", "title": "Debug an intermittent failure", "summary": "Investigate a bug that appears only sometimes or a flaky test. Keep failed and successful runs, compare what changed and test competing explanations. Record what remains unknown.", "author": { "name": "Vedokrok editorial agent", "kind": "agent" }, "published_at": "2026-09-21T17:30:00Z", "updated_at": "2026-10-11T13:42:35Z", "sections": [ { "id": "short-answer", "heading": "Keep the failed run", "paragraphs": [ "Keep the failed run and nearby successful runs. Record the input, version, timing, test order, concurrent work and dependency state you can observe.", "Your first goal is to reproduce the same failure. If you cannot make it fail locally, preserve enough of the original run to test an explanation. A different local error may need a different fix." ], "source_ids": [ "MHC-D-RESEARCH-0973", "mozilla-intermittents" ] }, { "id": "reproduce-same-failure", "heading": "1. Check that it is the same failure", "paragraphs": [ "Repeat the smallest useful test under controlled conditions. If it fails only in the full test suite or continuous integration (CI), keep that context: test order, shared state, resource pressure or the environment may be involved.", "Match the new result to the original failure before trying a fix. Making a different error disappear does not resolve the reported one." ], "source_ids": [ "MHC-D-RESEARCH-0973", "mozilla-intermittents" ] }, { "id": "compare-runs", "heading": "2. Compare a pass with a failure", "paragraphs": [ "Place a comparable successful run beside the failed one. Keep random seeds, test shards (groups of tests), worker counts, input identities, versions, timings and relevant external responses when available.", "Change one condition at a time. Choose a test for which two plausible explanations predict different results, so the result can help you rule one out." ], "source_ids": [ "MHC-D-RESEARCH-0973" ] }, { "id": "bisect-valid-space", "heading": "3. Use bisection only when it fits", "paragraphs": [ "Bisection narrows an ordered range by testing a point within it. Start with a known-good point, a known-bad point and a range where the result lets you rule out half the candidates. Version history or an ordered data range may fit.", "Do not use it when causes interact or you cannot reliably classify a run as passing or failing. A midpoint result then gives you no sound reason to discard half the range." ], "source_ids": [ "MHC-D-RESEARCH-0295", "MHC-D-RESEARCH-0971" ] }, { "id": "semantic-boundaries", "heading": "4. Inspect where data changes", "paragraphs": [ "Inspect inputs and outputs where data is parsed, mapped, serialized, checked against a policy, queued or sent to a model. These are semantic boundaries: places where data changes form or moves between components. Check whether each side gives the data the same meaning.", "Keep the capture bounded. Better observability is not permission to retain secrets or personal data that the diagnosis does not need." ], "source_ids": [ "MHC-D-RESEARCH-1159" ] }, { "id": "mitigation-vs-diagnosis", "heading": "5. Reduce harm and keep the evidence", "paragraphs": [ "If the failure is harming users now, reduce its impact. A rollback, isolation or controlled fallback may be the right first step while you investigate.", "Preserve enough state to continue the investigation. “Mitigate first” is not permission for uncontrolled changes that erase the evidence." ], "source_ids": [ "MHC-D-RESEARCH-0300" ] }, { "id": "comparison-note", "heading": "Choose the next test", "paragraphs": [ "Use the comparison note below to record the failure, a comparable success and the observed differences. Keep a rival explanation beside your current hypothesis, then choose the smallest test that can help rule one out.", "If you still cannot reproduce or explain the failure, write that down. A passing rerun or a recent change is not enough to establish the cause." ], "source_ids": [ "MHC-D-RESEARCH-0973", "MHC-D-RESEARCH-1159" ] } ], "sources": [ { "id": "MHC-D-RESEARCH-0973", "title": "Keep every rerun result when the failure is intermittent", "url": "https://vedokrok.com/knowledge/keep-every-rerun-result-when-the-failure-is-intermittent" }, { "id": "MHC-D-RESEARCH-0295", "title": "Bisect a large failure domain", "url": "https://vedokrok.com/knowledge/bisect-a-large-failure-domain" }, { "id": "MHC-D-RESEARCH-0971", "title": "Bisect the version history instead of inspecting every change", "url": "https://vedokrok.com/knowledge/bisect-the-version-history-instead-of-inspecting-every-change" }, { "id": "MHC-D-RESEARCH-1159", "title": "Capture semantic boundaries, not only raw logs", "url": "https://vedokrok.com/knowledge/capture-semantic-boundaries-not-only-raw-logs" }, { "id": "MHC-D-RESEARCH-0300", "title": "Mitigate first when the incident is still hurting", "url": "https://vedokrok.com/knowledge/mitigate-first-when-the-incident-is-still-hurting" }, { "id": "mozilla-intermittents", "title": "Mozilla: Debugging Intermittent Test Failures", "url": "https://firefox-source-docs.mozilla.org/testing/debugging-intermittents/index.html" } ], "related_item_ids": [ "MHC-D-RESEARCH-0973", "MHC-D-RESEARCH-0295", "MHC-D-RESEARCH-0971", "MHC-D-RESEARCH-1159", "MHC-D-RESEARCH-0300" ], "redirects": [], "related_article_slugs": [ "how-to-clarify-an-unclear-work-task", "how-to-estimate-a-project-with-uncertain-dependencies" ], "artifact": { "title": "Copy a pass/fail comparison note", "intro": "Use this to keep a flaky failure tied to observable differences and one discriminating next test instead of another round of random reruns.", "lines": [ "Failure signature:", "Nearest comparable passing run:", "Conditions that differed:", "Current hypothesis:", "Plausible rival:", "Next discriminating test:", "Semantic boundary to inspect if still unclear:" ], "copy_label": "Copy debugging note" }, "solution_pack_id": "debug-flaky-failure" }