{
  "schema": "vedokrok.public-item.v1",
  "release_id": "MHC-RPUB-20260920-75ad787a",
  "url": "/knowledge/separate-a-retrieval-failure-from-an-answer-generation-failure",
  "id": "MHC-D-RESEARCH-0994",
  "version": "0.1.0",
  "title": "Separate a retrieval failure from an answer-generation failure",
  "summary": "An excellent writer cannot quote the paragraph it never received.",
  "kind": "question",
  "body": "Inspect the question, retrieved passages and answer as separate artifacts. Determine whether the needed evidence was available, retrieved and used correctly. Also check whether the answer addresses the actual question. These are different failure locations and may need different repairs.",
  "limits": [
    "Faithfulness to a wrong source is still wrong. Automated evaluation scores need checking and should not replace source-version verification."
  ],
  "topics": [
    "work-03-ai-assurance"
  ],
  "intents": [],
  "source_ids": [
    "RS-A0FCA45732B1AA48"
  ],
  "evidence": [
    {
      "claim": "Ragas separates faithfulness to retrieved context, answer relevance and context relevance instead of treating response quality as a single property.",
      "source_id": "RS-A0FCA45732B1AA48",
      "role": "supports",
      "note": "Faithfulness to an incorrect source is not factual truth, and automated estimates can misclassify errors.",
      "locator": "Evaluation framework"
    }
  ],
  "use_when": [
    "A document-grounded assistant gives a wrong answer and the team immediately starts rewriting the prompt."
  ],
  "avoid_when": [
    "Faithfulness to a wrong source is still wrong. Automated evaluation scores need checking and should not replace source-version verification."
  ],
  "example": "The current procedure is in the library, but retrieval returns an obsolete version. A more forceful writing prompt is not the first repair.",
  "check": "The defect is assigned to a supported failure location with the relevant artifacts attached.",
  "question": "Was the necessary information present in the approved source collection? · Did retrieval supply that information with the right version and scope? · Does the answer accurately use the supplied evidence and answer the question?",
  "sources": [
    {
      "id": "RS-A0FCA45732B1AA48",
      "title": "Ragas: Automated Evaluation of Retrieval Augmented Generation",
      "url": "https://arxiv.org/abs/2309.15217"
    }
  ],
  "relations": [
    {
      "from": "MHC-D-RESEARCH-0994",
      "to": "MHC-D-RESEARCH-0996",
      "type": "useful_with",
      "url": "/knowledge/give-the-second-check-evidence-the-first-answer-did-not-create"
    }
  ],
  "collections": [
    {
      "id": "RC-620AACFCDEA000D1",
      "title": "Make AI-assisted work earn your trust",
      "url": "/collections/make-ai-assisted-work-earn-your-trust"
    }
  ]
}