ai-research-survey

scan.json (26468B)

---

 {
   "paper": {
     "title": "Classifying and Addressing the Diversity of Errors in Retrieval-Augmented Generation Systems",
     "authors": [
       "Kin Kwan Leung",
       "Mouloud Belbahri",
       "Yi Sui",
       "Alex Labach",
       "Xueying Zhang",
       "Stephen Anthony Rose",
       "Jesse C. Cresswell"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.13975"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link at https://github.com/layer6ai-labs/rag-error-classification and states 'Code and data are available' in the abstract. The annotated dataset is also released via this repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that the annotated error dataset is released at the GitHub repository. The DragonBall and CLAPnq datasets used are public datasets (CC BY-NC-SA 4.0 and Apache 2.0 respectively). The curated 406-annotation dataset is shared."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned in the paper. While specific model names are listed (e.g., gte-large-en-v1.5, rank-zephyr-7b-v1-full, Meta-Llama-3-8B-Instruct), there are no library versions or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While a GitHub repository is referenced and hyperparameters are listed in Table 5 and the appendix, there is no explicit 'Reproducing Results' section or README-level guidance described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates for agreement rates (92.9%, 57.8%, 40.3%) and error distributions without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper mentions 'both datasets exhibiting significant chi-squared test results' in Section C.3 when analyzing co-occurrence patterns of error types."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports the improvement from the original pipeline (73.3% correct) to the improved pipeline (82.8% correct) in Section 5, and provides full error count breakdowns with absolute numbers and percentages in Tables 2-3, giving sufficient context for the magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 406 examples were manually annotated out of 832 errors, nor is there a power analysis or discussion of whether this sample size is sufficient for the classification agreement analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The RAGEC system queries the LLM K=10 times per datapoint and reports mode frequency distributions (Table 4), but no standard deviation, IQR, or other spread measure is reported for the main results (agreement rates, classification accuracy)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section B.4 describes two alternative auto-evaluation approaches: single-step error type classification (41.1%/31.1% agreement) and stage-sequential error type classification (47.5%/36.3% agreement), compared to RAGEC (57.8%/40.3%)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "While RAGEC is compared against alternative internal approaches, it is not compared against existing RAG evaluation tools like RAGChecker, ARES, or RAGAs mentioned in the Related Work section. The baselines are only the authors' own design alternatives."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper effectively performs ablation by testing three different design approaches (single-step, stage-sequential, and the proposed backward-cascading RAGEC), each progressively adding design features. This demonstrates which components matter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports stage classification agreement (57.8%), error type classification accuracy (40.3%), answer evaluation agreement (92.9%), and mode frequency consistency (Table 4). Multiple metrics are used across different evaluation aspects."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "406 erroneous responses were manually annotated by the authors for stage and error type. Human annotations serve as ground truth for evaluating RAGEC. Section 5 describes the annotation process and compares RAGEC against human labels."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no separation between development and test sets for the human-annotated data. The 406 annotations appear to have been used both to develop and evaluate RAGEC, with no held-out portion mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides detailed per-domain (finance, law, medicine) and per-query-type (factual, unanswerable, numerical comparison, etc.) breakdowns. Tables 2 and 6 show per-error-type distributions for DragonBall-EN and CLAPnq respectively."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failure modes of both the RAG pipeline (16 error types with concrete examples in Section 4) and of the RAGEC auto-evaluation system itself (e.g., RAGEC over-blaming the generator vs. humans favoring retrieval, Table 1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that RAGEC achieves only 57.8% stage classification and 40.3% error-type classification agreement, stating these are 'far from perfect.' Alternative methods that performed worse are also reported in Section B.4."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a new taxonomy, an auto-evaluation method, and a curated annotated dataset — all of which are presented and validated in the paper. No performance claims are overstated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about the RAG pipeline improvements in Section 5 ('Using RAGEC to Improve RAG'), where specific changes (recursive chunking, increased k) reduced errors from 832 to 534. The controlled manipulation of pipeline components provides adequate justification for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states that the taxonomy is 'not exhaustive of all possible errors,' that the pipeline was 'not highly tuned' for the datasets, and that the study 'focuses on single-turn textual queries.' The scope is appropriately bounded to the tested configurations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses how dataset characteristics affect error distributions (e.g., CLAPnq has no chunking errors because it is pre-chunked), acknowledges that RAGEC over-blames the generator relative to humans, and explores why stage-sequential classification failed at chunking detection, providing substantive alternative explanations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model names: 'Meta-Llama-3-8B-Instruct' for generation, 'gte-large-en-v1.5' for embeddings, 'rank-zephyr-7b-v1-full' for reranking (Table 5 and Section A). GPT-4o and GPT-4o-mini are named for auto-evaluation. These are specific enough to identify exact models on Hugging Face, though API snapshot dates for GPT-4o/GPT-4o-mini are not provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts are provided in Listings 1-10 in the appendix for the reranker, generator, answer evaluation, stage classification, and all error type classification stages. Actual prompt templates with placeholders are shown with enough context to reconstruct inputs."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports key hyperparameters: chunk size (128 tokens), overlap (25 tokens), top-k for retrieval (8 and 5), top-k' for reranking (5 and 3). The K=10 repetitions for classification consistency are also specified. However, LLM temperature/sampling settings for GPT-4o are not stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The full RAG pipeline architecture is described in Section 3 and Figure 1, with each stage (chunking, embedding, retrieval, reranking, generation) detailed. The RAGEC auto-evaluation pipeline is described step-by-step in Section 5 and Appendix B."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and Appendix A describe the chunking strategy (fixed-length with 128 tokens, 25-token overlap for DragonBall; pre-chunked for CLAPnq), embedding procedure, and the datasets used. The DragonBall-CN uses 512 characters instead of 128 tokens."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after Section 6 (Conclusion), discussing non-exhaustiveness of the taxonomy, pipeline tuning limitations, single-turn focus, and the challenges of RAG error classification."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: the taxonomy may miss errors from additional pipeline stages, the pipeline was not tuned for the datasets, evaluation is limited to single-turn textual queries, and annotation was done only by the authors (not crowdsourced), which could introduce bias."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'Our taxonomy focuses on errors that occur within RAG systems, and hence we exclude failures caused by adversarial inputs, faults in the corpus, or similar anomalies' (Section 4). The Limitations section adds that multi-turn and multimodal inputs are out of scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The annotated error dataset is released at the GitHub repository. The underlying DragonBall and CLAPnq datasets are publicly available. This allows independent verification of the annotation and classification results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes the data curation process: 406 erroneous responses from DragonBall-EN were manually annotated for stage and error type. Annotators used all available context including ground truth, retrieved/reranked chunks, and the corpus."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions 'all annotation was done manually by the authors' but does not describe how annotators were selected from among the authors, whether all authors annotated, or whether there was any inter-annotator agreement measurement. For a paper proposing ground-truth annotations, this is a gap."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 3108 EN + 3601 CN queries from DragonBall, 4946 from CLAPnq; 27% (25%) of responses classified as incorrect yielding 832 (1222) errors; 406 manually annotated from DragonBall-EN; 377 confirmed as erroneous (92.9% agreement). Each step is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. All authors are affiliated with Layer 6 AI, a commercial entity, but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: all seven authors are from 'Layer 6 AI, Toronto, Canada' with email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Layer 6 AI is a commercial AI company (acquired by TD Bank). While no explicit funding is disclosed, the work is conducted by a corporate lab that has a financial interest in RAG system development and deployment. The funder's independence cannot be established."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Layer 6 AI is a commercial entity working on applied AI, which could benefit from the tools and frameworks described."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Meta-Llama-3-8B-Instruct for generation and GPT-4o/GPT-4o-mini for evaluation but does not state the training data cutoff dates for any of these models. This is relevant because the benchmarks used (DragonBall, CLAPnq) could potentially overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the DragonBall or CLAPnq datasets (or their source documents from Wikipedia for CLAPnq) may have been seen during Llama-3's pre-training. The DragonBall dataset uses domain-specific synthetic documents that may be less susceptible, but this is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CLAPnq is based on Natural Questions which uses Wikipedia articles — likely present in Llama-3's training data. The paper does not discuss this contamination risk, though the RAG setting partially mitigates it since answers should come from retrieved context."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is not a human subjects study. The 'human annotations' are expert annotations by the authors themselves to create ground truth labels, not a study of human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The annotation was done by the paper's authors as part of the research methodology."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. Author-annotators are not subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied; this is not an experimental study with conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or latency information is reported. RAGEC calls GPT-4o-mini K=10 times per datapoint across multiple stages, which has a non-trivial cost, but this is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or API spend is reported. Running Llama-3-8B-Instruct for generation on thousands of queries, plus GPT-4o/GPT-4o-mini calls for evaluation, would represent significant compute, but this is not disclosed."
       }
     }
   },
   "claims": [
     {
       "claim": "RAGEC achieves 92.9% agreement with human annotations for answer evaluation (determining if a RAG response is incorrect).",
       "evidence": "Section 5 reports that 406 queries were manually annotated and 377 were confirmed as erroneous, yielding a 92.9% agreement rate with RAGEC's answer evaluation step.",
       "supported": "strong"
     },
     {
       "claim": "RAGEC achieves 57.8% stage classification agreement and 40.3% error-type classification accuracy compared to human annotations.",
       "evidence": "Section 5 and Table 1 show the stage classification confusion matrix. The paper honestly acknowledges these rates are 'not close to perfect' but better than alternatives tested.",
       "supported": "strong"
     },
     {
       "claim": "Targeted improvements guided by RAGEC reduced errors from 832 (73.3% correct) to 534 (82.8% correct) on DragonBall-EN.",
       "evidence": "Section 5 ('Using RAGEC to Improve RAG') describes changing from fixed-length to recursive sentence-level chunking and increasing retrieved chunks, resulting in the improvement shown in Table 2 column 'Impr.'",
       "supported": "moderate"
     },
     {
       "claim": "Fabricated Content (hallucination) is relatively rare compared to other error types in RAG systems.",
       "evidence": "Section 5 and Table 2 show E10 (Fabricated Content) accounts for only 34 out of 832 modal errors in DragonBall-EN, while chunking, retrieval, and misinterpretation errors dominate. This is discussed as contrasting with 'the focus on hallucination in recent research.'",
       "supported": "moderate"
     },
     {
       "claim": "The proposed taxonomy covers 16 error types across 4 RAG pipeline stages, more comprehensive than prior work.",
       "evidence": "Section 2 compares to Barnett et al. (2024) with 7 types. Section 4 describes all 16 types with examples. The taxonomy covers chunking (3), retrieval (3), reranking (2), and generation (8) stages.",
       "supported": "strong"
     },
     {
       "claim": "RAGEC outperforms single-step prompting (41.1%/31.1%) and stage-sequential classification (47.5%/36.3%) for error classification.",
       "evidence": "Appendix B.4 describes both alternative approaches and their agreement rates compared to RAGEC's 57.8%/40.3%.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "The paper proposes a 16-type error taxonomy for RAG systems spanning chunking, retrieval, reranking, and generation stages, finding that hallucination (fabricated content) is relatively rare compared to retrieval and chunking errors. The RAGEC auto-evaluation system achieves 92.9% agreement for answer evaluation but only 57.8% for stage classification and 40.3% for error-type classification, indicating RAG error attribution remains a challenging problem. A case study demonstrates that RAGEC-guided improvements (sentence-level chunking and increased retrieval) reduced errors by 35.8% on DragonBall-EN, from 832 to 534 errors.",
   "red_flags": [
     {
       "flag": "Low auto-evaluation accuracy presented as useful",
       "detail": "The RAGEC system achieves only 40.3% error-type classification accuracy and 57.8% stage classification agreement, yet the paper frames this as a useful diagnostic tool. While the paper is honest about these numbers, the practical utility of a system that disagrees with human annotators more than it agrees is questionable."
     },
     {
       "flag": "Author-only annotations without inter-annotator agreement",
       "detail": "All 406 annotations were done by the authors themselves. No inter-annotator agreement (e.g., Cohen's kappa) is reported, no external annotators were used, and it is unclear how many authors annotated each example or whether annotations were reviewed independently."
     },
     {
       "flag": "No contamination analysis for benchmark datasets",
       "detail": "CLAPnq uses Wikipedia-based Natural Questions which are almost certainly in Llama-3's training data. While RAG partially mitigates this by providing retrieved context, the paper does not discuss how parametric knowledge contamination may affect the error distribution (e.g., E11 Parametric Overreliance rates)."
     },
     {
       "flag": "No cost or compute reporting",
       "detail": "The system involves running Llama-3-8B on thousands of queries plus GPT-4o/GPT-4o-mini calls (K=10 per error), but no cost or compute budget is reported. This makes it difficult to assess the practicality of RAGEC for practitioners."
     }
   ],
   "cited_papers": [
     {
       "title": "Seven failure points when engineering a retrieval augmented generation system",
       "authors": ["Scott Barnett", "Stefanus Kurniawan", "Srikanth Thudumu", "Zach Brannelly", "Mohamed Abdelrazek"],
       "year": 2024,
       "relevance": "Most comparable prior work on RAG error taxonomy with seven failure types, directly compared to the proposed 16-type taxonomy."
     },
     {
       "title": "RAGChecker: A Fine-grained Framework for Diagnosing Retrieval-Augmented Generation",
       "authors": ["Dongyu Ru"],
       "year": 2024,
       "relevance": "RAG evaluation framework that diagnoses retrieval-augmented generation at a fine-grained level, closely related to RAGEC's goals."
     },
     {
       "title": "ARES: An Automated Evaluation Framework for Retrieval-Augmented Generation Systems",
       "authors": ["Jon Saad-Falcon", "Omar Khattab", "Christopher Potts", "Matei Zaharia"],
       "year": 2024,
       "relevance": "Automated RAG evaluation framework using LLM-as-a-Judge, a key prior work in the auto-evaluation space this paper contributes to."
     },
     {
       "title": "RAGAs: Automated evaluation of retrieval augmented generation",
       "authors": ["Shahul Es", "Jithin James", "Luis Espinosa Anke", "Steven Schockaert"],
       "year": 2024,
       "relevance": "RAG evaluation metrics framework using LLM-based evaluation, one of the main existing tools for RAG quality assessment."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-Judge methodology used throughout the RAGEC system for error classification."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis"],
       "year": 2020,
       "relevance": "Original RAG paper that established the retrieval-augmented generation paradigm evaluated in this work."
     },
     {
       "title": "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection",
       "authors": ["Akari Asai"],
       "year": 2023,
       "relevance": "Self-reflective RAG approach relevant to generation-stage error mitigation strategies discussed in the paper."
     },
     {
       "title": "Chain-of-verification reduces hallucination in large language models",
       "authors": ["Shehzaad Dhuliawala"],
       "year": 2024,
       "relevance": "Post-generation validation technique discussed as a mitigation for fabricated content errors in RAG systems."
     },
     {
       "title": "Search Engines in an AI Era: The False Promise of Factual and Verifiable Source-Cited Responses",
       "authors": ["Pranav Narayanan Venkit"],
       "year": 2024,
       "relevance": "Classifies errors in public QA systems from human evaluator perspectives, a key comparison point for the proposed taxonomy."
     },
     {
       "title": "Hallucination-Free? Assessing the Reliability of Leading AI Legal Research Tools",
       "authors": ["Varun Magesh"],
       "year": 2025,
       "relevance": "Assesses reliability of AI legal research tools with focus on hallucination, relevant to RAG error analysis in domain-specific settings."
     },
     {
       "title": "A Survey on Hallucination in Large Language Models: Principles, Taxonomy, Challenges, and Open Questions",
       "authors": ["Lei Huang"],
       "year": 2025,
       "relevance": "Comprehensive survey on LLM hallucination taxonomy that this paper contrasts by showing hallucination is only one of many RAG error types."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang"],
       "year": 2023,
       "relevance": "Self-consistency technique used in RAGEC's K=10 repeated classification approach to improve reliability of LLM-based error classification."
     }
   ]
 }