ai-research-survey

scan.json (26730B)

---

 {
   "paper": {
     "title": "Towards a Robust Retrieval-Based Summarization System",
     "authors": [
       "Shengjie Liu",
       "Jing Wu",
       "Jingyuan Bao",
       "Wenyi Wang",
       "Naira Hovakimyan",
       "Christopher G Healey"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2403.19889",
     "doi": "10.48550/arXiv.2403.19889"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces LogicSumm, a 7-scenario evaluation framework for RAG-based summarization that reveals LLMs struggle with document relevance assessment, especially when irrelevant documents are present. SummRAG fine-tunes Mistral-7B Instruct on synthetically generated dialogues using LoRA, achieving high logical accuracy (≥0.79 across all scenarios) while matching GPT-3.5 Turbo's summarization quality. In multi-document settings (k=5,8,10), SummRAG maintains stable performance as irrelevant documents increase, unlike Stuff, Map-Reduce, and Refine frameworks which degrade.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in footnote 3: https://github.com/ncsulsj/Robust_Sumsystem. HuggingFace links for model weights and dataset also provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace: https://huggingface.co/datasets/zycjlsj123/ragsummdata, as stated in footnote 3."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section found in the paper. No library versions specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions provided in the paper. While code and data links are given, there is no README description or 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (Tables 2, 3, 4) report only point estimates (e.g., 0.96, 0.91) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims that SummRAG outperforms baselines are based solely on comparing numbers in tables without any test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2-4 provide absolute performance numbers for all baselines and SummRAG across all scenarios, allowing direct comparison of magnitudes. For example, SummRAG at 1.0 vs Mistral-7B Chat at 0.29 for Scenario 1 logical accuracy."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Test set sizes are stated per scenario (57, 48, 50, 36, 50, 43, 98 samples) but no justification for these sizes is given, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported anywhere. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines compared: GPT-3.5 Turbo, Claude 2, Jurassic, LLaMA2-13B Chat, Mistral-7B with various prompting strategies (Table 2), and Stuff/Map-Reduce/Refine frameworks (Table 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-3.5 Turbo, Claude 2, Mistral-7B (all 2023-2024 models), which were contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study showing which components of SummRAG (special tokens, dialogue format, system prefixes, LoRA) contribute to the improvement. The system has multiple components but none are ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: logical accuracy (GPT-4 assessed), BertScore (precision, recall, F1), and Rouge 1/2/L."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation included. Logical correctness is assessed by GPT-4 Turbo, and summarization quality is measured by automated metrics (BertScore, Rouge). The paper makes claims about summarization quality that would benefit from human assessment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states: 'We follow a procedure to generate test data similar to the method employed to create training data.' Test data is separately generated from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per scenario in Tables 2 and 3, and per document count (k=5,8,10) in Table 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 provides a supporting analysis with specific examples of failure cases (stock market query example) and discusses distribution shift from original to fine-tuned model. Figure 2 also illustrates specific limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Chain of Thought prompting fails in Scenario 6 for information conflicts, and that Mistral-7B with explicit logical instructions shows poor performance in Scenarios 1 and 5 (Table 2). SummRAG's Scenario 6 performance (0.79) is also acknowledged as the weakest."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'improved logical coherence and summarization quality' are supported by Tables 2-4 showing SummRAG's high logical accuracy and competitive summarization scores."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SummRAG 'enhances robustness' and 'improves' performance (causal language), but the intervention (fine-tuning) changes multiple variables simultaneously (dialogue format, special tokens, system prefixes, LoRA) without ablation. The comparison against zero-shot baselines also conflates the effect of additional training data with the specific framework design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Robust Retrieval-Based Summarization System' without qualification, but results are limited to CNN Daily Mail and XSum datasets, Mistral-7B as the fine-tuned model, and GPT-4 Turbo as gold standard. No bounding to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example: Could the improvements simply be due to additional training data exposure rather than the specific dialogue format? Could the synthetic test data generation process favor SummRAG's training distribution?"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 Turbo to assess 'logical correctness' and GPT-4 outputs as gold standard summaries, but does not discuss whether GPT-4's judgment is a valid proxy for actual logical correctness, or whether GPT-4 outputs are valid gold standards for summarization quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to as 'GPT-3.5 Turbo', 'GPT-4 Turbo', 'Mistral-7B Instruct', 'Claude 2', 'LLaMa2-13B' without specific version numbers or snapshot dates. Per schema rules, 'GPT-3.5 Turbo' without a version like 'gpt-3.5-turbo-0613' is insufficient."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendices A.1-A.4, including evaluation prompts, dialogue generation prompts, and benchmarking prompts for all scenarios with actual prompt templates and one-shot examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters reported for any model: no temperature, top-p, max tokens for API calls, and no learning rate, rank, or alpha for LoRA fine-tuning."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a RAG pipeline with a retriever and generator, not an agent with tool use, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.1-4.3 describe the dialogue generation process in detail, including how documents are selected from CNN Daily Mail and XSum, how scenarios are constructed, how special tokens are converted to natural language (Table in Appendix A.5), and how training data is formatted."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Conclusion, Limitation, and Future Work' contains substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: 'SummRAG's performance is linked to the scenarios in LogicSumm, which may not encompass all possible real-life situations' and 'the efficacy of our approach is influenced by the quality of the prompts used during dialogue generation.' These are specific to this study's design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings/populations are excluded. The limitations acknowledge incomplete scenarios but don't specify untested domains, languages, or tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace (https://huggingface.co/datasets/zycjlsj123/ragsummdata) for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.1-4.3 describe in detail how dialogues were generated using GPT-4 Turbo with specific prompts, how documents were selected from CNN Daily Mail and XSum, and how each scenario was constructed."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is synthetically generated from standard datasets (CNN Daily Mail, XSum)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the dialogue generation process is described, the full pipeline from raw data to final training set is incomplete: no training data counts are provided, no filtering criteria or rejection rates are stated, and no information on how many dialogues were generated vs. kept."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure found anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: North Carolina State University, University of Illinois Urbana-Champaign, and Northwestern University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed. The absence of disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the models used (GPT-3.5, GPT-4, Claude 2, Mistral-7B, LLaMA-2), despite evaluating them on tasks derived from CNN Daily Mail and XSum."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CNN Daily Mail or XSum data (used to generate test scenarios) appeared in the training data of the evaluated LLMs. These are widely-used datasets highly likely to be in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CNN Daily Mail and XSum were published well before the training cutoffs of GPT-3.5/4, Claude 2, and Mistral-7B. The paper does not discuss the contamination risk of using these well-known datasets."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency reported despite the system requiring multiple LLM API calls (GPT-4 Turbo for dialogue generation and evaluation, plus model fine-tuning)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget stated: no GPU hours for LoRA fine-tuning, no API costs for GPT-4 Turbo dialogue generation, no training time reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results reported across multiple random seeds. All results appear to be single-run numbers."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from a single run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "LoRA hyperparameters appear tuned (they use LoRA for fine-tuning) but no search budget, search method, or number of configurations tried is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final model configuration was selected. No mention of validation-based selection or comparison of multiple configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 7+ methods across 7 scenarios (many comparisons) but performs no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (SummRAG) against baselines without acknowledging the systematic bias of author-evaluation. Baselines are run by the authors in possibly non-optimal configurations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SummRAG requires additional fine-tuning compute compared to zero-shot baselines, but this compute difference is not discussed or accounted for in the comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether LogicSumm's 7 scenarios actually measure what is claimed (real-world RAG summarization robustness). No analysis of construct validity or comparison with alternative evaluation approaches."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The evaluation compares models and frameworks directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. CNN Daily Mail and XSum data used to construct test scenarios predate all evaluated models' training data, creating temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation setup could leak information since test scenarios are generated using similar procedures to training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Training and test data are generated using similar procedures from the same source datasets (CNN Daily Mail, XSum), potentially creating non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied. No canary strings, membership inference, or temporal splits used."
       }
     }
   },
   "claims": [
     {
       "claim": "General-purpose LLMs lack robustness in RAG-based summarization, particularly when retrieved documents are irrelevant to the query.",
       "evidence": "Table 2 shows Mistral-7B Chat with explicit logical instructions achieves only 0.29 accuracy in Scenario 1 (irrelevant retrieval), and Figure 2 illustrates specific failure cases where models follow irrelevant retrieved text instead of recognizing its irrelevance.",
       "supported": "moderate"
     },
     {
       "claim": "SummRAG achieves consistently high logical accuracy across all seven LogicSumm scenarios.",
       "evidence": "Table 2 shows SummRAG achieves 1.0 on Scenarios 1, 2, 5; 0.97 on Scenario 4; 0.79 on Scenario 6; and 0.86 on Scenario 7. These are equal to or better than all baselines across most scenarios.",
       "supported": "moderate"
     },
     {
       "claim": "SummRAG maintains summarization quality comparable to GPT-3.5 Turbo while enhancing robustness.",
       "evidence": "Table 3 shows SummRAG achieves BertScore F1 of 0.91 and Rouge-1 of 0.48 in Scenario 2, matching GPT-3.5 Turbo's 0.90 F1 and 0.48 Rouge-1.",
       "supported": "moderate"
     },
     {
       "claim": "SummRAG shows resilience against irrelevant documents in multi-document settings, unlike other RAG summarization frameworks.",
       "evidence": "Table 4 shows SummRAG maintains BertScore F1 of 0.87-0.88 across k=5,8,10, while Stuff drops from 0.87 to 0.85, Map-Reduce from 0.86 to 0.84, and Refine from 0.85 to 0.83.",
       "supported": "moderate"
     },
     {
       "claim": "Prompting strategies for Mistral-7B show inconsistent performance across different summarization scenarios.",
       "evidence": "Table 2 shows that explicit logical instructions yield 0.29 accuracy in Scenario 1 but 1.0 in Scenarios 4 and 2; zero-shot CoT gives 0.88 in Scenario 1 but fails in Scenario 6; one-shot CoT gives 1.0 in Scenario 1 but 0.19 in Scenario 2.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "GPT-4 as both gold standard and evaluator",
       "detail": "GPT-4 Turbo outputs serve as gold standard summaries for quality evaluation, and GPT-4 Turbo also assesses logical correctness. This creates circular evaluation — a model trained on GPT-4-generated dialogues is evaluated against GPT-4's judgments, biasing toward GPT-4-like outputs."
     },
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across Tables 2-4 are point estimates with no confidence intervals, error bars, or variance from multiple runs. With small test sets (36-98 per scenario), individual results could vary significantly."
     },
     {
       "flag": "Small synthetic test sets",
       "detail": "Test sets are small (36-98 samples per scenario) and synthetically generated using the same process as training data. This raises concerns about whether results generalize beyond the synthetic distribution."
     },
     {
       "flag": "Missing ablation study",
       "detail": "SummRAG has multiple components (special tokens, dialogue format, system prefixes, LoRA fine-tuning, aspect-specific training) but no ablation study isolates which components contribute to improvement."
     },
     {
       "flag": "Contamination risk from well-known datasets",
       "detail": "Test scenarios are constructed from CNN Daily Mail and XSum documents, which are likely in the training data of all evaluated LLMs (GPT-3.5/4, Claude 2, Mistral, LLaMA). This confounds the evaluation of 'robustness' since models may have memorized these documents."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "SummRAG (fine-tuned Mistral-7B) is compared against zero-shot or few-shot baselines. The fine-tuned model has an inherent advantage from task-specific training data. Fairer comparisons would fine-tune baselines on similar amounts of data."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-RAG: Learning to retrieve, generate, and critique through self-reflection",
       "authors": ["Akari Asai", "Zeqiu Wu", "Yizhong Wang", "Avirup Sil", "Hannaneh Hajishirzi"],
       "year": 2023,
       "arxiv_id": "2310.11511",
       "relevance": "Self-RAG is the direct predecessor to SummRAG; the paper modifies and extends Self-RAG for summarization tasks."
     },
     {
       "title": "Benchmarking large language models in retrieval-augmented generation",
       "authors": ["Jiawei Chen", "Hongyu Lin", "Xianpei Han", "Le Sun"],
       "year": 2023,
       "arxiv_id": "2309.01431",
       "relevance": "Evaluates LLM robustness in RAG settings, directly related to evaluation methodology for RAG systems."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["Edward J Hu", "Yelong Shen", "Phillip Wallis", "Zeyuan Allen-Zhu"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Core fine-tuning technique used in SummRAG for parameter-efficient model adaptation."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper establishing the retrieval-augmented generation paradigm."
     },
     {
       "title": "Chain-of-Thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Prompting technique used as baseline in the evaluation of LLM robustness for summarization."
     },
     {
       "title": "Mistral 7B",
       "authors": ["Albert Q Jiang", "Alexandre Sablayrolles", "Arthur Mensch"],
       "year": 2023,
       "arxiv_id": "2310.06825",
       "relevance": "Base model fine-tuned in SummRAG; represents the class of open-source LLMs for code/NLP tasks."
     },
     {
       "title": "LLaMA: Open and efficient foundation language models",
       "authors": ["Hugo Touvron", "Thibaut Lavril", "Gautier Izacard"],
       "year": 2023,
       "arxiv_id": "2302.13971",
       "relevance": "Open-source LLM used as baseline, relevant to understanding LLM capability evaluation."
     },
     {
       "title": "Tug-of-war between knowledge: Exploring and resolving knowledge conflicts in retrieval-augmented language models",
       "authors": ["Zhuoran Jin", "Pengfei Cao", "Yubo Chen"],
       "year": 2024,
       "arxiv_id": "2402.14409",
       "relevance": "Addresses knowledge conflicts in RAG systems, a core challenge also tackled in this paper's Scenario 6."
     },
     {
       "title": "ToolLLM: Facilitating large language models to master 16000+ real-world APIs",
       "authors": ["Yujia Qin", "Shihao Liang", "Yining Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Provides the function-calling token methodology adopted in SummRAG's dialogue generation."
     },
     {
       "title": "Benchmarking large language models for news summarization",
       "authors": ["Tianyi Zhang", "Faisal Ladhak", "Esin Durmus", "Percy Liang"],
       "year": 2024,
       "relevance": "Comprehensive LLM summarization benchmark directly relevant to evaluation methodology."
     }
   ]
 }