ai-research-survey

scan.json (34766B)

---

 {
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "A Library of LLM Intrinsics for Retrieval-Augmented Generation",
     "authors": [
       "Marina Danilevsky",
       "Kristjan Greenewald",
       "Chulaka Gunasekara",
       "Maeda Hanafi",
       "Lihong He",
       "Yannis Katsis",
       "Krishnateja Killamsetty",
       "Yulong Li",
       "Yatin Nandwani",
       "Lucian Popa",
       "Dinesh Raghu",
       "Frederick Reiss",
       "Vraj Shah",
       "Khoi-Nguyen Tran",
       "Huaiyu Zhu",
       "Luis Lastras"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.11704",
     "doi": "10.48550/arXiv.2504.11704"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "LoRA adapters released on HuggingFace at https://huggingface.co/ibm-granite/granite-3.3-8b-rag-agent-lib. Software framework released on GitHub at https://github.com/ibm-granite/granite-io. Example notebooks are also provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation uses public benchmarks (MT-RAG, BEIR, MMLU, RAGTruth, ALCE, LongBench-Cite), but training data is proprietary. Section 2.3 states: 'The training dataset is proprietary and was obtained in combination with a third-party company who contracted the human annotators.' Other intrinsics use synthetically generated training data that is also not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions vLLM as the inference platform and PEFT for fine-tuning, but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions. Individual LoRA training hyperparameters are given but not the full software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references example notebooks and documentation at the HuggingFace and GitHub links for using the intrinsics, but does not include step-by-step instructions for reproducing the training or evaluation experiments reported in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results tables (Tables 2-22) report only point estimates. No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., '9 percentage points jump', '22 percentage points improvement') based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports effect sizes as percentage point differences with baseline context. For example: 'there is an overall 9 percentage points jump when using query rewrite... This jump is more pronounced on the non-standalone fragment, where query rewrite... leads to 22 percentage points improvement over the no-rewrite strategy' (Section 2.2.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the sizes of training or evaluation datasets. For example, the MT-RAG evaluation uses 842 data points but no justification is given for this number. No power analysis is mentioned."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance measures, or spread statistics are reported across any experimental results. All numbers appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each intrinsic is compared against relevant baselines. QR is compared against no-rewrite, Mixtral 8x7b, and gold rewrites (Tables 2-7). CR is compared against Llama 3.3 70B, Granite Guardian, and base Granite (Tables 10-11). AD is compared against BigBird w/ MLP and LLaMA 2-7B (Tables 12-13). HD is compared against GPT-3.5/4, SelfCheckGPT, LMvLM, and finetuned Llama-2-13B (Table 17)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models: Llama-3.3-70B-Instruct, Mixtral-8x22B-Instruct, Granite Guardian (2024). The HD evaluation uses older baselines (GPT-3.5/4-turbo, Llama-2-13B) from the RAGTruth paper, but the other evaluations use current models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 presents a systematic ablation of composite intrinsics, evaluating four flows — None, QR only, AD only, and QR+AD — across answerability classification (Table 20), faithfulness (Table 21), and joint score (Table 22). This isolates the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used throughout: Recall@k at multiple k values, RAGAS Faithfulness, RAD-Bench, precision/recall/F1, NDCG@10, ECE, and JAFS. Each intrinsic is evaluated on at least two metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluations are fully automated using benchmark metrics (Recall@k, RAGAS-F, RAD-Bench, F1, NDCG@10, ECE). No human evaluation of the intrinsics' outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses established benchmark test sets (MT-RAG, SQUADRun, BEIR, MMLU, RAGTruth, ALCE, LongBench-Cite) that are separate from training data. Training uses different corpora — e.g., QR trains on Cloud corpus while evaluating on full MT-RAG; AD trains on Government corpus while evaluating on MT-RAG."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by multiple categories: standalone vs non-standalone subsets (Tables 2-7), answerable vs unanswerable classes (Tables 12-14), per-dataset in BEIR (Table 15), per-dataset in LongBench-Cite (Table 19), per-MMLU task (Figure 1), and per-benchmark for CR (Tables 10-11)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No systematic error analysis or qualitative examples of failures are provided. The paper mentions performance drops in reranking on some BEIR tasks and notes potential puzzling outcomes of composite intrinsics, but does not analyze specific failure cases or error categories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: reranking shows performance drops on 5/15 BEIR tasks including arguana (0.567→0.347) in Table 15. The composite flow analysis shows QR+AD degrades AD's unanswerable F1 from 69 to 51 (Table 20). The appendix notes Granite 3.3 is 'overall slightly worse than the 3.2 version' on LongBench-Cite (Table 26)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract is primarily descriptive, introducing the concept of LLM intrinsics and the library. It claims the paper 'describes the intended usage, training details, and evaluations for each intrinsic, as well as compositions of multiple intrinsics,' which is accurate. No unsupported empirical claims appear in the abstract."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims like 'using query rewrite directly improves the retriever performance, which in turn improves the answer generation performance.' These are supported by controlled comparisons where a single component is swapped (e.g., QR vs no-rewrite with everything else fixed). Section 10 provides a systematic ablation of QR and AD with four controlled flows."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'For illustrative purposes, these intrinsics are implemented using IBM Granite language models, with extension to other model families possible in the future' (Section 1). UQ Section 7.1 states the model 'is not intended to predict the certainty of responses generated by any other models besides itself.' The scope is bounded to Granite 3.3 8b."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed improvements are discussed. For example, the paper does not consider whether performance gains could be due to the additional compute from the LoRA, differences in tokenization, or other confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's measurements closely match its claims: Recall@k for retrieval quality, RAGAS Faithfulness for answer faithfulness, NDCG@10 for reranking, ECE for calibration quality, and F1 for classification accuracy. JAFS is carefully defined with a mathematical formula (Section 5.2.2). The paper does not make broader claims beyond what the metrics measure."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific HuggingFace model IDs are used throughout: 'ibm-granite/granite-3.3-8b-instruct', 'meta-llama/Llama-3.3-70B-Instruct', 'Mixtral-8x22B-Instruct', 'ibm-granite/granite-guardian-3.1-5b'. These include version numbers and sizes."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided for multiple intrinsics: the QR rewrite role prompt (Section 2), the passage reranking prompt (Section 6.1), the HD instruction text (Section 8.1), and the CG instruction text (Section 9.1). These are verbatim prompts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "LoRA training hyperparameters are reported for each intrinsic. For example, QR: 'rank = 32, learning rate = 3e-6, number of epochs = 25, with early stopping based on validation set, and 90/10 split' (Section 2.3). Similar detail for CR, AD, UQ, HD, and CG. Inference settings (k=20, top-5 passages) are also specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. The intrinsics are individual LoRA adapters and software components composed into pipeline flows, not agent-based systems with retry logic, memory, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Training data construction is documented for each intrinsic: QR uses human-created conversations from MT-RAG Cloud corpus (Section 2.3), CR uses synthetic data from CLAPNQ with Mixtral as judge (Section 4.3), AD uses Government corpus with human + synthetic data and Mixtral validation (Section 5.3), HD/CG use multi-step pipeline from CoQA/MultiDoc2Dial/QuAC (Sections 8.3, 9.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper proceeds from evaluation results directly to a brief conclusion (Section 11) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. No study-specific concerns about evaluation methodology, training data quality, or generalization are raised."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions that intrinsics are 'implemented using IBM Granite language models' and UQ states it is 'not intended to predict the certainty of responses generated by any other models,' there is no systematic statement of what the results do NOT show. No dedicated scope boundaries discussion exists."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw training data is proprietary and not released. Raw experimental outputs (model predictions, intermediate results) are not made available. Only aggregated metrics are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described for each intrinsic with sources, methods, and quality control. For example, QR training uses 'the publicly available Cloud corpus of technical documentation pages from MT-RAG' with 'high-quality, human-created conversations' (Section 2.3). Synthetic data generation and Mixtral-based validation are described for CR, AD."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data sources are standard benchmarks and corpora. Human annotators were used for training data creation but are not study participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the general training data generation process is described for each intrinsic, exact counts of generated examples, filtering rates, and specific transformation steps are missing. For example, Section 4.3 mentions Mixtral was used to 'validate the generated labels and filter out noisy samples' but doesn't state how many samples were generated or filtered."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section appears in the paper. The acknowledgments section (Section 12) only says 'Thanks to internal and external annotators' without disclosing funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All 16 authors are clearly listed as 'IBM Research' on the first page. The paper evaluates IBM Granite models, making the affiliation-product relationship transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "IBM Research employees evaluate IBM's own Granite models and release IBM's own intrinsics library. IBM has a direct commercial interest in demonstrating the effectiveness of its Granite models and RAG tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper. Given all authors are IBM employees evaluating IBM products, the absence of such a declaration is notable."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for the Granite 3.3 8b base model. The paper does not mention when the pre-training data was collected, making it impossible to assess temporal overlap with evaluation benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between Granite's pre-training data and the evaluation benchmarks (MT-RAG, BEIR, MMLU, SQUADRun, etc.). This is particularly relevant since MMLU and BEIR are widely used and likely in many training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses multiple public benchmarks (MMLU, BEIR, SQuAD, HotpotQA) that have been available since well before Granite's training, but does not discuss contamination risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The paper evaluates model intrinsics on automated benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency numbers are reported. Section 6.2 notes passage reranking 'only adds a small overhead' but provides no actual measurements. No API costs, tokens consumed, or wall-clock times are given for any intrinsic."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. GPU hours, hardware used, training time, and total compute cost are not mentioned for any of the six LoRA training processes or evaluations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single training runs and single evaluations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single runs or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While training hyperparameters (rank, learning rate, epochs) are reported for each LoRA, no search budget is described. It is unclear how these values were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Some intrinsics mention early stopping on validation sets (e.g., QR in Section 2.3), but no justification is given for the choice of hyperparameters like rank=32 or specific learning rates. The overall configuration selection process is not described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons. The paper makes numerous comparisons across 8 intrinsics, multiple benchmarks, and multiple baselines."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "IBM researchers evaluate their own Granite-based intrinsics against baselines that include their own reimplementations. The potential for self-evaluation bias is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute. The LoRA adapters add inference overhead and training cost, but these are not quantified or compared against simpler alternatives at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses multiple benchmarks (MT-RAG, BEIR, MMLU, RAGTruth, ALCE, LongBench-Cite) without discussing whether they adequately measure the claimed capabilities or their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The intrinsics are individual components and pipeline compositions, not scaffold-dependent agent systems."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Several evaluation benchmarks (SQuAD, HotpotQA, BEIR tasks) predate the Granite model's training and their solutions may be in the training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The paper does not address whether the evaluation setup provides any information that would not be available in real deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MT-RAG is used for both LoRA training (specific sub-corpora) and evaluation. While different portions are used (Cloud corpus for QR training, Government corpus for AD training, full MT-RAG for evaluation), the independence of train and test is not explicitly verified."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Query Rewrite LoRA improves Recall@20 by 9 percentage points on full MT-RAG (0.67 to 0.76) and by 22 percentage points on non-standalone queries (0.44 to 0.66), matching human gold rewrites.",
       "evidence": "Tables 2, 3, and 4 in Section 2.2.1 compare recall across no-rewrite, Mixtral, LoRA, and gold rewrite strategies on full, non-standalone, and standalone MT-RAG subsets.",
       "supported": "moderate"
     },
     {
       "claim": "Query Rewrite LoRA improves answer quality by 8pp in RAGAS Faithfulness and 4pp in RAD-Bench on full MT-RAG.",
       "evidence": "Tables 5-7 in Section 2.2.2 show answer generation quality metrics across rewrite strategies. The 18pp RAGAS-F improvement on non-standalone queries is notable.",
       "supported": "moderate"
     },
     {
       "claim": "Query Expansion achieves highest retrieval recall (87% Recall@50) outperforming all single-query methods.",
       "evidence": "Table 8 in Section 3.2.1 shows Recall@50 and @100 for query expansion vs individual strategies. However, query expansion uses up to 100 passages vs 20 for single methods.",
       "supported": "moderate"
     },
     {
       "claim": "Context Relevance LoRA outperforms Llama 3.3 70B, Granite Guardian 3.1 5b, and base Granite on precision for relevant labels and recall for irrelevant labels across 8 benchmarks.",
       "evidence": "Tables 10 and 11 in Section 4.2.3 show precision and recall scores across MTRAG, CLAPNQ, Drop, FinanceBench, BioASQ, Open Aus. Legal Corpus, and HotpotQA+SquadV2.",
       "supported": "strong"
     },
     {
       "claim": "Answerability Determination LoRA achieves 90% weighted F1 on MT-RAG, outperforming BigBird and LLaMA 2-7B.",
       "evidence": "Table 13 in Section 5.2.1 shows weighted F1 of 90% for the LoRA vs 69.6% for BigBird and 87.1% for LLaMA 2-7B on MT-RAG. Table 14 shows 17% JAFS lift over vanilla Granite.",
       "supported": "moderate"
     },
     {
       "claim": "UQ LoRA reduces average ECE from 0.20 (base model) to 0.064 across MMLU tasks with no degradation in zero-shot accuracy (89%).",
       "evidence": "Section 7.2 and Figure 1 report ECE values across MMLU tasks. Table 16 shows MAE for post-answer and pre-answer certainty prediction under different system prompts.",
       "supported": "moderate"
     },
     {
       "claim": "Hallucination Detection LoRA achieves 68.6% F1 on RAGTruth QA, competitive with finetuned Llama-2-13B (68.2%) and far exceeding prompted GPT-4-turbo (45.6%).",
       "evidence": "Table 17 in Section 8.2 compares response-level hallucination detection across baselines from the RAGTruth paper.",
       "supported": "moderate"
     },
     {
       "claim": "Citation Generation LoRA performs on par with Llama 70B on ALCE passage-level citations (F1 ~60-62) and significantly outperforms it on LongBench-Cite span-level citations (68.6 vs 42.0 avg F1).",
       "evidence": "Tables 18 and 19 in Sections 9.2.1 and 9.2.2 show citation evaluation results. The 8b LoRA achieves an impressive advantage on fine-grained span-level citations.",
       "supported": "strong"
     },
     {
       "claim": "Composite QR+AD flow achieves best overall JAFS of 66, a 17-point improvement over the no-intrinsic baseline (49).",
       "evidence": "Tables 20-22 in Section 10.1 present the four-flow ablation (None, QR, AD, QR+AD) on answerability classification, faithfulness, and joint score. However, QR+AD ties AD-only (both 66).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces a library of 8 RAG intrinsics implemented as LoRA adapters for Granite 3.3 8b, covering query rewrite, query expansion, context relevance, answerability determination, passage reranking, uncertainty quantification, hallucination detection, and citation generation. Query Rewrite shows the largest impact, improving retrieval recall by up to 22pp on context-dependent queries. Citation Generation stands out as an 8b LoRA that significantly outperforms Llama 70B on fine-grained span-level citations. The composite flow analysis reveals that combining intrinsics introduces tradeoffs — adding QR to AD degrades unanswerable detection while improving overall faithfulness.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All 16 authors are IBM Research employees evaluating IBM's own Granite models and intrinsics library. No independent evaluation is included. The paper has commercial interest in demonstrating Granite's capabilities."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All results across 22 tables are point estimates from apparently single runs. No confidence intervals, significance tests, or variance measures are reported, making it impossible to assess whether reported differences are meaningful."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations discussion, threats to validity, or scope boundaries section. For a paper proposing a general framework evaluated on specific benchmarks with a specific model, this is a significant omission."
     },
     {
       "flag": "Proprietary training data",
       "detail": "Training data for several intrinsics (notably Query Rewrite) is proprietary, and synthetically generated data for others is not released. This prevents independent reproduction of the LoRA training."
     },
     {
       "flag": "No contamination analysis",
       "detail": "The Granite base model's training data cutoff is never stated, and no contamination analysis is performed despite evaluating on well-known public benchmarks (MMLU, BEIR, SQuAD) that likely overlap with pre-training data."
     },
     {
       "flag": "Unfair comparison in Query Expansion",
       "detail": "Query Expansion uses 5 queries to retrieve up to 100 passages (Table 8), while single-query baselines retrieve only 20. The comparison is not compute-normalized, inflating the apparent advantage of expansion."
     }
   ],
   "cited_papers": [
     {
       "title": "MTRAG: A multi-turn conversational benchmark for evaluating retrieval-augmented generation systems",
       "authors": ["Yannis Katsis", "Sara Rosenthal", "Kshitij Fadnis", "Chulaka Gunasekara", "Young-Suk Lee", "Lucian Popa", "Vraj Shah", "Huaiyu Zhu", "Danish Contractor", "Marina Danilevsky"],
       "year": 2025,
       "arxiv_id": "2501.03468",
       "relevance": "Primary RAG evaluation benchmark used throughout the paper for multi-turn conversational retrieval and generation."
     },
     {
       "title": "RAGTruth: A hallucination corpus for developing trustworthy retrieval-augmented language models",
       "authors": ["Cheng Niu", "Yuanhao Wu", "Juno Zhu", "Siliang Xu", "Kashun Shum", "Randy Zhong", "Juntong Song", "Tong Zhang"],
       "year": 2024,
       "relevance": "Hallucination detection benchmark used to evaluate the HD intrinsic, relevant to RAG trustworthiness."
     },
     {
       "title": "Enabling large language models to generate text with citations",
       "authors": ["Tianyu Gao", "Howard Yen", "Jiatong Yu", "Danqi Chen"],
       "year": 2023,
       "doi": "10.18653/v1/2023.emnlp-main.398",
       "relevance": "ALCE citation benchmark used to evaluate the citation generation intrinsic; foundational work on LLM-generated citations."
     },
     {
       "title": "LongCite: Enabling LLMs to generate fine-grained citations in long-context QA",
       "authors": ["Jiajie Zhang", "Yushi Bai", "Xin Lv", "Wanjun Gu", "Danqing Liu"],
       "year": 2024,
       "arxiv_id": "2409.02897",
       "relevance": "Fine-grained citation benchmark used for evaluating span-level citation generation capabilities of LLMs."
     },
     {
       "title": "Thermometer: Towards universal calibration for large language models",
       "authors": ["Maohao Shen", "Subhro Das", "Kristjan Greenewald", "Prasanna Sattigeri", "Gregory Wornell", "Soumya Ghosh"],
       "year": 2024,
       "arxiv_id": "2403.08819",
       "relevance": "Calibration method used to train the uncertainty quantification intrinsic; relevant to LLM reliability and trustworthiness."
     },
     {
       "title": "Granite Guardian",
       "authors": ["Inkit Padhi", "Manish Nagireddy", "Giandomenico Cornacchia"],
       "year": 2024,
       "arxiv_id": "2412.07724",
       "relevance": "IBM's LLM guardrail model used as a baseline for context relevance evaluation."
     },
     {
       "title": "Activated LoRA: Fine-tuned LLMs for intrinsics",
       "authors": ["Kristjan Greenewald", "Luis Lastras", "Thomas Parnell", "Lucian Popa", "Vraj Shah", "Giulio Zizzo"],
       "year": 2025,
       "relevance": "Sister paper introducing the activated LoRA mechanism for efficient inference of LLM intrinsics."
     },
     {
       "title": "SelfCheckGPT: Zero-resource black-box hallucination detection for generative large language models",
       "authors": ["Potsawee Manakul", "Adian Liusie", "Mark Gales"],
       "year": 2023,
       "relevance": "Black-box hallucination detection method used as a baseline; relevant to LLM output verification."
     },
     {
       "title": "Retrieval augmented generation (RAG) and beyond: A comprehensive survey on how to make your LLMs use external data more wisely",
       "authors": ["Siyun Zhao", "Yuqing Yang", "Zilong Wang"],
       "year": 2024,
       "arxiv_id": "2409.14924",
       "relevance": "Comprehensive RAG survey covering retrieval strategies and generation quality, directly relevant to the RAG pipeline components this paper addresses."
     },
     {
       "title": "Know what you don't know: Unanswerable questions for SQuAD",
       "authors": ["Pranav Rajpurkar", "Robin Jia", "Percy Liang"],
       "year": 2018,
       "relevance": "SQUADRun benchmark used for answerability determination evaluation; foundational work on unanswerable question detection."
     },
     {
       "title": "BEIR: A heterogeneous benchmark for zero-shot evaluation of information retrieval models",
       "authors": ["Nandan Thakur", "Nils Reimers", "Andreas Rücklé", "Abhishek Srivastava", "Iryna Gurevych"],
       "year": 2021,
       "arxiv_id": "2104.08663",
       "relevance": "Standard information retrieval benchmark used to evaluate the passage reranking intrinsic across 15 diverse tasks."
     },
     {
       "title": "CLAPnq: Cohesive long-form answers from passages in natural questions for RAG systems",
       "authors": ["Sara Rosenthal", "Avirup Sil", "Radu Florian", "Salim Roukos"],
       "year": 2025,
       "doi": "10.1162/tacl_a_00729",
       "relevance": "RAG question answering benchmark used for context relevance evaluation and as training data source."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Released LoRA adapters on HuggingFace and a Python SDK on GitHub that developers can immediately integrate into RAG pipelines."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected benefits of standard RAG pipeline components (query rewriting, reranking, hallucination detection) without challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the intrinsics are intended to improve RAG reliability."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward technical contribution."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Models are pip-installable via HuggingFace with example notebooks provided for immediate use."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "IBM Research is well-known in enterprise AI but not among the highest-profile consumer AI labs."
     }
   }
 }