ai-research-survey

scan.json (29065B)

---

 {
   "paper": {
     "title": "Shifting from Ranking to Set Selection for Retrieval Augmented Generation",
     "authors": [
       "Dahyun Lee",
       "Yongrae Jo",
       "Haeju Park",
       "Moontae Lee"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2507.06838",
     "doi": "10.18653/v1/2025.acl-long.861"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SETR, a set-wise passage selection approach for RAG, outperforms traditional reranking methods (including GPT-4o-based RankGPT) on four multi-hop QA benchmarks while using 40-50% fewer passages on average. Chain-of-Thought reasoning with explicit information requirement identification (IRI) contributes distinct gains to retrieval precision and information coverage. Controlled experiments isolating method-level effects (same base model, same teacher supervision) confirm the gains stem from the set-wise formulation itself rather than model capacity differences.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The code is available at https://github.com/LGAI-Research/SetR' and Section 3 references the open-source contribution."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks (HotpotQA, 2WikiMultiHopQA, MuSiQue, MultiHopRAG) are public. Training data is referenced from a public HuggingFace dataset (castorini/rank_zephyr_training_data, Section 3.3.1)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '16×A100 GPUs', Axolotl framework, and Rankify toolkit (Appendix A.3), but provides no requirements.txt, Dockerfile, or detailed library version specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Training and evaluation procedures are described at a high level, but there is no 'Reproducing Results' section or commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-5 report only point estimates (e.g., '36.62' F1) with no confidence intervals, error bars, or ± notation anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SETR 'significantly outperforms' baselines (Section 4.2) but provides no statistical significance tests (no p-values, t-tests, bootstrap tests, or any other test). Differences are assessed by comparing point estimates only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-4 provide baseline and method results allowing direct computation of effect sizes. Section 4.3 reports '3.8%-4.6% higher precision' with baseline context. All results are contextualized against multiple baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why these particular benchmark sizes or training set size (40K) were chosen. No power analysis or discussion of whether sample sizes are sufficient for the claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or any spread measure is reported across experimental runs. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: BM25, bge-reranker-large, RankLlama, RankVicuna, RankZephyr, FirstMistral, and RankGPT (gpt-4o). Tables 1-4 compare against all of them."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent models: FirstMistral (2024), RankGPT with gpt-4o-2024-08-06, RankZephyr (2023), and bge-reranker-large (2023). These represent the current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Three SETR variants are ablated: SETR-Selection only (no reasoning), SETR-CoT (general CoT), and SETR-CoT & IRI (full model). Results in Tables 1, 2, 3 isolate contributions of CoT and IRI components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: EM, F1, Accuracy for QA; MRR@10, NDCG@10, Precision@5, Recall@5 for retrieval (Table 2); Hit@k and information coverage (Figure 3); token efficiency (Table 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated using benchmark metrics (EM, F1, Accuracy, MRR, NDCG, Precision, Recall)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SETR is trained on MS MARCO data (Section 3.3.1) and evaluated on completely separate benchmarks: HotpotQA, 2WikiMultiHopQA, MuSiQue, and MultiHopRAG. This is a zero-shot transfer evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset across all four benchmarks (Tables 1-4), with separate retrieval metrics in Table 2 and efficiency analysis per dataset in Table 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure examples are shown. Section 5.1 discusses performance degradation trends with more passages (Figure 3), but no specific failure cases or error analysis is provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: SETR-CoT underperforms SETR-Selection only on some benchmarks (e.g., MultiHopRAG with BM25: 41.63 vs 43.62 in Table 1). Figure 3 shows that increasing passages degrades performance for all methods."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SETR 'outperforms both proprietary LLM-based rerankers and open-source baselines in terms of answer correctness and retrieval quality.' Tables 1 and 2 support this across four benchmarks with multiple metrics."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('X improves Y') are supported by: (1) ablation studies isolating CoT and IRI components (Tables 1, 3), (2) controlled teacher-model experiments (Table 3), and (3) unified-setting comparison controlling for base model, data, and supervision (Table 4, Section 5.3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract bounds claims to 'multi-hop RAG benchmarks.' The Limitations section explicitly states: 'our method optimizes retrieval for multi-hop and complex queries but has not yet been validated across diverse RAG domains such as code generation or conversational AI.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 explicitly designs experiments to rule out alternative explanations: that gains come from model capacity or teacher supervision rather than the method itself. They test with the teacher model as upper bound and in a unified training setting."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures EM, F1, and Accuracy and claims 'answer correctness' — the measurements directly match the claimed outcomes. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific versions are given: 'gpt-4o-2024-08-06' (footnote 3), 'Llama-3.1-8B-Instruct' with HuggingFace URL (footnote 4). Baseline model versions are also specified (e.g., Zephyr-7B-β, Mistral 7B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Figures 2, 4, 5, 6, and 7 for all methods (SETR-CoT & IRI, SETR-CoT, SETR-Selection only, QA generation prompts). The placeholders are for actual input data (questions, passages)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Training hyperparameters are reported (5 epochs, batch size 512, LR 5×10⁻⁶, AdamW — Section 3.3.2 and Appendix A.3). However, inference parameters (temperature, top-p, max tokens) for both SETR and GPT-4o are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. SETR is a single-pass retrieve → select → generate pipeline with no tools, loops, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3.1 describes data construction: 40K questions from Pradeep et al., top-20 passages retrieved, GPT-4o labeling, text preprocessing with ftfy, and replacing [n] with (n) to prevent model confusion."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears at the end of the paper with three substantive paragraphs covering dependence on initial retrieval, domain-specific validation gaps, and LLM reasoning quality."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) dependence on initial retrieval stage quality, (2) not validated on diverse RAG domains like code generation or conversational AI, (3) effectiveness depends on the LLM's reasoning ability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what was not tested: 'has not yet been validated across diverse RAG domains such as code generation or conversational AI' and notes dependence on initial retrieval quality."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation benchmarks are publicly available, but the GPT-4o-generated teacher annotations used for SETR training (40K examples) are not stated as released. Individual model predictions and intermediate results are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3.1 describes data construction in detail: 40K training questions from Pradeep et al. (2023b), derived from MS MARCO v1, each paired with top-20 retrieved passages, labeled by GPT-4o with zero-shot prompting."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (HotpotQA, 2WikiMultiHopQA, MuSiQue, MultiHopRAG) and an existing training dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: source training data → top-20 passage retrieval → GPT-4o annotation → text preprocessing with ftfy and [n]→(n) replacement → supervised fine-tuning. Evaluation pipeline is also described (retrieval → selection → generation)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. All authors are from LG AI Research (corporate lab) but no funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: LG AI Research and University of Illinois Chicago, with email addresses at lgresearch.ai."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. The work comes from LG AI Research, a corporate lab. While they don't evaluate LG products specifically, the absence of funding disclosure makes independence unverifiable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for Llama-3.1-8B-Instruct or GPT-4o, despite both models being used for evaluation and potentially having seen the benchmark data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between model pretraining data and evaluation benchmarks. HotpotQA (2018), 2WikiMultiHopQA (2020), and MuSiQue (2022) predate both models and could be in their training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Three of four benchmarks (HotpotQA 2018, 2WikiMultiHopQA 2020, MuSiQue 2022) were published well before Llama-3.1 and GPT-4o training cutoffs. No contamination risk discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Table 5 report token-level efficiency analysis: output tokens generated during retrieval and input tokens fed to the generator for all methods across all four benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (16×A100 GPUs, Appendix A.3) but total compute budget is not quantified — no GPU hours, training time, or total API spend for the 40K GPT-4o labeling is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The final hyperparameters (LR 5×10⁻⁶, batch size 512, 5 epochs) are stated without any description of how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final hyperparameter configuration was selected. No mention of validation set selection or search procedure."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so the question of correction for multiple comparisons does not arise."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of author-evaluation bias. The authors compare SETR against baselines implemented via the Rankify toolkit but do not discuss whether their own system receives more careful tuning."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Table 5 explicitly compare token usage (as a compute proxy) across methods alongside their performance. The paper discusses the accuracy-efficiency trade-off spectrum across SETR variants."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the multi-hop QA benchmarks actually measure what the paper claims to evaluate. The benchmarks are used without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The retrieval and generation components are held constant across all comparisons (same bge-large-en-v1.5 retriever, same Llama-3.1-8B-Instruct or GPT-4o generator). Section 5.3 further controls for base model and training data."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. HotpotQA (2018), 2WikiMultiHopQA (2020), and MuSiQue (2022) were all published before the training data collection periods of Llama-3.1 and GPT-4o."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage or whether the evaluation setup provides information not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data. While SETR is trained on MS MARCO and evaluated on different benchmarks, potential overlap in the pretraining data of the base LLMs is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied (no canary strings, membership inference, n-gram overlap, or decontamination pipelines)."
       }
     }
   },
   "claims": [
     {
       "claim": "SETR outperforms both proprietary LLM-based rerankers and open-source baselines in terms of answer correctness across multi-hop QA benchmarks.",
       "evidence": "Tables 1 and 4 show SETR-CoT & IRI achieves the highest F1/EM/Accuracy on most benchmarks. With bge retriever: F1 38.11 on HotpotQA vs 34.45 for RankGPT (gpt-4o), Accuracy 47.14 on MultiHopRAG vs 45.69 for RankGPT.",
       "supported": "strong"
     },
     {
       "claim": "Set-wise passage selection achieves comparable or better performance while using 40-50% fewer passages on average.",
       "evidence": "Table 1 shows SETR uses ~2.63-2.91 passages on average vs 5.00 for all reranking baselines, while achieving higher F1 and Accuracy scores.",
       "supported": "strong"
     },
     {
       "claim": "Both CoT reasoning and information requirement identification (IRI) contribute distinct performance gains.",
       "evidence": "Ablation in Tables 1, 3, and 4 compares SETR-Selection only, SETR-CoT, and SETR-CoT & IRI. Results show mixed but generally positive contributions of each component, with some benchmarks showing CoT underperforming Selection only.",
       "supported": "moderate"
     },
     {
       "claim": "SETR improves information coverage from 19.33% to 36.49% compared to reranking methods.",
       "evidence": "Figure 3d shows information coverage percentages on MultiHopRAG benchmark, with SETR achieving substantially higher coverage than reranking baselines.",
       "supported": "moderate"
     },
     {
       "claim": "Performance gains stem from the set-wise selection formulation itself, not from stronger base models or teacher supervision.",
       "evidence": "Section 5.3 and Tables 3-4 present controlled experiments: teacher model as upper bound (Table 3 using GPT-4o for all methods) and unified setting (Table 4 using same Llama-3.1-8B-Instruct, same training data, same teacher). SETR outperforms reranking in both controlled settings.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "The paper uses 'significantly outperforms' language (Section 4.2) but provides no statistical significance tests anywhere. All claims of superiority are based on comparing point estimates across single runs."
     },
     {
       "flag": "No variance or error bars reported",
       "detail": "No standard deviation, confidence intervals, or any uncertainty quantification across all tables and figures. With single-run results, it is impossible to know whether observed differences exceed natural variance."
     },
     {
       "flag": "Benchmark contamination risk ignored",
       "detail": "Three of four benchmarks (HotpotQA 2018, 2WikiMultiHopQA 2020, MuSiQue 2022) predate the training of both Llama-3.1 and GPT-4o. No contamination analysis or discussion is provided."
     },
     {
       "flag": "Corporate affiliation without funding disclosure",
       "detail": "All lead authors are from LG AI Research, a corporate lab. No funding source, competing interests, or financial interests are disclosed."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection",
       "authors": ["Akari Asai", "Zeqiu Wu", "Yizhong Wang", "Avirup Sil", "Hannaneh Hajishirzi"],
       "year": 2023,
       "arxiv_id": "2310.11511",
       "relevance": "Key RAG system that dynamically determines retrieval necessity with self-reflection, directly comparable to the proposed set-wise selection approach."
     },
     {
       "title": "Is ChatGPT Good at Search? Investigating Large Language Models as Re-Ranking Agents",
       "authors": ["Weiwei Sun", "Lingyong Yan", "Xinyu Ma", "Shuaiqiang Wang", "Pengjie Ren", "Zhumin Chen", "Dawei Yin", "Zhaochun Ren"],
       "year": 2024,
       "arxiv_id": "2304.09542",
       "relevance": "Introduces RankGPT and evaluates LLMs as reranking agents, serving as the primary proprietary baseline in this paper."
     },
     {
       "title": "RankZephyr: Effective and Robust Zero-Shot Listwise Reranking is a Breeze!",
       "authors": ["Ronak Pradeep", "Sahel Sharifymoghaddam", "Jimmy Lin"],
       "year": 2023,
       "arxiv_id": "2312.02724",
       "relevance": "Open-source listwise reranker that serves as a key baseline and provides the training data used by SETR."
     },
     {
       "title": "RankVicuna: Zero-Shot Listwise Document Reranking with Open-Source Large Language Models",
       "authors": ["Ronak Pradeep", "Sahel Sharifymoghaddam", "Jimmy Lin"],
       "year": 2023,
       "arxiv_id": "2309.15088",
       "relevance": "LLM-based listwise reranker baseline trained with GPT-generated supervision for passage ranking."
     },
     {
       "title": "MultiHop-RAG: Benchmarking Retrieval-Augmented Generation for Multi-Hop Queries",
       "authors": ["Yixuan Tang", "Yi Yang"],
       "year": 2024,
       "arxiv_id": "2401.15391",
       "relevance": "Primary evaluation benchmark specifically designed for multi-hop RAG systems with gold evidence annotations."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "Fabio Petroni", "Vladimir Karpukhin", "Naman Goyal"],
       "year": 2021,
       "arxiv_id": "2005.11401",
       "relevance": "Foundational RAG paper that established the retrieval-augmented generation paradigm evaluated in this work."
     },
     {
       "title": "Adaptive-RAG: Learning to Adapt Retrieval-Augmented Large Language Models through Question Complexity",
       "authors": ["Soyeong Jeong", "Jinheon Baek", "Sukmin Cho", "Sung Ju Hwang", "Jong C. Park"],
       "year": 2024,
       "arxiv_id": "2403.14403",
       "relevance": "Adaptive retrieval strategy for RAG that dynamically determines retrieval necessity based on query complexity."
     },
     {
       "title": "RAPTOR: Recursive Abstractive Processing for Tree-Organized Retrieval",
       "authors": ["Parth Sarthi", "Salman Abdullah", "Aditi Tuli", "Shubh Khanna", "Anna Goldie", "Christopher D. Manning"],
       "year": 2024,
       "arxiv_id": "2401.18059",
       "relevance": "Iterative retrieval approach that decomposes and refines complex questions through recursive processing."
     },
     {
       "title": "Large Language Models are Effective Text Rankers with Pairwise Ranking Prompting",
       "authors": ["Zhen Qin", "Rolf Jagerman", "Kai Hui", "Honglei Zhuang", "Junru Wu"],
       "year": 2023,
       "arxiv_id": "2306.17563",
       "relevance": "Evaluates LLMs for pairwise ranking in retrieval systems, relevant to understanding ranking strategies for RAG."
     },
     {
       "title": "CoRAG: A Cost-Constrained Retrieval Optimization System for Retrieval-Augmented Generation",
       "authors": ["Ziting Wang", "Haitao Yuan", "Wei Dong", "Gao Cong", "Feifei Li"],
       "year": 2024,
       "arxiv_id": "2411.00744",
       "relevance": "Multi-round retrieval approach for RAG with cost constraints, representing iterative retrieval strategies compared to SETR's single-step approach."
     },
     {
       "title": "Rankify: A Comprehensive Python Toolkit for Retrieval, Re-Ranking, and Retrieval-Augmented Generation",
       "authors": ["Abdelrahman Abdallah", "Jamshid Mozafari", "Bhawna Piryani", "Mohammed Ali", "Adam Jatowt"],
       "year": 2025,
       "arxiv_id": "2502.02464",
       "relevance": "Toolkit used to implement the full RAG pipeline in this paper's experiments, relevant to reproducibility of RAG evaluations."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "SETR is a drop-in replacement for rerankers in RAG systems with code released, directly usable by practitioners building multi-hop QA systems."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Challenges the dominance of ranking-based retrieval for RAG but the idea that set-level properties matter is intuitive rather than shocking."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward methodological improvement paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub (https://github.com/LGAI-Research/SetR) with model based on publicly available Llama-3.1-8B-Instruct."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "LG AI Research is a known corporate lab but not a top-tier name in NLP research; published at ACL which adds credibility."
     }
   }
 }