ai-research-survey

scan.json (25062B)

---

 {
   "paper": {
     "title": "Aligned Query Expansion: Efficient Query Expansion for Information Retrieval through LLM Alignment",
     "authors": [
       "Adam Yang",
       "Gustavo Penha",
       "Enrico Palumbo",
       "Hugues Bouchard"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2507.11042"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. No mention of code availability."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses four publicly available datasets: Natural Questions, TriviaQA, WebQA, and Entity Questions, all cited with their original references. These are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies the model (T0 3B) and some hyperparameters but provides no requirements.txt, Dockerfile, library versions, or environment setup details beyond listing the model name and optimizer."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The implementation details section (Section 4.2) gives some parameters but not enough to fully reproduce without additional information and code."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-4 report only point estimates (e.g., '30.8', '53.6') with no confidence intervals, error bars, or ± notation. The only statistical information is significance test superscripts."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 use Student's t-tests with Bonferroni correction, indicated by superscripts denoting statistically significant improvements. Tables 3 and 4 also use the same tests with the ‡ superscript notation."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw accuracy differences (e.g., '17.8% improvement in top-1 accuracy') and percentage reductions (71.1% memory, 69.5% time) but does not report standardized effect sizes such as Cohen's d or odds ratios. The raw differences with baseline context in the efficiency comparison (Section 5.3) approach adequate reporting, but the main retrieval results in Tables 1-4 present only point estimates without baseline context in the text."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why these particular dataset sizes or test set sizes were chosen. No power analysis is discussed. The paper simply uses the standard splits of existing benchmarks without discussing whether the sample sizes are adequate for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or any spread measures are reported. All results appear to be from single runs. There is no mention of averaging over multiple seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: (0) original query without expansion, (1) zero-shot query expansion, and (2) filtering approach (generate-then-filter using a DeBERTa reranker). These are described in Section 4.2 and results are in Tables 1-4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include the EAR filtering approach (Chuang et al., 2023) and Doc2Query-- (Gospodinov et al., 2023), which are contemporary. The filtering baseline follows the setup from Chuang et al. [2] which is the most relevant prior work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests three alignment variants: RSFT alone, DPO alone, and RSFT + DPO combined. This effectively ablates the contribution of each alignment component, shown across Tables 1-4."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across six Top-N retrieval accuracy metrics: Top-1, Top-5, Top-10, Top-20, Top-50, and Top-100. Additionally, the efficiency analysis reports GPU memory, computational time, and accuracy."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. The quality of query expansions is assessed only through automated retrieval accuracy metrics. Given that the paper discusses hallucination and query quality (Table 5 shows qualitative examples), human evaluation of expansion quality would be relevant but is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.2 explicitly describes using training splits for fine-tuning and test splits for evaluation. The out-of-distribution experiments further test on entirely different datasets (WebQA, EntityQuestions) than those used for training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across four different datasets (Natural Questions, TriviaQA, WebQA, EntityQuestions) in Tables 1-4, and across multiple top-N values. Out-of-distribution results are shown separately per training source."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 shows qualitative examples of query expansions but focuses on successes. There is no systematic error analysis or discussion of failure cases where AQE underperforms or produces harmful expansions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that RSFT alone lags behind filtering on Natural Questions (Table 1), that zero-shot expansion sometimes underperforms the original query, and that RSFT and DPO individually reduce diversity (Section 5.4). The filtering approach failing in out-of-distribution settings is also a negative result for that baseline."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AQE 'outperforms baseline models for query expansion in both in-domain and out-of-domain settings.' Tables 1-4 support this for the RSFT+DPO combination. The abstract also claims 'reducing computational costs' which is supported by Figure 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation ('RSFT + DPO provides the best results' by testing each component separately and in combination). The alignment training is a controlled intervention on the model, and comparisons are made against the same base model without alignment. The experimental design of controlled single-variable manipulation supports these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to address 'Information Retrieval' generally, but experiments are limited to passage retrieval in open-domain question answering using BM25 as the sparse retriever. The paper does not bound claims to this specific setting. Claims like 'model-agnostic' (Section 5.2) are made based on testing only a single model (T0 3B)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why AQE outperforms filtering. For example, it does not consider whether the improvements come from the specific training data distribution rather than the alignment mechanism, or whether the reranker model capacity is a confound."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'T0 3B model [21]' and 'DeBERTa V3 base [7]' but does not provide exact version identifiers, checkpoint dates, or Hugging Face model card names with version info. 'T0 3B' is a model family name without a specific checkpoint."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides the actual prompt text used: 'To answer this query, we need to know:' prepended with the original query. This is the full prompt used for zero-shot query expansion generation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports temperature τ=1.0, top-k=50, learning rate η=5×10⁻⁵, batch size B=16, one epoch of fine-tuning, β=0.1 for DPO, and the AdamW optimizer."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. AQE is a fine-tuning approach applied to a language model for single-shot query generation, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how the training data from Natural Questions and TriviaQA was preprocessed, what filtering was applied, or how the 50 generated expansions per query were stored and managed. The pipeline from raw datasets to training pairs is not fully described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion mentions 'Future work could explore further enhancements' but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. There is no mention of potential confounds, limitations of the experimental setup, or risks to the conclusions drawn."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. For instance, it does not acknowledge that results are limited to a single language model (T0 3B), a single retriever (BM25), or the open-domain QA setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The underlying datasets are publicly available, but the generated query expansions, preference pairs used for DPO training, and the trained model weights are not released. There is no way to verify the specific training data or intermediate outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes how query expansions are generated (zero-shot prompting with n=50 per query), Section 3.2 describes how they are ranked using BM25 retrieval, and Section 4.1 describes the source datasets. The data creation pipeline is conceptually clear."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper uses standard public benchmarks (Natural Questions, TriviaQA, WebQA, Entity Questions)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) generate 50 expansions per query via zero-shot prompting, (2) rank expansions by BM25 retrieval effectiveness, (3) select best/worst pairs, (4) fine-tune with RSFT and/or DPO. Figure 1 illustrates this pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Adam Yang at Mistral AI, Gustavo Penha, Enrico Palumbo, and Hugues Bouchard at Spotify. A footnote notes the work was done during Yang's internship at Spotify."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Spotify, where 3 of 4 authors work and where the first author interned, is an information retrieval company that would benefit from improvements in query expansion. The paper cites concurrent work deploying aligned query generation at a 'large industrial setting' [16] (Taobao). No discussion of potential conflicts."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. The authors work at companies (Spotify, Mistral AI) that have commercial interests in information retrieval technology."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The T0 3B model's training data cutoff is not stated. While the paper fine-tunes T0 3B rather than evaluating its zero-shot capability, the pre-training data could influence the quality of generated expansions, and this is not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Natural Questions or TriviaQA test examples appeared in T0's pre-training data. Since T0 was trained on a variety of NLP datasets, overlap with these popular QA benchmarks is plausible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Natural Questions (2019), TriviaQA (2017), and Entity Questions (2021) were all published before T0's training. The paper does not discuss whether these benchmarks were in T0's pre-training data, which could inflate performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 reports GPU memory occupancy (with 71.1% reduction) and computational time (with 69.5% reduction) for inference on TriviaQA, comparing filtering vs. AQE. Specific values are shown in the bar charts."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget for training (GPU hours, hardware used, total training time) is not stated. Only inference efficiency is compared. The paper does not report what hardware was used for training or how long training took."
       }
     }
   },
   "claims": [
     {
       "claim": "AQE (RSFT + DPO) outperforms baseline methods including filtering in both in-domain and out-of-domain retrieval tasks.",
       "evidence": "Tables 1-4 show RSFT+DPO achieving the highest retrieval accuracy across most Top-N metrics on Natural Questions, TriviaQA, WebQA, and EntityQuestions, with statistical significance indicated by superscripts (Student's t-tests with Bonferroni correction).",
       "supported": "strong"
     },
     {
       "claim": "AQE reduces memory consumption by 71.1% and computational time by 69.5% compared to filtering, while improving top-1 retrieval accuracy by 17.8%.",
       "evidence": "Figure 2 presents bar charts comparing GPU memory occupancy, computational time, and top-1 accuracy between filtering and AQE on TriviaQA inference.",
       "supported": "moderate"
     },
     {
       "claim": "The filtering approach struggles in out-of-distribution settings, with 5-10% drops in top-1 accuracy.",
       "evidence": "Tables 3 and 4 show filtering underperforming both original query and zero-shot expansion in OOD settings (e.g., 9.1% top-1 on WebQA vs. 18.9% for original query when trained on Natural Questions).",
       "supported": "strong"
     },
     {
       "claim": "RSFT + DPO increases generation diversity compared to individual alignment methods.",
       "evidence": "Figure 3 shows average pairwise cosine distance for different methods, with RSFT+DPO showing higher diversity than RSFT or DPO alone, though the visual difference is small.",
       "supported": "moderate"
     },
     {
       "claim": "AQE is model-agnostic and delivers strong performance across diverse retrieval contexts.",
       "evidence": "Section 5.2 makes this claim based on OOD generalization results, but only a single model (T0 3B) and a single retriever (BM25) are tested. The claim significantly exceeds the evidence.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Aligned Query Expansion (AQE) uses LLM alignment techniques (RSFT and DPO) to fine-tune query expansion models, eliminating costly generate-then-filter steps. On Natural Questions and TriviaQA, RSFT+DPO achieves statistically significant improvements over filtering baselines in both in-domain and out-of-domain settings. AQE reduces inference memory by 71.1% and time by 69.5% compared to filtering while improving top-1 accuracy by 17.8% on TriviaQA. The filtering approach degrades severely in out-of-distribution settings while AQE maintains robust performance.",
   "red_flags": [
     {
       "flag": "No variance or error bars",
       "detail": "All results appear to be from single experimental runs. No standard deviations, confidence intervals, or multi-seed experiments are reported despite making comparative claims. The statistical significance tests partially mitigate this but uncertainty in the point estimates is unknown."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations section, no threats to validity discussion, and no explicit acknowledgment of what the results do not show. This is a significant omission for a paper making broad claims about query expansion."
     },
     {
       "flag": "Overclaimed generalizability",
       "detail": "The paper claims AQE is 'model-agnostic' (Section 5.2) based on testing a single model (T0 3B) with a single retriever (BM25). The title suggests general 'Information Retrieval' applicability when results are limited to open-domain QA passage retrieval."
     },
     {
       "flag": "Single model tested",
       "detail": "Only T0 3B is used as the language model. No experiments with other LLMs are conducted, making it impossible to assess whether the alignment approach generalizes across different model architectures and sizes."
     },
     {
       "flag": "Potential contamination not addressed",
       "detail": "T0 3B may have been trained on data containing the Natural Questions, TriviaQA, and Entity Questions benchmarks. This could inflate the quality of generated expansions, but contamination is never discussed."
     },
     {
       "flag": "Industry affiliation undisclosed as conflict",
       "detail": "Three of four authors work at Spotify, an IR company, and the first author was a Spotify intern. No competing interests statement is provided despite the obvious commercial relevance of the work."
     }
   ],
   "cited_papers": [
     {
       "title": "Expand, rerank, and retrieve: Query reranking for open-domain question answering",
       "authors": ["Yung-Sung Chuang", "Wei Fang", "Shang-Wen Li", "Wen-tau Yih", "James Glass"],
       "year": 2023,
       "arxiv_id": "2305.17080",
       "relevance": "Key baseline for query expansion with generate-then-filter approach, directly compared against in this paper."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Christopher D Manning", "Stefano Ermon", "Chelsea Finn"],
       "year": 2024,
       "relevance": "Core alignment technique (DPO) used in AQE, fundamental method for LLM alignment without explicit reward modeling."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Abhimanyu Dubey", "Abhinav Jauhri"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Source of the RSFT alignment technique used in AQE; relevant as a major LLM release with alignment methodology."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper for LLM alignment, establishing the reward modeling paradigm used as background for this work."
     },
     {
       "title": "Query2doc: Query expansion with large language models",
       "authors": ["Liang Wang", "Nan Yang", "Furu Wei"],
       "year": 2023,
       "arxiv_id": "2303.07678",
       "relevance": "LLM-based query expansion method addressing vocabulary mismatch, directly relevant to the survey's coverage of LLM capabilities in IR."
     },
     {
       "title": "A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions",
       "authors": ["Lei Huang", "Weijiang Yu", "Weitao Ma"],
       "year": 2023,
       "arxiv_id": "2311.05232",
       "relevance": "Survey on LLM hallucination, relevant to understanding how hallucination affects query expansion quality."
     },
     {
       "title": "Large language model based long-tail query rewriting in taobao search",
       "authors": ["Wenjun Peng", "Guiyang Li", "Yue Jiang"],
       "year": 2024,
       "relevance": "Concurrent industrial deployment of aligned query generation at Taobao, demonstrating real-world impact of the approach."
     },
     {
       "title": "Generation-augmented retrieval for open-domain question answering",
       "authors": ["Yuning Mao", "Pengcheng He", "Xiaodong Liu"],
       "year": 2020,
       "arxiv_id": "2009.08553",
       "relevance": "Introduced the GAR framework for query expansion using pre-trained language models, foundational to this line of work."
     },
     {
       "title": "Instruction Tuning With Loss Over Instructions",
       "authors": ["Zhengyan Shi", "Adam X Yang", "Bin Wu", "Laurence Aitchison", "Emine Yilmaz", "Aldo Lipani"],
       "year": 2024,
       "arxiv_id": "2405.14394",
       "relevance": "Instruction tuning methodology for LLMs, relevant to understanding alignment techniques applied to language models."
     },
     {
       "title": "Bayesian reward models for LLM alignment",
       "authors": ["Adam X Yang", "Maxime Robeyns", "Thomas Coste", "Jun Wang", "Haitham Bou-Ammar", "Laurence Aitchison"],
       "year": 2024,
       "arxiv_id": "2402.13210",
       "relevance": "Alternative reward modeling approach for LLM alignment, relevant to the survey's coverage of alignment methodologies."
     }
   ]
 }