ai-research-survey

scan.json (31147B)

---

 {
   "paper": {
     "title": "Understand What LLM Needs: Dual Preference Alignment for Retrieval-Augmented Generation",
     "authors": [
       "Guanting Dong",
       "Yutao Zhu",
       "Chenghao Zhang",
       "Zechen Wang",
       "Zhicheng Dou",
       "Ji-Rong Wen"
     ],
     "year": 2024,
     "venue": "The Web Conference",
     "arxiv_id": "2406.18676",
     "doi": "10.1145/3696410.3714717"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DPA-RAG introduces a dual preference alignment framework for RAG systems, combining an externally preference-aligned reranker (via point-wise, pair-wise, and contrastive learning) with an LLM internal self-alignment stage. The framework outperforms baselines across four QA datasets (NQ, TriviaQA, HotpotQA, WebQSP) and generalizes across multiple LLMs from 500M to 13B parameters as well as GPT-3.5 and GPT-4. Ablations show the preference-aligned reranker contributes the largest gains, and sequential pre-alignment followed by SFT outperforms mixed multi-task training.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is publicly available at https://github.com/dongguanting/DPA-RAG.' This is a definitive release, not a future promise."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All four evaluation datasets (NQ, TriviaQA, HotpotQA, WebQSP) are publicly available standard benchmarks. The paper states in Section B.1 that 'all datasets and evaluation benchmarks used in our experiments have been open-source.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section B.3 mentions 'eight A100 80g GPUs', LLaMA Factory versions 0.6.3 and 0.8.1, and AdamW optimizer, but there is no requirements.txt, Dockerfile, or comprehensive dependency listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While implementation details are provided in Section B.3 and algorithms in Appendix A, the paper lacks step-by-step reproduction instructions. No README-style commands or scripts for replicating experiments are given in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Despite reporting 'the average performance from five experiments, each with a different random seed' (Section B.3), Tables 2-5 present only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DPA-RAG 'significantly outperforms all baselines' (Section 4.2) but provides no p-values, t-tests, or any statistical significance tests. Differences are assessed by comparing raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports absolute improvements with baseline context (e.g., '+5.09' for NQ Hit@1 from 50.94 to 56.03 with LLaMA2-7B). The improvement magnitudes are shown relative to baselines throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of five random seeds, the number of test examples, or a power analysis. Standard benchmark test set sizes are used without discussion of whether they are sufficient for the claimed comparisons."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section B.3 states results are 'the average performance from five experiments, each with a different random seed,' but no standard deviation, IQR, or any spread measure is reported in any table or figure. The reader cannot assess result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 includes extensive baselines: traditional RAG with DPR across 6 LLMs, 7 reranker-based methods (RankGPT, LRL, PRP, RankLLaMA, BGE, BCEmbedding, ColBERTv2), and 6 preference-aligned methods (KnowPAT, REPLUG, RA-Judgement, RRHF, RAFT, FILCO)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include methods from 2023-2024 such as RAFT (2024), FILCO (2023), RankGPT (2023), BGE (2023), LLaMA3 (2024), and Qwen2 (2023). These are contemporary for a 2024 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 5 present detailed ablation studies removing individual components: PA-Reranker, Pre-Align, Query Augmentation, Filtering, point-wise/pair-wise/CPA alignment, and MGDA-UB optimization. Each component's contribution is individually measured."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two evaluation metrics are used throughout: Hit@1 for top-ranked accuracy and F1 score for quality/similarity to ground truth (Section 4.1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is conducted. Evaluation is entirely automated using Hit@1 and F1 metrics against ground-truth answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows explicit train/dev/test splits for all four datasets. Results in Table 2 are reported on the standard test sets for each benchmark."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (Table 2), per model size (Figure 3), per preference category (Figure 4: Both Correct, Aligned Knowledge, Unaligned Knowledge, Both Incorrect), per augmentation strategy (Table 4), and per ablation component (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 and Figure 1 discuss the 'Unaligned Knowledge' failure mode where retrieval misleads the LLM. Appendix C.3 provides detailed case studies showing baseline failures (e.g., NQ-Case1 baseline outputs 'New Westminster' incorrectly). Figure 5 discusses mixed training failure."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 (right) explicitly reports that mixed training 'leads to a noticeable performance decline and fluctuations' compared to standard QA training, identifying an approach that doesn't work. The paper also notes that some augmentation strategies are less effective than others (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims DPA-RAG 'outperforms all baselines' on four QA datasets, supported by Table 2. The claim of seamless integration with 'black-box and open-sourced LLM readers' is supported by results with GPT-3.5, GPT-4, LLaMA2, LLaMA3, and Qwen2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('DPA-RAG improves performance') are supported by controlled ablation studies (Tables 3, 5) that remove individual components and measure the effect. The ablation design with single-variable manipulation is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper calls DPA-RAG 'a universal framework designed to align diverse knowledge preferences within RAG systems' (abstract), but only tests on four English QA datasets. No testing on other languages, other knowledge-intensive tasks (e.g., summarization, dialogue), or other retrieval corpora. The 'universal' claim significantly overstates the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why DPA-RAG works. For example, improvements could stem from additional training data rather than the alignment mechanism, or from the augmented data providing a regularization effect. No robustness checks or confound analysis is presented."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Hit@1 and F1 on QA benchmarks and claims improvements in QA performance within RAG systems. The measurements match the claimed outcomes — no proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "While 'gpt-3.5-turbo-0613' is specified for data augmentation (Section C.2), the main evaluation uses only 'GPT-3.5' and 'GPT-4' without snapshot dates or API versions (Table 2). LLaMA2-7B/13B and LLaMA3-8B are specified by size but not by exact checkpoint or release version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Section B.2: the SFT stage prompt, the pre-aligned stage prompt with judgment instruction, and the query augmentation prompt. These include the actual text with placeholders whose fill values are defined."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section B.3 reports: batch size 16 (reranker) and 128 (QA), learning rates 1e-5 and 7e-5, 10 and 3 epochs, AdamW optimizer, 3% warmup, temperature 1.0 for augmentation, a=0.8 for scoring weight, top-k=3 documents."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. DPA-RAG is a retriever-reranker-reader pipeline with fine-tuning, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the preference knowledge construction pipeline in detail: hierarchical document sampling (positions 1, 25, 50, 100), four-category classification, five augmentation strategies, NLI quality filtering removing ~20% of augmented data. Algorithm 1 formalizes the full process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section in the paper. The conclusion (Section 5) is a brief summary of contributions with no discussion of limitations or threats to validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no consideration of specific methodological weaknesses or potential confounds."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state what it does NOT show. It claims 'universal' applicability without stating boundaries — no mention of untested languages, task types, retrieval corpora, or settings where the approach might not work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark datasets are public, the constructed preference dataset, per-run experimental outputs, and individual seed results are not made available. The reader cannot verify the reported averaged numbers."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes in detail how preference data is collected: direct vs. referencing document comparisons on GPT-3.5, four-category classification, hierarchical sampling of documents at positions 1, 25, 50, 100. Section B.1 describes all four benchmark datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public QA benchmarks (NQ, TriviaQA, HotpotQA, WebQSP)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 provides the complete pipeline from training data to preference dataset construction including extraction, augmentation, and NLI filtering. Algorithm 2 documents the reader training pipeline. Section 3.2 states the preference dataset is ~20% of the training set."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper text. The paper does not disclose whether the work was funded."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Gaoling School of Artificial Intelligence at Renmin University of China, and Beijing University of Posts and Telecommunications."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Absence of disclosure is not the same as absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (GPT-3.5, GPT-4, LLaMA2, LLaMA3, Qwen2). This is needed to assess whether benchmark test data could be in the training corpus."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between benchmark test sets (NQ, TriviaQA, HotpotQA, WebQSP) and the pre-training data of the LLMs used. These benchmarks all predate the models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NQ (2019), TriviaQA (2017), HotpotQA (2018), and WebQSP (2016) were all publicly available well before the training cutoffs of GPT-3.5, GPT-4, LLaMA2/3, and Qwen2. No contamination analysis or mitigation is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments use automated evaluation on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example API cost is reported. The system involves retrieval, reranking, and LLM generation but no cost figures are given for any stage."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section B.3 mentions 'eight A100 80g GPUs' for training but does not quantify total GPU hours, wall-clock training time, or API costs for GPT-3.5/GPT-4 experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section B.3 states 'the average performance from five experiments, each with a different random seed,' but no per-seed results, standard deviations, or sensitivity analysis is reported. The reader cannot assess how much results vary across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section B.3 explicitly states: 'We report the average performance from five experiments, each with a different random seed.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Specific hyperparameters are reported (Section B.3) but the paper does not describe how they were selected — no search budget, search method, or number of configurations tried. The scoring weight a=0.8 is noted as following prior work without grid search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final hyperparameter configuration was selected. Learning rates, batch sizes, and epochs appear chosen but the selection process is not described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 4 datasets, 6+ reader models, and 13+ baselines without any correction for multiple comparisons (Bonferroni, Holm, etc.)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own baselines (Table 2) without acknowledging the well-documented bias that authors' re-implementations of baselines systematically underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "DPA-RAG adds a fine-tuned reranker and a pre-alignment training stage on top of standard RAG. The additional compute cost is never compared against the performance gains, nor are baselines compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses NQ, TriviaQA, HotpotQA, and WebQSP as QA benchmarks without discussing whether these benchmarks adequately measure the claimed capability of 'preference alignment' in RAG systems."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. DPA-RAG is a retriever-reranker-reader pipeline, not an agent scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All four benchmarks (NQ 2019, TriviaQA 2017, HotpotQA 2018, WebQSP 2016) predate the models used (GPT-3.5/4, LLaMA2/3, Qwen2). The models may have seen solutions in training. This is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the retrieval setup or the preference knowledge construction pipeline introduces information leakage between train and test."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples share structural similarities or come from overlapping sources (e.g., Wikipedia articles appearing in both retrieval corpus and test answers)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "DPA-RAG significantly outperforms all baselines across four QA datasets with different LLM readers.",
       "evidence": "Table 2 shows DPA-RAG with LLaMA2-7B achieving +5.09 Hit@1 on NQ, +6.26 on TriviaQA, +3.83 on HotpotQA, +3.88 on WebQSP over RAG baseline. Improvements are consistent across GPT-3.5, GPT-4, LLaMA2-13B, LLaMA3-8B, and Qwen2-7B.",
       "supported": "moderate"
     },
     {
       "claim": "The preference-aligned reranker provides the largest performance contribution among DPA-RAG's components.",
       "evidence": "Table 3 ablation shows removing PA-Reranker causes the largest drop (-3.23 Hit@1 on NQ, -3.64 on TQA), larger than removing Pre-Align (-1.72, -2.21) or Query Augmentation (-2.13, -2.62).",
       "supported": "moderate"
     },
     {
       "claim": "Integrating both external and internal alignment yields a mutually reinforcing effect beyond additive gains.",
       "evidence": "Table 3 shows combined removal (w/o Pre-Align + PA-Rerank) causes -4.12 on NQ, while individual removals sum to -4.95, suggesting some synergy. However this is only shown on 2 datasets.",
       "supported": "weak"
     },
     {
       "claim": "Sequential training (pre-alignment then SFT) outperforms mixed multi-task training.",
       "evidence": "Figure 5 (right) shows mixed training leads to 'a noticeable performance decline and fluctuations' while sequential training yields stable gains on NQ. Only demonstrated on one dataset.",
       "supported": "moderate"
     },
     {
       "claim": "Query augmentation effectiveness correlates with data complexity and diversity scores.",
       "evidence": "Table 4 and Figure 5 (left) show that augmentation strategies with higher complexity+diversity scores (Complexity: 2.33, NQ 54.81) outperform those with lower scores (Rephrasing: 2.03, NQ 52.27). A monotonic trend is observed.",
       "supported": "moderate"
     },
     {
       "claim": "DPA-RAG delivers stable performance improvements as model parameter size increases from 500M to 14B.",
       "evidence": "Figure 3 shows DPA-RAG improvements remain relatively consistent across Qwen1.5-0.5B through Qwen1.5-14B on both HQA and TQA datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Retrieved documents often fail to match the knowledge preferences of LLMs, with high-similarity documents sometimes misleading the model.",
       "evidence": "Figure 1 preliminary analysis on GPT-3.5 across NQ, TriviaQA, and HotpotQA shows 'Unaligned Knowledge' cases where high-similarity documents mislead, and 'Aligned Knowledge' cases where low-similarity documents (100th) still help.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "The paper averages over 5 random seeds but never reports standard deviation, confidence intervals, or any spread measure. Without variance, the reader cannot assess whether reported improvements are within noise margins. Given the small absolute differences in some comparisons (e.g., +1.44 on WebQSP Hit@1 for LLaMA3), this is a significant omission."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'significantly outperforms' are made based on comparing raw numbers without any statistical tests. With many comparisons across 4 datasets, 6+ readers, and 13+ baselines, some apparent improvements could be due to chance."
     },
     {
       "flag": "Overclaiming with 'universal' label",
       "detail": "The paper calls DPA-RAG 'a universal framework' but only evaluates on English-language QA tasks with Wikipedia-based retrieval. No testing on other languages, other task types (summarization, dialogue, code), or other knowledge sources."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations discussion. This is unusual for a venue like The Web Conference and means known weaknesses (e.g., dependence on GPT-3.5 for preference scoring, generalization beyond QA) are never acknowledged."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "All four benchmarks (NQ, TriviaQA, HotpotQA, WebQSP) were published years before the training cutoffs of GPT-3.5/4 and LLaMA2/3. The baseline LLMs may have memorized test answers, confounding the preference alignment evaluation."
     },
     {
       "flag": "Self-comparison bias",
       "detail": "Authors implement all baselines themselves (Table 2) without acknowledging the well-documented bias that authors' re-implementations of competing methods tend to underperform. No independent evaluation or use of official baseline implementations is confirmed."
     }
   ],
   "cited_papers": [
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper for aligning LLMs with human preferences, which DPA-RAG extends to retrieval alignment."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2023,
       "relevance": "DPO is the dominant alternative to RLHF for preference alignment; DPA-RAG's pair-wise alignment draws on similar principles."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper that established the retrieve-then-read paradigm that DPA-RAG extends."
     },
     {
       "title": "REPLUG: Retrieval-augmented black-box language models",
       "authors": ["Weijia Shi", "Sewon Min", "Michihiro Yasunaga"],
       "year": 2023,
       "arxiv_id": "2301.12652",
       "relevance": "Baseline method for aligning retrievers with black-box LLMs using output probability, directly compared with DPA-RAG."
     },
     {
       "title": "RAFT: adapting language model to domain specific RAG",
       "authors": ["Tianjun Zhang", "Shishir G. Patil", "Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.10131",
       "relevance": "Baseline that trains LLMs to ignore irrelevant retrieved documents, directly compared with DPA-RAG."
     },
     {
       "title": "Learning to filter context for retrieval-augmented generation",
       "authors": ["Zhiruo Wang", "Jun Araki", "Zhengbao Jiang"],
       "year": 2023,
       "arxiv_id": "2311.08377",
       "relevance": "FILCO baseline that filters training context using data selection methods, a strong competing approach to DPA-RAG."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Primary open-source LLM used as reader in DPA-RAG experiments across multiple model sizes."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Black-box LLM reader used to demonstrate DPA-RAG's cross-model generalization capability."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique for LLM reasoning, relevant to how LLMs process retrieved knowledge."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "relevance": "Agent framework combining reasoning and retrieval, relevant to multi-round retrieval paradigm in RAG alignment."
     },
     {
       "title": "Knowledgeable preference alignment for LLMs in domain-specific question answering",
       "authors": ["Yichi Zhang", "Zhuo Chen", "Yin Fang"],
       "year": 2024,
       "relevance": "KnowPAT baseline for aligning LLM preferences with knowledge, directly compared with DPA-RAG."
     },
     {
       "title": "Investigating the factual knowledge boundary of large language models with retrieval augmentation",
       "authors": ["Ruiyang Ren", "Yuhao Wang", "Yingqi Qu"],
       "year": 2023,
       "arxiv_id": "2307.11019",
       "relevance": "RA-Judgement baseline that explores knowledge boundaries of RAG, directly compared with DPA-RAG."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RAG is widely deployed in production systems; the reranker alignment approach is implementable with released code, though requires training infrastructure."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that retrieved documents don't match LLM preferences is somewhat expected in the RAG community; the dual alignment approach is incremental rather than surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on improving QA accuracy in RAG systems."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; this is a straightforward method paper with incremental improvements."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but requires significant setup (training reranker, fine-tuning LLMs on 8 A100 GPUs) — not easily demoed."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Renmin University is well-known in NLP research but not a household name; paper uses GPT-3.5/4 as baselines but is not from OpenAI."
     }
   }
 }