ai-research-survey

scan.json (29577B)

---

 {
   "paper": {
     "title": "SEAKR: Self-aware Knowledge Retrieval for Adaptive Retrieval Augmented Generation",
     "authors": [
       "Zijun Yao",
       "Weijian Qi",
       "Liangming Pan",
       "Shulin Cao",
       "Linmei Hu",
       "Weichuan Liu",
       "Lei Hou",
       "Juanzi Li"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2406.19215",
     "doi": "10.48550/arXiv.2406.19215"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SEAKR achieves 36.0%, 39.7%, and 23.5% F1 on 2WikiMultiHop, HotpotQA, and IIRC complex QA benchmarks using LLaMA-2-chat 7B, outperforming the best baseline (DRAGIN) by up to 6.0 percentage points. Ablation study reveals that self-aware re-ranking (knowledge integration) contributes more performance gain than self-aware retrieval (deciding when to retrieve), highlighting the importance of adaptive knowledge integration in RAG. The method leverages Gram determinant of internal hidden representations as an uncertainty measure, outperforming output-level alternatives (perplexity, prompting, energy scores), and scales positively from LLaMA-2 (7B) to LLaMA-3 (8B).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'We release our code in our Github repository' but no URL or repository link is provided anywhere in the paper text or appendices."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All six datasets (2WikiMultiHopQA, HotpotQA, IIRC, NaturalQuestions, TriviaQA, SQuAD) are publicly available standard benchmarks. The Wikipedia dump (December 20, 2018) is also public."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C states vLLM 0.4.2, PyTorch 2.3.0, Elasticsearch 7.17.9. Limitations section specifies NVidia 3090 GPU with 24GiB GRAM. LLaMA-2-chat 7B is the backbone."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided in the paper. The paper describes the method but not how to execute the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-4 report only point estimates (e.g., '36.0% F1') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'SEAKR outperforms existing adaptive RAG methods' based solely on comparing point estimates across tables. No statistical significance tests (p-values, t-tests, bootstrap, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 reports improvements with baseline context: 'outperforms the best baselines by 6.0%, 5.5%, and 0.6%' on the three complex QA datasets, with all baseline numbers shown in Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Standard benchmark test/dev splits are used without justification. For analysis experiments, Section 5 states 'sample 500 questions from each dataset to reduce the cost' but no power analysis or formal sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 lists five baselines: CoT, IRCoT, Self-RAG, FLARE, and DRAGIN, covering both non-adaptive and adaptive RAG methods."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DRAGIN (2024), Self-RAG (2023), FLARE (2023), and IRCoT (2022), which are the most recent adaptive RAG methods at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 provides extensive ablation: removing self-aware retrieval, self-aware re-ranking, self-aware reasoning, and substituting alternative uncertainty estimators (prompting, perplexity, multi-perplexity, LN-entropy, energy). Results in Table 3."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "All experiments report both Exact Match (EM) and F1 score across all six datasets (Tables 1 and 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. All evaluation is automated via EM and F1 metrics. Human evaluation of reasoning chain quality could have added value."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 states hyperparameters are searched on NQ's training split, while results are reported on separate official development sets (complex QA following IRCoT, simple QA following DPR). The evaluation sets are not used for tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across all six datasets individually (3 complex QA in Table 1, 3 simple QA in Table 2), and ablation results are shown per-dataset in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Table 5 show a case study where the LLM generates an incorrect pseudo-generation (wrong birthday for Jodorowsky) but SEAKR correctly identifies high uncertainty and retrieves corrective knowledge. Additional cases in Appendix A show the system's behavior."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 acknowledges limited improvement on IIRC due to numerical reasoning difficulty. The ablation study (Table 3) shows degradation when components are removed. Section 4.2.2 notes SEAKR lags behind Self-RAG on NQ."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'SEAKR outperforms existing adaptive RAG methods' on 'both complex and simple Question Answering datasets.' However, on NaturalQuestions (simple QA), Self-RAG achieves 40.2 F1 vs SEAKR's 35.5 F1 (Table 2), contradicting the claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made through controlled ablation studies (Table 3) that remove individual components (self-aware retrieval, re-ranking, reasoning) while holding other components fixed. This controlled single-variable manipulation adequately supports causal attribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds scope: only open-source LLMs (requires internal states), only short-form QA (not long-form or creative writing), only models up to 8B parameters. These specific boundaries are stated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the ~20x additional compute (20 pseudo-generations per step) could explain performance gains vs baselines. No confounds are discussed. The limitations section discusses computational cost but not as an alternative explanation for results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures EM and F1 on QA benchmarks and claims QA performance improvements. The measurements match the granularity of the claims without broader framing (no claims about 'intelligence' or 'understanding')."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 specifies 'LLaMA-2-chat with 7 billion parameters' and Table 4 lists 'LLaMA-3 with 8B Parameters' with base vs chat/instruct versions. These are specific, fixed-weight open-source model releases."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides full prompt templates for self-aware retrieval (B.1), re-ranking (B.2), and reasoning (B.3). Figures 4-7 provide all in-context learning examples for all six datasets. A reader can reconstruct every prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1.3 reports N=3 (knowledge recall), k=20 (Gram determinant samples), l=L/2 (layer), 10 ICL examples. However, LLM sampling parameters (temperature, top-p) for the 20 pseudo-generations are not stated, and these significantly affect the uncertainty estimation."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides a detailed system diagram. Sections 3.1-3.4 describe the full pipeline: self-aware retrieval (uncertainty threshold → retrieval decision), query generation, re-ranking (pairwise uncertainty comparison), and reasoning strategy selection."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 describes dataset selection and open-domain QA setting. Section 4.1.3 specifies Wikipedia dump date (December 20, 2018), BM25 search engine, and dataset splitting conventions (following IRCoT and DPR). Table 6 provides dataset statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section with five numbered points covers scope of usage, task coverage, computation issues, model scaling, and information retrieval."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this study: (1) requires access to internal states, limiting to open-source LLMs; (2) only tested on short-form QA; (3) 20 pseudo-generations are computationally costly; (4) maximum 8B parameter models tested; (5) self-aware re-ranking could be surpassed by advanced IR methods."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what was NOT tested: closed-source LLMs (GPT series), long-form QA, creative writing, models larger than 8B. Task coverage limitation (point 2) directly states 'neglecting a broad spectrum of natural language processing tasks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark datasets are publicly available, the paper's experimental outputs (model predictions, uncertainty scores, intermediate reasoning chains) are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 describes each dataset, Section 4.1.3 specifies the Wikipedia dump (December 20, 2018), search engine (BM25/ElasticSearch), and dataset splitting conventions. Table 6 provides dataset statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The overall experimental setup is described but intermediate processing steps are not documented. No details on how Wikipedia was indexed, how benchmark questions were preprocessed, or how final metrics were computed from raw model outputs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information appears anywhere in the paper. One author is from Siemens Technology but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed in the header: Tsinghua University, UC Santa Barbara, Beijing Institute of Technology, and Siemens Technology. The paper does not evaluate products from these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The Siemens affiliation is noted but no funding relationship is described."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state when LLaMA-2's training data was collected. LLaMA-2 was trained on data with a cutoff around September 2022, but this is not mentioned in the paper."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether benchmark questions (HotpotQA 2018, NQ 2019, SQuAD 2016, etc.) appeared in LLaMA-2's training data. These benchmarks predate the model and overlap risk is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All six benchmarks (2018 or earlier for most) were publicly available well before LLaMA-2's training. No contamination analysis or discussion is provided despite this clear risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on QA benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper includes an Ethical Considerations section but it discusses potential misuse, not IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions 20 pseudo-generations are 'computationally costly' and vLLM parallelization reduces latency, but no actual inference time, cost per query, or tokens consumed is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states experiments run on 'a single NVidia 3090 GPU with 24GiB GRAM' but does not report total GPU hours, wall-clock time, or computational budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many experimental runs produced the reported results. No mention of 'averaged over K runs' or similar."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.3 and Figure 3 show hyperparameter search results for N, k, δ, and layer selection on NQ training data. However, the total number of configurations tried and total compute spent on search are not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 describes hyperparameter selection on NQ's training split (separate from evaluation data). The search process and selection criteria are shown in Figure 3 with clear rationale for each choice."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement FLARE with IRCoT strategy for complex QA evaluation. No acknowledgment that their re-implementation of baselines could systematically underperform the original authors' implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SEAKR requires 20 pseudo-generations per reasoning step for uncertainty estimation, roughly 20x the compute of baselines like FLARE/DRAGIN. This compute difference is mentioned in limitations but performance is never compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses six QA benchmarks without discussing whether EM/F1 on these benchmarks actually measures the adaptive RAG capabilities claimed. No construct validity analysis is provided."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 states 'These choices and constraints are also applied to all our baseline methods for fair comparison,' using the same backbone LLM (LLaMA-2-chat 7B), search engine (BM25/ElasticSearch), and Wikipedia dump across all methods."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All six benchmarks (SQuAD 2016, TriviaQA 2017, HotpotQA 2018, NQ 2019, 2WikiMultiHopQA 2020, IIRC 2020) predate LLaMA-2's training. The model may have seen benchmark solutions during training, but this temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through context or whether in-context learning examples provide inappropriate hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples share structural similarities or whether the NQ training split used for hyperparameter search shares properties with other evaluation datasets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "SEAKR outperforms existing adaptive RAG methods on complex QA benchmarks, achieving 36.0%, 39.7%, and 23.5% F1 on 2WikiMultiHop, HotpotQA, and IIRC respectively.",
       "evidence": "Table 1 (Section 4.2.1) shows SEAKR's F1 scores exceed the best baseline (DRAGIN at 30.0%, 34.2%, 22.9%) by 6.0, 5.5, and 0.6 percentage points.",
       "supported": "moderate"
     },
     {
       "claim": "Self-aware re-ranking contributes more to performance than self-aware retrieval in adaptive RAG.",
       "evidence": "Table 3 ablation (Section 5.1): removing re-ranking causes larger drops (−2.8 F1 on 2Wiki, −1.5 on HPQA) than removing retrieval (−2.1 on 2Wiki, −0.5 on HPQA). Authors state 'ablating self-aware re-ranking reduces the performance of SEAKR more than removing self-aware retrieval.'",
       "supported": "moderate"
     },
     {
       "claim": "Internal-state uncertainty estimation outperforms output-level alternatives for adaptive RAG.",
       "evidence": "Table 3 compares SEAKR's Gram determinant approach against prompting-based, perplexity, multi-perplexity, LN-entropy, and energy score alternatives. SEAKR achieves 37.8 F1 on 2Wiki vs best alternative (LN-Entropy) at 36.0.",
       "supported": "moderate"
     },
     {
       "claim": "SEAKR scales positively with more powerful backbone LLMs.",
       "evidence": "Table 4 (Section 5.2) shows improvement from LLaMA-2 (7B) to LLaMA-3 (8B): 37.8→48.1 F1 on 2Wiki, 38.1→47.7 on HPQA, 36.1→43.0 on NQ.",
       "supported": "moderate"
     },
     {
       "claim": "SEAKR is effective on both complex and simple QA tasks.",
       "evidence": "Tables 1-2 show strong results on complex QA but mixed results on simple QA: SEAKR leads on TriviaQA (63.1 F1) and SQuAD (36.5 F1) but trails Self-RAG on NQ (35.5 vs 40.2 F1).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All results across Tables 1-4 are single-run point estimates with no variance, confidence intervals, or significance tests. Performance differences (e.g., 0.6% F1 improvement on IIRC) could be within random variation."
     },
     {
       "flag": "Abstract overclaims",
       "detail": "The abstract states SEAKR 'outperforms existing adaptive RAG methods' on 'both complex and simple QA datasets,' but Self-RAG outperforms SEAKR on NaturalQuestions (40.2 vs 35.5 F1, Table 2)."
     },
     {
       "flag": "No contamination analysis",
       "detail": "All six benchmarks (2016-2020) predate LLaMA-2's training data. The model may have memorized benchmark answers, inflating all results. No contamination check or temporal analysis is performed."
     },
     {
       "flag": "Uncontrolled compute overhead as confound",
       "detail": "SEAKR requires 20 pseudo-generations per reasoning step for uncertainty estimation, roughly 20x the compute of single-generation baselines like FLARE/DRAGIN. Performance improvement could partly stem from this additional compute budget rather than the self-aware mechanism itself."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection",
       "authors": ["Akari Asai", "Zeqiu Wu", "Yizhong Wang", "Avirup Sil", "Hannaneh Hajishirzi"],
       "year": 2023,
       "relevance": "Adaptive RAG method that fine-tunes LLMs with special retrieval/critique tokens; direct baseline comparison for SEAKR."
     },
     {
       "title": "Active Retrieval Augmented Generation",
       "authors": ["Zhengbao Jiang", "Frank F Xu", "Luyu Gao", "Zhiqing Sun", "Qian Liu", "Jane Dwivedi-Yu", "Yiming Yang", "Jamie Callan", "Graham Neubig"],
       "year": 2023,
       "relevance": "FLARE triggers retrieval on low-probability tokens; key adaptive RAG baseline that SEAKR aims to improve upon."
     },
     {
       "title": "DRAGIN: Dynamic Retrieval Augmented Generation Based on the Real-time Information Needs of Large Language Models",
       "authors": ["Weihang Su", "Yichen Tang", "Qingyao Ai", "Zhijing Wu", "Yiqun Liu"],
       "year": 2024,
       "relevance": "Dynamic RAG using attention weights for retrieval decisions; strongest baseline that SEAKR outperforms on complex QA."
     },
     {
       "title": "Interleaving Retrieval with Chain-of-Thought Reasoning for Knowledge-intensive Multi-step Questions",
       "authors": ["Harsh Trivedi", "Niranjan Balasubramanian", "Tushar Khot", "Ashish Sabharwal"],
       "year": 2022,
       "relevance": "IRCoT combines CoT reasoning with retrieval for multi-hop QA; foundational retrieval-augmented reasoning method and baseline."
     },
     {
       "title": "INSIDE: LLMs' Internal States Retain the Power of Hallucination Detection",
       "authors": ["Chao Chen", "Kai Liu", "Ze Chen", "Yi Gu", "Yue Wu", "Mingyuan Tao", "Zhihang Fu", "Jieping Ye"],
       "year": 2023,
       "relevance": "Key methodological foundation for SEAKR's uncertainty estimation using Gram determinant of internal hidden representations."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "Fabio Petroni", "Vladimir Karpukhin", "Naman Goyal"],
       "year": 2020,
       "relevance": "Foundational RAG paper combining retrieval with generation for knowledge-intensive tasks."
     },
     {
       "title": "Self-knowledge Guided Retrieval Augmentation for Large Language Models",
       "authors": ["Yile Wang", "Peng Li", "Maosong Sun", "Yang Liu"],
       "year": 2023,
       "relevance": "Trains a classifier to judge factuality of LLM generation for adaptive retrieval; alternative approach to SEAKR's internal-state method."
     },
     {
       "title": "Language Models (Mostly) Know What They Know",
       "authors": ["Saurav Kadavath", "Tom Conerly", "Amanda Askell"],
       "year": 2022,
       "arxiv_id": "2207.05221",
       "relevance": "Demonstrates LLMs are aware of their uncertainty, foundational motivation for SEAKR's self-aware uncertainty approach."
     },
     {
       "title": "The Internal State of an LLM Knows When Its Lying",
       "authors": ["Amos Azaria", "Tom Mitchell"],
       "year": 2023,
       "relevance": "Shows internal states can detect factuality, supporting the premise that LLM internals can guide retrieval decisions."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Fei Xia", "Ed Chi", "Quoc V Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "CoT reasoning is the backbone reasoning strategy that SEAKR augments with self-aware retrieval."
     },
     {
       "title": "Probabilistic Tree-of-Thought Reasoning for Answering Knowledge-intensive Complex Questions",
       "authors": ["Shulin Cao", "Jiajie Zhang", "Jiaxin Shi", "Xin Lv", "Zijun Yao"],
       "year": 2023,
       "relevance": "ProbTree decomposes complex questions into sub-questions solved with RAG; retrieval-augmented reasoning approach."
     },
     {
       "title": "Adaptive-RAG: Learning to Adapt Retrieval-Augmented Large Language Models through Question Complexity",
       "authors": ["Soyeong Jeong", "Jinheon Baek", "Sukmin Cho", "Sung Ju Hwang", "Jong C Park"],
       "year": 2024,
       "arxiv_id": "2403.14403",
       "relevance": "Adapts RAG strategy based on question complexity; closely related adaptive RAG approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Method is applicable to QA systems using open-source LLMs, but requires access to model internal states which limits deployment to local model serving."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Novel application of internal-state uncertainty for RAG decisions, but the general idea that LLM internals contain useful signals is well-established."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk implications; the paper addresses factual accuracy improvement in QA."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or challenges to established institutions or practices."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code claimed to be released but no URL provided; requires a GPU and local LLM deployment to use."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Tsinghua University is well-known in AI research but not a household name like OpenAI or Google DeepMind."
     }
   }
 }