ai-research-survey

scan.json (30139B)

---

 {
   "paper": {
     "title": "PlanRAG: A Plan-then-Retrieval Augmented Generation for Generative Large Language Models as Decision Makers",
     "authors": [
       "Myeonghwa Lee",
       "Seonho An",
       "Min-Soo Kim"
     ],
     "year": 2024,
     "venue": "North American Chapter of the Association for Computational Linguistics",
     "arxiv_id": "2406.12430",
     "doi": "10.48550/arXiv.2406.12430"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We release our code and benchmark at https://github.com/myeon9h/PlanRAG' and provides a working URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The DQA benchmark (301 question-database pairs with both RDB and GDB versions) is released at the same GitHub repository. Game simulators for annotation are also released under the MIT license (Section 9)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided in the paper. They mention using LangChain, MySQL, Neo4j, and vLLM but do not specify versions or provide a reproducible environment file."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. While code is released, the paper itself does not contain a 'Reproducing Results' section or describe how to run experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimate accuracies (e.g., '64.3%' in Table 4) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PlanRAG 'outperforms' and 'significantly improves' over IterRAG based solely on comparing accuracy numbers (e.g., 64.3% vs 48.5%) without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Improvements are reported with baseline context: '15.8% in the Locating scenario and by 7.4% in the Building scenario' over IterRAG (Table 4). Per-category breakdowns (Tables 5, 6, Figure 5) provide baseline-relative context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 200 Locating and 101 Building pairs (Table 3) with no justification for why these sizes were chosen. No power analysis or discussion of whether these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 explicitly states 'All experiments are conducted in a zero-shot and single run setting.' No variance, standard deviation, or spread measure is reported across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 compares four methods: SingleRAG-LM, IterRAG-LM, PlanRAG-LM, and PlanRAG-LM w/o RP. IterRAG-LM represents the state-of-the-art iterative RAG technique."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The iterative RAG baseline is based on ReAct (Yao et al., 2023) and references Trivedi et al. (2023) and Jiang et al. (2023b), all contemporary methods. All baselines use GPT-4."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "PlanRAG-LM w/o RP (without re-planning) in Table 4 ablates the re-planning component, showing 10.8% drop in Locating and 0.9% in Building. Appendix A.6.2 (Table 10) ablates prompt structure variations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (exact match with ground-truth decision) is used as an evaluation metric. No alternative metrics such as latency, cost per question, partial credit, or ranking quality are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via semantic matching to ground-truth answers computed by game simulators. No human evaluation of answer quality, reasoning quality, or plan quality is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The prompt structure was selected using 10% of questions sampled from each scenario (Table 10, Appendix A.6.2), but the main results in Table 4 are on the full benchmark including that 10%. No held-out test set is maintained."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by scenario (Locating vs Building, Table 4), database type (RDB vs GDB, Table 5), question difficulty (SR vs MR, Figure 5), error category (Figure 6), and re-planning frequency (Table 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 provides detailed failure case analysis with five error categories (CAN, MIS, DEEP, QUR, OTH) and Figure 6 shows the distribution. Specific examples are given in Figure 4(a)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "PlanRAG-LM has slightly more DEEP errors than IterRAG-LM (Figure 6). Re-planning accuracy degrades with more re-planning iterations (Table 7). PlanRAG with GPT-3.5 performs worse than IterRAG (Table 11). Llama 2 and Phi-2 achieve 0% on all conditions."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '15.8% improvement in Locating and 7.4% in Building over state-of-the-art iterative RAG' which matches Table 4 (64.3-48.5=15.8 for Locating, 45.0-37.6=7.4 for Building)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PlanRAG 'significantly enhances the capability of decision making' (causal). The ablation (w/o RP) partially supports re-planning's effect, but the main comparison between PlanRAG and IterRAG confounds planning with prompt structure changes — Table 10 shows prompt structure alone affects accuracy substantially. The compute/token difference between methods is also uncontrolled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Generative Large Language Models as Decision Makers' broadly, but results are on only two video game scenarios using only GPT-4 (other models achieve 0%). The paper motivates with Pfizer business examples but tests only on game-derived data."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for PlanRAG's improvement: whether it's the planning structure, the additional prompt tokens, the extra retrieval iterations, or simply having a more verbose reasoning format. The prompt structure comparison in Table 10 hints at this but is not discussed as an alternative explanation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames video game decision accuracy as evidence that LLMs can serve as business decision makers (Introduction motivates with Pfizer examples), but does not acknowledge the gap between game-derived benchmarks and actual business decision making. The Limitations section discusses database type scope but not the game-to-business proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The main experiments use 'GPT-4 (OpenAI, 2023)' without a version specifier or snapshot date. Appendix A.7 footnote 9 specifies 'gpt-3.5-turbo-0125' for the GPT-3.5 variant, but the primary GPT-4 model lacks version specification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendix A.6: Figures 8-11 show complete prompts for SingleRAG, IterRAG, and PlanRAG on the GDB Locating scenario, including format instructions and tool descriptions. Business rules are provided in Figure 7."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'GPT-4 with a zero temperature.' Appendix A.7 states 'temperature to zero and 0.1 for GPT-3.5-turbo and other open models, respectively.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The PlanRAG architecture is described in detail in Section 4 with Figure 3: planning, retrieving & answering, and re-planning steps. The LangChain interface, SQL/Cypher query execution, and ReAct-based prompt structure are all documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes data collection from game savefiles via parser, storage according to DB schema, question generation via templates, and quality control (omitting low-TPcountry countries, filtering multiple-answer Building problems)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' contains three substantive paragraphs discussing database type scope, low-level method scope, and single-LM architecture limitation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section discusses what wasn't covered (other database types, fine-tuned models, multi-LM frameworks) but does not address specific threats to the validity of the reported results, such as single-run reliability, game-to-business generalization, or GPT-4-specific effects."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states: focused on graph/relational databases only, does not focus on low-level methods like fine-tuned Cypher generators, and PlanRAG was implemented using a single LM only. These are specific boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark databases, questions, and game simulators are released at the GitHub repository. The simulators allow deterministic reproduction of ground-truth annotations (Appendix A.3-A.4 provide full algorithms)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes selecting savefiles, extracting data via game savefile parser, storing in DB schema, generating questions from templates, and quality control criteria for both scenarios."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from video game savefiles and deterministic simulators. This is a standard benchmark construction, not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from game savefiles → parser → DB schema → question templates → quality filtering is documented in Appendix A.1. Simulator algorithms are provided in Appendix A.4. Statistics of the resulting data are in Table 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists NRF grant No. 2018R1A5A1060031 and RS-2023-00281635, and IITP grant No. 2019-0-01267, all funded by the Korean government (MSIT)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with School of Computing, KAIST. As an academic institution, there is no product conflict with the evaluated techniques."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from the Korean National Research Foundation and IITP, government agencies with no financial interest in PlanRAG's performance outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 to answer questions about Europa Universalis IV and Victoria 3 game mechanics but does not state GPT-4's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Europa Universalis IV and Victoria 3 have extensive online documentation (wikis, forums, strategy guides) that GPT-4 was likely trained on. The paper does not discuss whether this prior knowledge could inflate performance."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the specific DQA database instances are new, the game mechanics, business rules, and strategies are extensively documented online. GPT-4 likely has knowledge of EU4/V3 trade and building mechanics. This contamination vector is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated via game simulators."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Ethical Considerations section (Section 9) discusses game license compliance and historical sensitivity, not human subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or per-question latency are reported despite using GPT-4 API for 602 question-database pairs across multiple methods with iterative retrieval."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.7 mentions 'eight Nvidia A100 (80GB) GPUs' for open model inference, but no total GPU hours, API spend, or wall-clock time for the main GPT-4 experiments is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 'All experiments are conducted in a zero-shot and single run setting.' No seed sensitivity analysis is performed despite using temperature 0.1 for open models."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly states 'single run setting,' making it clear that each result comes from exactly one run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Table 10 shows prompt structure was explored with four variants on 10% of questions, but no formal search budget, number of total configurations tried, or compute spent on this search is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The prompt structure was selected based on Table 10 results on 10% of questions, but this selection subset is included in the final evaluation (Table 4 reports on the full benchmark). The selection is not on a separate validation set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both their own PlanRAG and the baseline IterRAG. They do not acknowledge the author-evaluation bias where authors' implementations of baselines may systematically underperform (Lucic et al., 2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "PlanRAG involves additional planning and re-planning iterations compared to IterRAG, consuming more tokens and API calls, but this compute difference is not quantified or discussed as a confound."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The DQA benchmark is derived from Europa Universalis IV and Victoria 3 video games as a proxy for real business decision making. The paper claims these games 'well imitate real business situations' but provides no evidence for this construct validity claim."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PlanRAG uses a fundamentally different prompt structure (Plan-Thought-Action-Observation-Replan) compared to IterRAG (Thought-Action-Observation). Table 10 shows prompt structure alone significantly affects accuracy, but this confound is not addressed when interpreting the main results."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 was likely trained on extensive EU4 and Victoria 3 game mechanics documentation, strategy guides, and wiki pages. The paper does not discuss whether this temporal overlap could help the model on DQA questions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The business rules R are provided as input to the model. Whether the model also benefits from pre-existing knowledge of these game mechanics (feature leakage from training data) is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple questions share the same database D (Section A.2: 'Some instances share the same database D, but not all use the same one'). Non-independence between questions sharing a database is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference, or decontamination analysis is performed."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "claims": [
     {
       "claim": "PlanRAG outperforms the state-of-the-art iterative RAG by 15.8% on Locating and 7.4% on Building scenarios.",
       "evidence": "Table 4 shows PlanRAG-LM achieves 64.3% on Locating (vs 48.5% for IterRAG-LM) and 45.0% on Building (vs 37.6% for IterRAG-LM). Single run, zero-shot, GPT-4.",
       "supported": "moderate"
     },
     {
       "claim": "Re-planning is important for decision making, improving accuracy by 10.8% on Locating and 0.9% on Building.",
       "evidence": "Table 4 ablation: PlanRAG-LM w/o RP achieves 53.5% vs 64.3% on Locating and 44.1% vs 45.0% on Building. Table 7 shows re-planning statistics.",
       "supported": "moderate"
     },
     {
       "claim": "PlanRAG reduces the rate of missed data analysis compared to IterRAG (1.3% vs 3.3% on Locating, 21.8% vs 33.2% on Building).",
       "evidence": "Table 6 directly measures missed analysis rates for critical values (IV, TPtotal for Locating; CO, PD for Building).",
       "supported": "moderate"
     },
     {
       "claim": "PlanRAG significantly reduces CAN (improper candidate) and MIS (missed data) errors for both scenarios.",
       "evidence": "Figure 6 shows error category distribution for IterRAG-LM and PlanRAG-LM, with visible reductions in CAN and MIS categories.",
       "supported": "moderate"
     },
     {
       "claim": "PlanRAG is more effective for Single Retrieval (SR) questions than Multiple Retrieval (MR) questions relative to IterRAG.",
       "evidence": "Figure 5 shows larger accuracy gaps for SR questions (e.g., 65.6% vs 43.9% in Locating SR) than MR questions (61.9% vs 49.4% in Locating MR).",
       "supported": "moderate"
     },
     {
       "claim": "LLMs smaller than GPT-4 (Llama 2, Phi-2) cannot solve Decision QA problems at all, and GPT-3.5 performs poorly with PlanRAG.",
       "evidence": "Table 11: Llama 2 (70B, 13B) and Phi-2 achieve 0% across all settings. GPT-3.5 with PlanRAG achieves 0-4% on most settings, performing worse than IterRAG with GPT-3.5.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "PlanRAG, an iterative plan-then-retrieval augmented generation technique, outperforms iterative RAG by 15.8% and 7.4% on two Decision QA scenarios built from grand strategy video games. The planning step is most beneficial for questions that IterRAG underestimates in difficulty, and re-planning provides substantial gains on the simpler Locating scenario but minimal improvement on the harder Building scenario. Only GPT-4 can meaningfully address Decision QA; smaller models (Llama 2, Phi-2) score 0%, and GPT-3.5 cannot follow PlanRAG's complex prompt structure.",
   "red_flags": [
     {
       "flag": "Single-run evaluation with no variance",
       "detail": "All experiments are conducted as single runs with zero-temperature GPT-4. No variance, confidence intervals, or significance tests are reported, yet the paper uses language like 'significantly improves' and 'far more effective.' The 15.8% and 7.4% improvements have no statistical backing."
     },
     {
       "flag": "Video game benchmark presented as business decision making",
       "detail": "The paper motivates with Pfizer distribution network examples and frames the work as LLMs for business decision making, but all evaluation is on Europa Universalis IV and Victoria 3 game mechanics. The claim that these games 'well imitate real business situations' lacks evidence."
     },
     {
       "flag": "Potential game knowledge contamination",
       "detail": "EU4 and Victoria 3 have extensive online wikis, strategy guides, and forum discussions that GPT-4 was almost certainly trained on. The model may already know optimal trade steering and building strategies, inflating performance independently of the RAG technique."
     },
     {
       "flag": "Scaffold confound between methods",
       "detail": "PlanRAG uses a different prompt structure (Plan-Thought-Action-Observation-Replan) than IterRAG (Thought-Action-Observation). Table 10 shows prompt structure alone causes substantial accuracy variation (27.5% to 57.5%), yet the main comparison attributes all improvement to the planning concept rather than the prompt engineering."
     },
     {
       "flag": "Unquantified compute difference",
       "detail": "PlanRAG performs additional planning and re-planning iterations (Table 7 shows up to 4+ re-plannings), consuming more API calls and tokens than IterRAG. This compute difference is never quantified, making it unclear whether PlanRAG's improvement comes from planning or simply from using more compute."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R. Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Core baseline technique; PlanRAG extends ReAct's Thought-Action-Observation framework with planning and re-planning steps."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "Fabio Petroni", "Vladimir Karpukhin"],
       "year": 2020,
       "relevance": "Foundational RAG technique that PlanRAG builds upon; defines the retrieve-then-generate paradigm for language models."
     },
     {
       "title": "Interleaving Retrieval with Chain-of-Thought Reasoning for Knowledge-Intensive Multi-Step Questions",
       "authors": ["Harsh Trivedi", "Niranjan Balasubramanian", "Tushar Khot", "Ashish Sabharwal"],
       "year": 2023,
       "relevance": "Iterative RAG baseline method that PlanRAG compares against; interleaves retrieval with reasoning for complex questions."
     },
     {
       "title": "Active Retrieval Augmented Generation",
       "authors": ["Zhengbao Jiang", "Frank F Xu", "Luyu Gao", "Zhiqing Sun", "Qian Liu"],
       "year": 2023,
       "arxiv_id": "2305.06983",
       "relevance": "Iterative RAG technique that decides when to retrieve; represents the state-of-the-art RAG approach compared against PlanRAG."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Fei Xia"],
       "year": 2022,
       "relevance": "Foundational prompting technique for LLM reasoning that informs the Thought step in PlanRAG's prompt structure."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Primary model used for all main experiments; only model capable of solving Decision QA problems in this study."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-tuned Chat Models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Open-source LLM evaluated in Appendix A.7 showing 0% accuracy on Decision QA, demonstrating the task's difficulty for smaller models."
     },
     {
       "title": "StructGPT: A General Framework for Large Language Model to Reason over Structured Data",
       "authors": ["Jinhao Jiang", "Kun Zhou", "Zican Dong", "Keming Ye"],
       "year": 2023,
       "arxiv_id": "2305.09645",
       "relevance": "Framework for LLM reasoning over structured databases, directly related to Decision QA's task of querying structured data for decision making."
     },
     {
       "title": "REPLUG: Retrieval-Augmented Black-Box Language Models",
       "authors": ["Weijia Shi", "Sewon Min", "Michihiro Yasunaga", "Minjoon Seo"],
       "year": 2023,
       "arxiv_id": "2301.12652",
       "relevance": "RAG technique for augmenting black-box LMs with retrieval, representing the broader family of retrieval-augmented approaches."
     },
     {
       "title": "Dense Passage Retrieval for Open-Domain Question Answering",
       "authors": ["Vladimir Karpukhin", "Barlas Oguz", "Sewon Min", "Patrick Lewis"],
       "year": 2020,
       "relevance": "Foundational retrieval method for knowledge-based QA; paper contrasts Decision QA with knowledge-based QA approaches."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The PlanRAG technique is conceptually applicable to business decision support, but the video game evaluation setting limits immediate practical adoption."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Adding planning to RAG is an intuitive extension; the finding that only GPT-4 can handle Decision QA while smaller models score 0% is somewhat surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; a straightforward technical contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and benchmark released on GitHub with game simulators; someone could reproduce the experiments with API access."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "KAIST is a respected institution but not a top AI brand; the paper uses GPT-4 which adds some recognition."
     }
   }
 }