ai-research-survey

scan.json (30690B)

---

 {
   "paper": {
     "title": "Demonstrate-Search-Predict: Composing retrieval and language models for knowledge-intensive NLP",
     "authors": [
       "Omar Khattab",
       "Keshav Santhanam",
       "Xiang Lisa Li",
       "David Hall",
       "Percy Liang",
       "Christopher Potts",
       "Matei Zaharia"
     ],
     "year": 2022,
     "venue": "arXiv preprint",
     "arxiv_id": "2212.14024",
     "doi": "10.48550/arXiv.2212.14024"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DSP framework composes frozen language models and retrieval models in multi-step pipelines for knowledge-intensive NLP, achieving 37–120% gains over vanilla GPT-3.5, 8–39% over retrieve-then-read, and 80–290% over self-ask on Open-SQuAD, HotPotQA, and QReCC. The DEMONSTRATE stage bootstraps pipeline-aware demonstrations from end-task labels without hand-labeling intermediate transformations. The paper identifies a 'self-distraction' failure mode in self-ask where delegating control flow to LM completions produces tangential decompositions.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract and §1 state 'We release DSP at https://github.com/stanfordnlp/dsp' with a working URL to the public repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation uses publicly available benchmarks: Open-SQuAD (Rajpurkar et al., 2016), HotPotQA (Yang et al., 2018), and QReCC (Anantha et al., 2020), with public Wikipedia corpus dumps from Chen et al. (2017), Yang et al. (2018), and Karpukhin et al. (2020)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided in the paper. The paper mentions Python implementation but provides no dependency or version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. Code is released but the paper itself contains no 'Reproducing Results' section or commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., 36.6 EM, 49.0 F1). Despite averaging across 5 seeds (§3.1), no confidence intervals, error bars, or ± notation are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'outperforms the vanilla LM baseline by 126% EM relative gains' are made by comparing point estimates in Table 1 without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative gains with baseline context: '37–120%, 8–39%, and 80–290% relative gains' (abstract). Table 1 provides absolute numbers for all systems, allowing readers to compute effect magnitudes (e.g., 16.2→36.6 EM for Open-SQuAD)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 1000 validation questions subsampled across 5 seeds (200 per seed), and 16-shot training examples. No justification is given for these choices and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "§3.1 states results are averaged across 5 seeds, but no standard deviation, IQR, or any spread measure is reported in Table 1 or the text. The reader cannot assess result stability across seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against vanilla LM, retrieve-then-read, self-ask (two configurations), and reports concurrent in-context learning SoTA results from Si et al. (2022), Sun et al. (2022), Wang et al. (2022b), and Yao et al. (2022)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporaneous work: self-ask (Press et al., 2022), ReAct (Yao et al., 2022), recite-and-answer (Sun et al., 2022), and self-consistency prompting (Wang et al., 2022b), all from 2022."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "DSP has three stages (Demonstrate, Search, Predict) with multiple design choices (self-consistency, query fusion, multi-hop), but no systematic ablation study removes individual components to measure their contribution. The baselines are different systems, not ablated versions of DSP."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both EM and F1 for Open-SQuAD and HotPotQA, and both F1 and novel-F1 (nF1) for QReCC."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using EM and F1 metrics. No human evaluation of answer quality, groundedness, or coherence is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "§3.1 explicitly states: 'We report the validation set accuracy on all three datasets.' Held-out test datasets (Open-NaturalQuestions, FEVER) are mentioned but deferred to 'a future version of this report.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only aggregate scores per dataset. No per-category breakdown (e.g., by question type, difficulty, number of hops) is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§3.5 discusses the 'self-distraction' failure mode of self-ask with a concrete example (Edgardo Mortara question producing tangential decomposition). The paper also discusses limitations of retrieve-then-read in Figure 1."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows DSP outperforming all baselines. No configurations, design choices, or approaches that were tried and failed are reported. The paper also notes that branch, knn, and crossval primitives are 'work-in-progress' but doesn't discuss attempted approaches that didn't work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '37–120%, 8–39%, and 80–290% relative gains' are supported by Table 1 results. The 'new state-of-the-art in-context learning results' claim is supported by comparison with concurrent work in §3.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims DSP 'delivers gains' over baselines. The study design holds the LM (GPT-3.5) and RM (ColBERTv2) constant across all conditions, varying only the pipeline architecture. This controlled single-variable manipulation is adequate for claiming the pipeline design causes different performance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'knowledge-intensive NLP' broadly but results are limited to three QA-style tasks with a single LM (GPT-3.5) and single RM (ColBERTv2). The paper acknowledges 'Future versions of this report will include additional test tasks and LM choices' but the title and abstract generalize beyond the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the gains. For instance, DSP's self-consistency uses n=20 samples, consuming much more compute than baselines — the gains could partly reflect this compute gap rather than pipeline design. No robustness checks or alternative factors are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures EM and F1 on three QA benchmarks but frames contributions as advancing 'knowledge-intensive NLP' broadly. No discussion acknowledges that QA metrics are a narrow proxy for the broader capabilities claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "§3.2 specifies 'GPT-3.5 (text-davinci-002; Brown et al. 2020; Ouyang et al. 2022)' and 'ColBERTv2 (Santhanam et al., 2022b)' with precise model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "§2.4 and §2.5 provide actual prompt text (e.g., 'My task is to write a simple query that gathers information...', 'My task is to answer questions using Web documents...'). The full code is also released at the GitHub repository where all prompt templates are available."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§3.1–3.4 report key hyperparameters: greedy decoding for n=1, temperature t=0.7 for n>1, k=7 passages for open-domain QA, k=5 for multi-hop, n=20 for self-consistency, n=10 for query fusion, 16-shot training examples."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper describes the DSP pipeline architecture in detail: the three-stage framework (Demonstrate, Search, Predict), data flow between components, control flow with code snippets, and specific configurations for each task. §2.1–2.5 provide comprehensive descriptions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 documents evaluation subsampling (1000 questions, 5 seeds, 200 per seed). §3.5 describes HotPotQA filtering to 'hard' examples and 90/10 train/val splits. QReCC filtering criteria are explicitly stated (removing empty answers, short conversations, and 'other interesting' questions)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion discusses the framework's potential but does not substantively address limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper mentions 'Future versions of this report will include additional test tasks and LM choices' but does not state what the current results do NOT show or what claims are NOT being made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used are publicly available: SQuAD, HotPotQA, and QReCC datasets and their associated Wikipedia corpus dumps are downloadable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 and §3.5 describe which datasets were used, which splits, which Wikipedia corpus versions (Dec 2016, Nov 2017 abstracts, Dec 2018), and how evaluation was conducted across seeds."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (SQuAD, HotPotQA, QReCC)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from data to results is documented: dataset selection, corpus alignment, subsampling strategy, seed-based evaluation split, and QReCC filtering criteria with explicit markers (§3.1, §3.5)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists: IBM (HAI founding member), Stanford DAWN project supported by Ant Financial, Facebook, Google, VMware, plus Cisco, SAP, and NSF CAREER grant CNS-1651570."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All seven authors are listed as Stanford University. The paper evaluates GPT-3.5 (OpenAI) and ColBERTv2 (developed by some of the same authors at Stanford), and affiliations are transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from academic (NSF, Stanford HAI) and corporate (IBM, Google, Facebook, etc.) sources supporting general AI research. None of the funders have a specific financial interest in DSP outperforming the baselines tested."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Some authors developed ColBERTv2, which is used as the RM, and this potential interest is not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5 (text-davinci-002) is used but its training data cutoff date is not stated anywhere in the paper."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SQuAD (2016), HotPotQA (2018), or QReCC (2020) examples appeared in GPT-3.5's training data, despite all being publicly available before the model's training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All three benchmarks (SQuAD 2016, HotPotQA 2018, QReCC 2020) were published and publicly available online well before GPT-3.5's training cutoff. No contamination risk discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on NLP benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "DSP programs call GPT-3.5 many times per example (e.g., n=20 self-consistency samples, multi-hop search with n=10 query fusion, plus demonstration bootstrapping) but no API costs, token counts, or latency measurements are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "§3.1 mentions controlling 'the language model API spending budget' by splitting evaluation across seeds, but the actual budget, total API spend, or compute resources used are never quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "§3.1 states results are averaged across 5 seeds, but per-seed results are not reported and no sensitivity analysis shows how much results vary across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 clearly states: 'report average quality across five seeds where each seed fixes a single k-shot training set of examples.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Various hyperparameters are set (k=7 passages, n=20 for self-consistency, k=3 demonstrations, etc.) but no search budget, search method, or justification for these choices is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameter choices (k values for retrieval, n values for generation, number of demonstrations) appear tuned but no selection criterion or validation-based justification is described."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (vanilla LM, retrieve-then-read) and compare against their own DSP framework. They also modified the self-ask prompt for comparison. No acknowledgment of author-evaluation bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "DSP uses dramatically more compute than baselines: multi-hop retrieval with n=10 query fusion, n=20 self-consistency, plus demonstration bootstrapping. The vanilla LM uses a single LM call. This compute gap is never discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses SQuAD, HotPotQA, and QReCC with EM/F1 metrics to evaluate 'knowledge-intensive NLP' capabilities without discussing whether these benchmarks and metrics adequately measure the claimed construct."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly controls for the scaffold confound by holding the LM (GPT-3.5 text-davinci-002) and RM (ColBERTv2) constant across all pipeline variants. Differences are attributable to pipeline design, which is the stated variable under study."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5 was trained on data that may include SQuAD (2016), HotPotQA (2018), and QReCC (2020) questions and answers. No temporal leakage discussion is present."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup introduces information leakage (e.g., whether retrieved passages or demonstration bootstrapping provide unfair hints)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence. The 16-shot training examples are sampled from the same distribution as test examples, and the bootstrapping process uses the same retrieval corpus."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied (no canary strings, membership inference, n-gram overlap analysis, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "DSP programs deliver 37–120% relative EM gains over vanilla GPT-3.5 across three knowledge-intensive QA tasks.",
       "evidence": "Table 1: Open-SQuAD 16.2→36.6 EM (126% gain), HotPotQA 28.3→51.4 EM (82% gain). All using GPT-3.5 and ColBERTv2 with 16-shot training examples.",
       "supported": "strong"
     },
     {
       "claim": "DSP programs deliver 8–39% relative EM gains over a standard retrieve-then-read pipeline.",
       "evidence": "Table 1: Open-SQuAD 33.8→36.6 EM (8% gain), HotPotQA 36.9→51.4 EM (39% gain).",
       "supported": "moderate"
     },
     {
       "claim": "DSP programs deliver 80–290% relative EM gains over the self-ask pipeline.",
       "evidence": "Table 1: HotPotQA 28.6→51.4 EM (80% gain), Open-SQuAD 9.0→36.6 EM (307% gain). The self-ask pipeline was modified with refinements for fairer comparison.",
       "supported": "strong"
     },
     {
       "claim": "The DEMONSTRATE stage can bootstrap pipeline-aware demonstrations from end-task labels without hand-labeling intermediate transformations.",
       "evidence": "§2.3 describes the annotate primitive and §3.4 shows its use in all three programs. No ablation isolates the DEMONSTRATE stage's contribution to the final numbers.",
       "supported": "moderate"
     },
     {
       "claim": "Self-ask suffers from a 'self-distraction' problem where the LM decomposes questions tangentially.",
       "evidence": "§3.5 provides one concrete example (Edgardo Mortara question), and self-ask's low Open-SQuAD score (9.3% EM) supports the claim. However, this is a single illustrative example without systematic error analysis.",
       "supported": "moderate"
     },
     {
       "claim": "DSP in-context learning results are competitive with fine-tuned systems like Fusion-in-Decoder on Open-SQuAD.",
       "evidence": "§3.5 compares DSP's 36.6% EM with FiD-base at ~36% EM (with 5 passages), though FiD reaches 48% EM with 100 passages. Comparison is qualitative and on different evaluation conditions.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars despite multiple seeds",
       "detail": "Results are averaged across 5 seeds but no variance, standard deviation, or confidence intervals are reported. The reader cannot assess whether the differences between DSP and baselines are within seed-to-seed noise."
     },
     {
       "flag": "No ablation study",
       "detail": "DSP has multiple components (Demonstrate, Search with multi-hop and query fusion, Predict with self-consistency) but no ablation isolates their individual contributions. The gains could be driven primarily by self-consistency (n=20 samples) rather than the novel architectural contributions."
     },
     {
       "flag": "Massive compute gap not addressed",
       "detail": "DSP calls GPT-3.5 dozens of times per example (demonstration bootstrapping, multi-hop with n=10 query fusion, n=20 self-consistency) while the vanilla LM uses 1 call and retrieve-then-read uses 1–2 calls. The cost difference is never reported or controlled for."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "SQuAD (2016), HotPotQA (2018), and QReCC (2020) were all publicly available before GPT-3.5's training data cutoff. The model may have memorized answers, and this is never discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations, threats to validity, or scope boundaries section despite making broad claims about 'knowledge-intensive NLP' from three QA tasks with one LM."
     },
     {
       "flag": "Baseline modifications may disadvantage comparison",
       "detail": "The self-ask baseline was modified from its original form (different passages, added training examples, prompt changes). While presented as improvements, the authors also note some primitives (branch, knn, crossval) are 'work-in-progress', raising questions about the completeness of the DSP implementation tested."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": [
         "T. Brown",
         "B. Mann",
         "N. Ryder"
       ],
       "year": 2020,
       "relevance": "Foundational in-context learning paper establishing the few-shot prompting paradigm that DSP builds upon."
     },
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": [
         "J. Wei",
         "X. Wang",
         "D. Schuurmans"
       ],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Introduces chain-of-thought prompting for LLM reasoning, a key component used in DSP's PREDICT stage."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": [
         "X. Wang",
         "J. Wei",
         "D. Schuurmans"
       ],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Self-consistency voting method used in DSP's PREDICT stage for selecting among multiple generated candidates."
     },
     {
       "title": "Measuring and narrowing the compositionality gap in language models",
       "authors": [
         "O. Press",
         "M. Zhang",
         "S. Min"
       ],
       "year": 2022,
       "arxiv_id": "2210.03350",
       "relevance": "Introduces self-ask pipeline, the primary comparison baseline, representing LLM self-decomposition for multi-hop QA."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": [
         "S. Yao",
         "J. Zhao",
         "D. Yu"
       ],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Contemporaneous framework for combining LLM reasoning with tool use (Wikipedia API search), achieving 35.1% EM on HotPotQA."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": [
         "P. Lewis",
         "E. Perez",
         "A. Piktus"
       ],
       "year": 2020,
       "relevance": "Foundational RAG paper combining retrieval and generation for knowledge-intensive tasks, directly motivating DSP's architecture."
     },
     {
       "title": "Language model cascades",
       "authors": [
         "D. Dohan",
         "W. Xu",
         "A. Lewkowycz"
       ],
       "year": 2022,
       "arxiv_id": "2207.10342",
       "relevance": "Theoretical framework for composing language model calls in cascaded pipelines, conceptually related to DSP's composition approach."
     },
     {
       "title": "Decomposed prompting: A modular approach for solving complex tasks",
       "authors": [
         "T. Khot",
         "H. Trivedi",
         "M. Finlayson"
       ],
       "year": 2022,
       "arxiv_id": "2210.02406",
       "relevance": "Modular decomposition approach for complex tasks using multiple LM calls, closely related to DSP's multi-step pipeline design."
     },
     {
       "title": "Star: Bootstrapping reasoning with reasoning",
       "authors": [
         "E. Zelikman",
         "Y. Wu",
         "N. Goodman"
       ],
       "year": 2022,
       "arxiv_id": "2203.14465",
       "relevance": "Self-bootstrapping approach for LLM rationale generation, generalized by DSP's DEMONSTRATE stage for pipeline-aware annotation."
     },
     {
       "title": "ColBERTv2: Effective and efficient retrieval via lightweight late interaction",
       "authors": [
         "K. Santhanam",
         "O. Khattab",
         "J. Saad-Falcon"
       ],
       "year": 2022,
       "doi": "10.18653/v1/2022.naacl-main.272",
       "relevance": "The retrieval model used in all DSP experiments; represents state-of-the-art dense retrieval for knowledge-intensive tasks."
     },
     {
       "title": "Few-shot learning with retrieval augmented language models",
       "authors": [
         "G. Izacard",
         "P. Lewis",
         "M. Lomeli"
       ],
       "year": 2022,
       "arxiv_id": "2208.03299",
       "relevance": "Concurrent work on retrieval-augmented few-shot learning, representing the retrieve-then-read paradigm that DSP aims to improve upon."
     },
     {
       "title": "Large language models can self-improve",
       "authors": [
         "J. Huang",
         "S. Gu",
         "L. Hou"
       ],
       "year": 2022,
       "arxiv_id": "2210.11610",
       "relevance": "LLM self-improvement via self-generated rationales, related to DSP's DEMONSTRATE bootstrapping approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "DSP is released as a pip-installable Python library (stanfordnlp/dsp) that practitioners can directly use to build RAG pipelines for knowledge-intensive tasks."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The 'self-distraction' failure mode of self-ask is a minor novel observation, but the overall finding that structured pipelines beat naive prompting is expected."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised or relevant to the work."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The 80-290% gains over self-ask and the 'self-distraction' critique mildly challenge that popular prompting approach, but without strong controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The GitHub repo is public with code examples, but requires API keys for GPT-3.5 and a ColBERTv2 index setup, making it moderate-effort to reproduce."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Stanford NLP (Percy Liang, Matei Zaharia, Omar Khattab) — well-known in the NLP/ML community, and this became the foundation for the widely-used DSPy framework."
     }
   }
 }