ai-research-survey

scan.json (26130B)

---

 {
   "paper": {
     "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
     "authors": ["Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari", "Zhiyuan Zhang", "Keshav Santhanam", "Sri Vardhamanan", "Saiful Haq", "Ashutosh Sharma", "Thomas T. Joshi", "Hanna Moazam", "Heather Miller", "Matei Zaharia", "Christopher Potts"],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2310.03714"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DSPy introduces a programming model that replaces hand-crafted prompt engineering with declarative modules and automatic compilation (teleprompters). On GSM8K, compiled DSPy programs improve GPT-3.5 from 25% to 82% and llama2-13b-chat from 9% to 47%. On HotPotQA, multi-hop programs compiled with bootstrapping achieve 45.6% answer EM with GPT-3.5 and 41.0% with llama2-13b-chat, competitive with much larger models. Small models like T5-Large (770M) can be fine-tuned via DSPy to be competitive with GPT-3.5.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/stanfordnlp/dspy, mentioned in the abstract and Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: GSM8K (Cobbe et al., 2021) and HotPotQA (Yang et al., 2018), both standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is described in the paper. Only model names are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released, the paper does not include step-by-step reproduction instructions or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report point estimates only (e.g., '81.6', '46.9') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DSPy programs outperform baselines but provides no statistical significance tests — only raw accuracy comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., 'from 33% to 82%' (Sec 1), 'from 32% to 46%' (Sec 1), and absolute numbers in Tables 1 and 2."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Training set sizes (200, 300 examples) and dev/test splits are stated but not justified. No power analysis or rationale for why these sizes are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 mentions 'average of 3–5 runs' for the fewshot setting with random sampling, but no standard deviations or variance measures are reported in the results tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: zero-shot (none), few-shot with random demos, few-shot with human CoT demos, and bootstrapped variants. Also informal comparisons to prior work (Zhang et al., Wang et al., Touvron et al.)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons include contemporary work: Wang et al. (2022b) self-consistency with PaLM-540B, Zhao et al. (2023b) with gpt-3.5-turbo, Yao et al. (2022) ReAct with PaLM-540B, and Trivedi et al. (2022)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 systematically vary programs (vanilla, CoT, reflection, multihop) and compilation strategies (none, fewshot, bootstrap, ensemble), effectively serving as an ablation study across modules and teleprompters."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "GSM8K uses accuracy; HotPotQA uses both answer exact match (Ans) and passage retrieval accuracy (Psg) in Table 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is included. All evaluation is automated (accuracy, EM, passage accuracy)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "GSM8K uses the official 1.3k test set separate from train/dev (Sec 6). HotPotQA reserves the official validation set for testing (Sec 7). Dev results are reported separately."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by program type, compilation strategy, and LM in Tables 1 and 2. Dev vs test results are also separated."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis or failure case discussion. The paper shows only aggregate accuracy numbers without examining where or why the system fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Some configurations show worse performance: ensemble hurts vanilla on GPT-3.5 test (61.9 vs 61.7), ReAct with bootstrap underperforms fewshot+human in Table 2, and zero-shot results are very poor (4-20%). These are honestly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 25%/65% improvements and 5-46%/16-40% over expert demos are supported by Tables 1 and 2. The claim that T5 and llama2-13b are competitive with GPT-3.5 is supported by HotPotQA results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims compiling modules improves performance. The ablation-style evaluation (Tables 1-2) systematically varies one factor at a time (program type or compilation strategy), providing adequate evidence for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Self-Improving Pipelines' generally, but results are on only two tasks (GSM8K, HotPotQA) with only a few LMs. The paper acknowledges in Sec 8 that it leaves 'reporting on such tasks under controlled experimental conditions to future work' but the abstract and title overstate the generality."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, the bootstrapped demonstrations might simply be better few-shot examples rather than demonstrating the value of the modular programming model itself."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy/EM on specific benchmarks and reports these directly without inflating them into broader claims about general AI capability. Claims match measurement granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-3.5' and 'llama2-13b-chat' and 'T5-Large' without specifying exact API versions or snapshot dates. No version like 'gpt-3.5-turbo-0613' is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F (Figures 9-11) provides the actual automatically generated prompts used in the experiments, including full demonstration text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters like temperature, top-p, and max tokens are not reported. Some parameters like k=8 for few-shot and num_attempts=5 for reflection are stated in code, but LM API settings are missing."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The DSPy framework itself is the scaffolding, and it is described in extensive detail in Sections 3-4, with pseudocode in Appendices D and E for modules and teleprompters."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 describes sampling 200/300 examples from GSM8K training set. Section 7 describes HotPotQA splits (70/30 train/val, keeping only 'hard' examples, sampling 200/300). The retrieval index is specified as ColBERTv2 over Wikipedia 2017 abstracts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion (Sec 8) briefly mentions leaving other tasks to future work but does not discuss limitations substantively."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no consideration of whether the improvements generalize beyond the two tested tasks or whether the compilation cost is prohibitive."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Section 5 states evaluation goals (H1-H3) but does not bound the scope of claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual predictions, per-example results) is made available. Only aggregate accuracy numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are well described: GSM8K official train/test splits (Sec 6), HotPotQA official splits with the fullwiki setting (Sec 7), with specific sample sizes."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper uses standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from training examples through compilation (bootstrapping, random search, ensembling) to evaluation is documented in Sections 4, 6, and 7 with code examples."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding from IBM, Oracle, Virtusa, Cigna Healthcare, HAI Azure compute grant, Stanford DAWN project (Facebook, Google, VMware), and NSF CAREER grant CNS-1651570. Omar Khattab's Apple fellowship is also disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Stanford, UC Berkeley, CMU, Amazon Alexa AI, Dashworks, IIT Bombay, Calera Capital, Microsoft, Two Sigma."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (IBM, Oracle, NSF, etc.) are not evaluated in the paper. The paper evaluates open models and OpenAI's GPT-3.5; none of the funders have a direct stake in DSPy's performance claims."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included. Some authors have industry affiliations (Amazon, Microsoft, Two Sigma, Calera Capital) that are not discussed as potential conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-3.5 or llama2-13b-chat. The paper notes GPT-4 was 'pre-trained on a subset of GSM8K's training set' (Sec 6) but does not address this for GPT-3.5."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K or HotPotQA data appeared in the training data of GPT-3.5 or Llama2 models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and HotPotQA (2018) were both available online before GPT-3.5 and Llama2 training. The paper only mentions GPT-4's known contamination with GSM8K training data but does not address contamination for the models actually used."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs reported. The reflection program calls the LM 5 times per example, and bootstrap compilation runs the program thousands of times, but costs are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper says compilation runs 'on the order of minutes (or tens of minutes)' (Sec 6) but provides no specific compute budget, GPU hours, or API costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The paper mentions 'average of 3–5 runs' for the fewshot setting but does not report variance across seeds for bootstrapped results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states 'We report the average of 3–5 runs (depending on the setting) when applying such random sampling' for the fewshot compiler. However, it's unclear how many runs other compilation strategies use."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "BootstrapFewShotWithRandomSearch is used but the total number of trials, configurations explored, and compute spent on search is not reported in the main experiments."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states 'We report extensive comparisons on the development set to avoid overfitting on test.' Best configurations are selected on dev before test evaluation. The random search optimizes over a validation set (Appendix E.2)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across programs, compilers, and LMs without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement DSPy and evaluate it against their own implementations. No acknowledgment of self-comparison bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Bootstrap and ensemble methods use substantially more compute than simple few-shot, but performance is not reported as a function of compute budget. The reflection program calls the LM 5x more per example than vanilla."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K and HotPotQA actually measure the capabilities DSPy claims to improve. The paper takes benchmark validity for granted."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "DSPy IS the scaffold being evaluated. The paper systematically compares the same scaffold (DSPy programs) across different LMs (GPT-3.5, Llama2-13b, T5), isolating the model variable. When comparing compilation strategies, the same model is held constant."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and HotPotQA (2018) predate GPT-3.5 and Llama2 training. The paper mentions GPT-4's GSM8K contamination but does not address temporal leakage for the models actually evaluated."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The bootstrapped demonstrations come from the same distribution as the test set, but this is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence or whether bootstrapped demonstrations could overlap with or be similar to test examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used."
       }
     }
   },
   "claims": [
     {
       "claim": "DSPy programs compiled with bootstrapping outperform standard few-shot prompting by over 25% for GPT-3.5 and 65% for llama2-13b-chat",
       "evidence": "Table 1 shows GSM8K vanilla fewshot at 33.1% vs bootstrap×2 at 61.7% for GPT-3.5 (28.6pp gain), and 4.3% vs 36.5% for Llama2 (32.2pp gain). CoT bootstrap reaches 72.9% vs fewshot 63.0% for GPT-3.5.",
       "supported": "strong"
     },
     {
       "claim": "DSPy can match or outperform expert-created human reasoning chain demonstrations",
       "evidence": "Table 1, GSM8K CoT: bootstrap (72.9% test) vs human CoT fewshot (72.4% test) for GPT-3.5. Table 2, HotPotQA react: bootstrap×2 (40.0% dev) vs human demos (28.3% dev) for Llama2.",
       "supported": "moderate"
     },
     {
       "claim": "Small open LMs compiled with DSPy are competitive with proprietary GPT-3.5 using expert prompts",
       "evidence": "Table 2 HotPotQA: Llama2-13b multihop ensemble achieves 41.0% vs GPT-3.5 multihop ensemble 45.6%. T5-Large (770M) achieves 39.3% dev via finetuning (Sec 7).",
       "supported": "moderate"
     },
     {
       "claim": "A few lines of DSPy code can express sophisticated multi-stage NLP pipelines",
       "evidence": "Section 6 shows the ThoughtReflection program in ~10 lines, and Section 7 shows BasicMultiHop in ~15 lines. Both achieve strong results when compiled.",
       "supported": "strong"
     },
     {
       "claim": "DSPy compilation improves programs from 4-20% to 49-88% accuracy on GSM8K",
       "evidence": "Table 1: vanilla none (24.0/7.0 dev) to reflection bootstrap+ensemble (86.7/49.0 dev) for GPT-3.5/Llama2. The 88.3% figure is CoT ensemble on GPT-3.5 dev.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Despite averaging over 3-5 runs for some settings, no standard deviations, confidence intervals, or error bars are reported. It's impossible to assess whether differences between methods are meaningful or within noise."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "GSM8K and HotPotQA were publicly available years before GPT-3.5 and Llama2 training. The paper even notes GPT-4 trained on GSM8K data but doesn't address this for the models it actually evaluates."
     },
     {
       "flag": "Self-evaluation without acknowledgment",
       "detail": "The authors designed DSPy and evaluate it. All baselines (including zero-shot and few-shot variants) are run within the DSPy framework, meaning the framework authors control both the system and the evaluation setup."
     },
     {
       "flag": "Informal comparisons to prior work",
       "detail": "Section 6-7 compare informally to prior work ('we can informally compare') with different models, evaluation splits, and settings. These comparisons are not controlled and could be misleading."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section discussing failure modes, scalability concerns, or settings where DSPy would not work well."
     },
     {
       "flag": "Missing cost analysis",
       "detail": "Bootstrapping runs programs thousands of times, and ensemble/reflection multiply inference costs. No cost analysis is provided despite this being a major practical concern."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique that DSPy modularizes; key baseline for comparison."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Agent framework that DSPy implements as a module; directly compared in HotPotQA experiments."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Prompting technique related to DSPy's ensemble strategy; compared on GSM8K."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown"],
       "year": 2020,
       "relevance": "Foundational in-context learning work that DSPy builds upon and seeks to automate."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive nlp tasks",
       "authors": ["Patrick Lewis"],
       "year": 2020,
       "relevance": "RAG paradigm that DSPy implements as a composable module pattern."
     },
     {
       "title": "Automatic chain of thought prompting in large language models",
       "authors": ["Zhuosheng Zhang"],
       "year": 2022,
       "arxiv_id": "2210.03493",
       "relevance": "Automatic CoT method compared on GSM8K; related approach to DSPy's bootstrapping."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "arxiv_id": "2303.17651",
       "relevance": "Self-improvement technique for LMs that DSPy generalizes through its module and compiler system."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Tool-use approach for LMs relevant to DSPy's tool integration capabilities."
     },
     {
       "title": "Demonstrate-search-predict: Composing retrieval and language models for knowledge-intensive nlp",
       "authors": ["Omar Khattab"],
       "year": 2022,
       "arxiv_id": "2212.14024",
       "relevance": "DSPy's predecessor framework (DSP); directly evolved into the DSPy programming model."
     },
     {
       "title": "Automatic prompt optimization with gradient descent and beam search",
       "authors": ["Reid Pryzant"],
       "year": 2023,
       "arxiv_id": "2305.03495",
       "relevance": "Prompt optimization technique related to DSPy's teleprompter concept."
     },
     {
       "title": "Large language models as optimizers",
       "authors": ["Chengrun Yang"],
       "year": 2023,
       "arxiv_id": "2309.03409",
       "relevance": "LLM-based optimization approach relevant to DSPy's automated prompt tuning."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Cost-aware LLM usage relevant to DSPy's model selection and compilation capabilities."
     }
   ]
 }