ai-research-survey

scan.json (27918B)

---

 {
   "paper": {
     "title": "Successive Prompting for Decomposing Complex Questions",
     "authors": ["Dheeru Dua", "Shivanshu Gupta", "Sameer Singh", "Matt Gardner"],
     "year": 2022,
     "venue": "Conference on Empirical Methods in Natural Language Processing",
     "arxiv_id": "2212.04092",
     "doi": "10.48550/arXiv.2212.04092"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Successive prompting iteratively decomposes complex questions into simple QA pairs with decoupled supervision, enabling tailored in-context examples per step and specialized modules for symbolic reasoning. On a few-shot variant of DROP (300 training examples), the best model achieves 50.2% F1 on the test set, a ~5% absolute improvement over the state-of-the-art TASE model with similar supervision. Synthetic data generated from Wikipedia tables universally improves all model architectures, and the modular design enables mixing in-context and fine-tuned components—replacing in-context QA with a fine-tuned QA module yields ~10% F1 improvement.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The code and data are available at https://github.com/dDua/succesive_prompting' in Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository claims to include data. DROP is a publicly available benchmark. Synthetic data generation from Wikipedia tables is also described and included."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions HuggingFace transformers, T5-large UnifiedQA, and GPT-J (6B), but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While training details are described (learning rate, input length, beam size), no step-by-step reproduction instructions or runnable scripts are provided in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 all report single F1 point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims successive prompting outperforms baselines by comparing F1 numbers directly (e.g., '∼5% improvement in F1') without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute F1 improvements with baseline context throughout: e.g., 51.3 vs 45.9 F1 on dev (Table 3), 50.2 vs 45.1 on test, 31.9 vs 27.6 for successive prompting vs CoT (Table 2). The reader can assess the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 300 training examples are selected via vertex cover to cover the majority of DROP training data, but there is no statistical justification for why 300 is sufficient, no power analysis, and no discussion of whether 300 examples provide adequate signal."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are single-run numbers. No standard deviations, variance across seeds, or spread measures are reported in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Standard prompting, Chain-of-Thought, UnifiedQA, PReasM, and TASE across multiple experimental settings (Tables 2, 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include CoT (Wei et al., 2022), PReasM (Yoran et al., 2021), TASE (Segal et al., 2020), and UnifiedQA (Khashabi et al., 2020), all published within 2 years of this 2022 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 ablates the symbolic calculator (w/ calc vs w/o calc). Table 4 ablates QD and QA modules between in-context and fine-tuned variants. Tables 2 and 3 ablate data sources (Syn-Only, DROP-Only, Syn+DROP) and supervision levels (0-shot, w/o decomp, w/ decomp)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only F1 is reported as an evaluation metric across all experiments. No other metrics (e.g., exact match, accuracy) are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 describes manual analysis of 50 correct and 50 incorrect predictions, evaluating decomposition quality (88% accuracy for in-context, 96% for fine-tuned) and categorizing error types."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'We follow the standard practice of using test set for only our final best performing model' (Section 4.2), reporting dev set for all models and test set only for the best (50.2 F1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down results by model type (symbolic vs non-symbolic), supervision level (0-shot, w/o decomp, w/ decomp), and data source (with/without synthetic). Section 4.4 breaks down errors by category (incorrect QA, incorrect QD, out-of-scope)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 provides detailed error analysis with three error categories and specific percentages. Figure 4 shows qualitative failure examples including incorrect decompositions and implicit reasoning failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reveals that fine-tuned QD with in-context QA (31.4 F1) barely improves over all in-context (30.8), showing that better decomposition alone doesn't help without better answering. Table 2 shows removing the calculator hurts. The Limitations section discusses where the approach fails."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '∼5% absolute F1' improvement. Table 3 shows 51.3 vs 45.9 on dev (5.4 F1) and test results of 50.2 vs 45.1 (5.1 F1), supporting this claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via controlled ablations: removing the calculator (Table 2), swapping QD/QA components between in-context and fine-tuned (Table 4), and varying data sources (Tables 2, 3). These are single-variable manipulations adequate for the claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states 'we have only shown the applicability of one specific version of this idea in one specific setting' and discusses challenges in applying the method to other domains (commonsense, causal reasoning)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the observed improvements. For example, successive prompting makes multiple LM queries per question, so the improvement could partly reflect more compute rather than better decomposition. This compute confound is noted only briefly in Limitations but not analyzed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures F1 on DROP and claims F1 on DROP. No proxy gap exists — the metric matches the granularity of the claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "GPT-J (6B) is specified by name and size—it is a single open-source model with no version ambiguity. T5-large version of UnifiedQA is similarly specific."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows complete prompt examples with actual text for both QD and QA stages. Appendix A provides the control codes used for fine-tuning. The prompting format is fully reproducible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports: learning rate 5e-5, max input length 768, beam search size 5, max decomposition steps 10, 6 in-context examples, 2 epochs cross-entropy then contrastive, up to 3 negative samples, 80K dynamic sampling per epoch."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The iterative QD→QA pipeline is the core contribution and is described in detail: separate index lookup for QD and QA stages, alternating generation steps, termination condition ('There are no more questions left to ask'), and symbolic calculator routing. Figures 1 and 2 provide visual walkthroughs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes selecting 300 examples via sentence embedding + cosine similarity + vertex cover. Section 3 describes synthetic data generation from Wikipedia tables with curated templates, including statistics (141K complex questions, 525K QD examples, 257K QA examples). Dynamic sampling strategy per epoch is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the Conclusion, providing substantive discussion of the method's constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: decomposition data may be impossible to obtain for some domains, some complex questions resist decomposition (commonsense, causal reasoning), granularity depends on model capabilities, and increased computational cost from multiple queries."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what was NOT tested: other kinds of complex questions (commonsense, causal reasoning), other domains beyond DROP, different decomposition granularities. The paper says 'we have only shown the applicability of one specific version of this idea in one specific setting.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "DROP is publicly available. The code and data repository is linked (https://github.com/dDua/succesive_prompting). The 300 annotated examples and synthetic data are described as available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the 300-example selection pipeline: QQP-based sentence embedding → cosine similarity → top-50 neighbors → connection graph → vertex cover. Section 3 details synthetic data generation from Wikipedia tables with 10 operation types and higher-order combinations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are the public DROP benchmark and synthetically generated data from Wikipedia tables."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: Wikipedia tables → template-based paragraph and question generation → 10 operation types with higher-order combinations → 141K complex questions → 525K QD and 257K QA examples (Table 5). Dynamic sampling at each epoch is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements list DARPA MCS program (Contract No. N660011924033), AI2, NSF IIS-1817183, and Hasso Plattner Institute fellowship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UC Irvine, Allen Institute for AI, and Microsoft Semantic Machines."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "DARPA, NSF, AI2, and HPI are government/academic funders with no financial stake in the evaluation outcomes. The paper evaluates open-source models (GPT-J, T5/UnifiedQA), not products from the funders."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is provided. One author is at Microsoft Semantic Machines, but no conflict disclosure is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-J (6B) was trained on The Pile with no training cutoff date stated in the paper. T5/UnifiedQA training data cutoff is also not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "DROP was published in 2019 and GPT-J was trained on internet data collected after that. No discussion of whether DROP examples appear in GPT-J's or T5's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DROP (2019) was available online before the models were trained. No contamination analysis is performed despite the risk that these models may have seen DROP examples during pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates language models on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Ethics Statement confirms: 'This work does not deal with any social impacts or biases.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section acknowledges 'this method also increases the computational requirements' due to multiple LM queries per question, but no actual cost, latency, or token counts are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, API costs, or hardware specifications are provided despite using large language models for both in-context learning and fine-tuning."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-seed. No mention of multiple random seeds or seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing the reported results is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported (learning rate 5e-5, etc.) but no search budget, number of configurations tried, or search method is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'We follow the standard practice of using test set for only our final best performing model' and uses dev set performance for all model selection decisions (Table 3)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and compare against baselines (TASE, UnifiedQA, PReasM) without acknowledging potential bias from evaluating their own system against their implementations of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Successive prompting makes multiple LM queries per question while CoT makes one, but performance is never reported as a function of compute. The compute disparity is acknowledged qualitatively in Limitations but not controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "DROP is used without discussion of whether it adequately measures the claimed compositional reasoning capability. No comparison with alternative benchmarks or analysis of what DROP specifically tests."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The successive prompting scaffold IS the contribution being evaluated. Different prompting methods (Standard, CoT, SP) are the independent variable, so the scaffold confound is the experimental design itself."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DROP was published in 2019 and the models (GPT-J, T5) were trained on later data. No discussion of temporal leakage—the models may have encountered DROP passages or questions during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether in-context examples or the evaluation setup leak answer information. The QA index retrieval could potentially surface related examples that leak information about the test question."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of structural similarities between training and test data. The 300 DROP training examples were selected to cover training data via vertex cover, but independence from the test set is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used—no canary strings, membership inference, n-gram overlap analysis, or decontamination pipeline."
       }
     }
   },
   "claims": [
     {
       "claim": "Successive prompting achieves ~5% absolute F1 improvement over SOTA on few-shot DROP",
       "evidence": "Table 3: SP w/ decomp = 51.3 F1 (dev) vs TASE+Synthetic w/ decomp = 45.9 F1 (dev). Test set: SP = 50.2 vs TASE = 45.1 (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Successive prompting outperforms Chain-of-Thought by 4.3% F1 with in-context learning",
       "evidence": "Table 2: Succ.(w/ calc.) = 31.9 vs CoT = 27.6 for Syn+DROP setting. The 3.5% gap in Syn-Only (28.8 vs 25.3) also supports this.",
       "supported": "moderate"
     },
     {
       "claim": "Synthetic data from Wikipedia tables universally improves model performance",
       "evidence": "Table 3 shows consistent improvements for all model types when synthetic data is added: UnifiedQA (+2.1 to +5.4), PReasM (+0.6 to +5.3), TASE (+17.8 to +18.3 in some settings).",
       "supported": "strong"
     },
     {
       "claim": "Modular approach with symbolic calculator is better than end-to-end LM",
       "evidence": "Table 2: Succ.(w/ calc.) = 31.9 vs Succ.(w/o calc.) = 29.9 for Syn+DROP (2.0 F1 improvement). Paper states '1.5% F1' referring to DROP-Only column (30.8 vs 29.3).",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuned QA module provides ~10% F1 improvement over in-context QA",
       "evidence": "Table 4: QD In-Context + QA Fine-tuning = 40.3 vs QD In-Context + QA In-Context = 30.8 (9.5 F1 improvement).",
       "supported": "moderate"
     },
     {
       "claim": "QD accuracy is 88% for in-context and 96% for fine-tuned model on correct predictions",
       "evidence": "Section 4.4: Manual analysis of 50 correct predictions. Incorrect decompositions were 'mainly because the decomposed question is identical to the original question.'",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across Tables 2, 3, and 4 are single-run point estimates with no variance, standard deviation, confidence intervals, or significance tests. The ~5% F1 improvement claim cannot be assessed for statistical reliability."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only F1 is reported. DROP also supports Exact Match, and compositional reasoning could benefit from decomposition accuracy or reasoning step correctness metrics."
     },
     {
       "flag": "No contamination analysis",
       "detail": "DROP (2019) was publicly available before GPT-J and T5 were trained. The paper does not discuss whether these models may have seen DROP passages or answers during pre-training, which would inflate results for all methods."
     },
     {
       "flag": "Uncontrolled compute comparison",
       "detail": "Successive prompting makes multiple LM queries per question while CoT makes one. Performance improvements could partly reflect more compute rather than better decomposition. The Limitations section acknowledges this qualitatively but does not control for it."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique for LLM reasoning that successive prompting builds upon and compares against."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Introduced few-shot prompting as a paradigm for large language models, foundational to the in-context learning approach used in this paper."
     },
     {
       "title": "Show your work: Scratchpads for intermediate computation with language models",
       "authors": ["Maxwell Nye", "Anders Johan Andreassen", "Guy Gur-Ari"],
       "year": 2021,
       "arxiv_id": "2112.00114",
       "relevance": "Early work on intermediate reasoning steps for LMs, precursor to chain-of-thought and successive prompting approaches."
     },
     {
       "title": "MRKL systems: A modular, neuro-symbolic architecture that combines large language models, external knowledge sources and discrete reasoning",
       "authors": ["Ehud Karpas", "Omri Abend", "Yonatan Belinkov"],
       "year": 2022,
       "arxiv_id": "2205.00445",
       "relevance": "Modular neuro-symbolic approach combining LLMs with external tools and discrete reasoning, closely related to successive prompting's modular design."
     },
     {
       "title": "Least-to-most prompting enables complex reasoning in large language models",
       "authors": ["Denny Zhou", "Nathanael Schärli", "Le Hou", "Jason Wei"],
       "year": 2022,
       "arxiv_id": "2205.10625",
       "relevance": "Concurrent work with similar decomposition approach, but decomposes all sub-problems first then solves sequentially, unlike the interleaved approach of successive prompting."
     },
     {
       "title": "Decomposed prompting: A modular approach for solving complex tasks",
       "authors": ["Tushar Khot", "Harsh Trivedi", "Matthew Finlayson", "Yao Fu"],
       "year": 2022,
       "arxiv_id": "2210.02406",
       "relevance": "Contemporary modular prompting approach for complex task decomposition, part of the emerging paradigm of compositional LLM reasoning."
     },
     {
       "title": "Measuring and narrowing the compositionality gap in language models",
       "authors": ["Ofir Press", "Muru Zhang", "Sewon Min"],
       "year": 2022,
       "arxiv_id": "2210.03350",
       "relevance": "Measures how well LMs compose reasoning steps and shows gaps in compositional reasoning that methods like successive prompting aim to address."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "Large-scale language model demonstrating few-shot prompting capabilities, context for the scaling trends that prompting methods leverage."
     },
     {
       "title": "Text modular networks: Learning to decompose tasks in the language of existing models",
       "authors": ["Tushar Khot", "Daniel Khashabi", "Kyle Richardson"],
       "year": 2021,
       "relevance": "Direct predecessor to successive prompting's modular QD/QA design, limited to first-order reasoning which successive prompting extends."
     },
     {
       "title": "STaR: Bootstrapping reasoning with reasoning",
       "authors": ["Eric Zelikman", "Yuhuai Wu", "Noah D. Goodman"],
       "year": 2022,
       "arxiv_id": "2203.14465",
       "relevance": "Uses self-generated reasoning traces to bootstrap LLM reasoning capability, related to the synthetic data bootstrapping approach in successive prompting."
     },
     {
       "title": "UNIFIEDQA: Crossing format boundaries with a single QA system",
       "authors": ["Daniel Khashabi", "Sewon Min", "Tushar Khot"],
       "year": 2020,
       "relevance": "Pre-trained QA system used as the base model for fine-tuning experiments in this paper."
     }
   ]
 }