ai-research-survey

scan.json (31542B)

---

 {
   "paper": {
     "title": "Decomposed Prompting: A Modular Approach for Solving Complex Tasks",
     "authors": [
       "Tushar Khot",
       "Harsh Trivedi",
       "Matthew Finlayson",
       "Yao Fu",
       "Kyle Richardson",
       "Peter Clark",
       "Ashish Sabharwal"
     ],
     "year": 2022,
     "venue": "International Conference on Learning Representations (ICLR 2023)",
     "arxiv_id": "2210.02406",
     "doi": "10.48550/arXiv.2210.02406"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Decomposed Prompting (DECOMP) outperforms Chain-of-Thought and Least-to-Most prompting across symbolic reasoning, long-context QA, and open-domain multi-hop QA by decomposing complex tasks into modular sub-task handlers with independent few-shot prompts. The approach enables hierarchical decomposition (for hard sub-tasks like kth letter extraction), recursive decomposition (achieving near-perfect length generalization on list reversal), and seamless integration of external tools like Elasticsearch retrieval. A simple post-processing decomposition for answer extraction in math QA yields 14-17 point improvements over standard CoT.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in Section 1 footnote: 'Datasets, Code and Prompts available at https://github.com/allenai/DecomP.' A working repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states datasets are available at the GitHub repository. The evaluation uses publicly available datasets (HotpotQA, MuSiQue, 2WikiMultihopQA, CommaQA-E, GSM8K, MultiArith) and provides custom evaluation data at the repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model names (text-davinci-002, code-davinci-002, Flan-T5 variants) but does not provide environment specifications such as requirements.txt, Dockerfile, or library versions needed to reproduce the experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "All prompts are provided verbatim in Appendix G (spanning ~25 pages). The decomposition structure, operators, and sub-task handlers are described in detail. Combined with the code release, a researcher could reconstruct the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 98.0%, 42.0%, 25.4%). While results are averaged across three prompts, no confidence intervals or error bars are shown in the figures or tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims like 'DECOMP outperforms chain-of-thought' are based solely on comparing point estimates without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout, e.g., '17 pt improvement on MultiArith (78 → 95) and 14 pt improvement on GSM8K (36 → 50.6)' (Section 4.5). Results figures show both baseline and method scores."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation uses 100 examples for letter concatenation, 300 for GSM8K, 200 for MultiArith, and 300 for open-domain QA. The only justification given is 'due to costs with API usage' (Appendix B footnotes), not a power analysis or statistical justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While results are averaged across 3 prompt variants, standard deviation is not systematically reported. Footnote 6 mentions 'the std. dev is zero here' for one case, but main results lack spread measures. Per-prompt results are shown in Appendix D but without aggregate variance statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: standard CoT, CoT with rollout, Least-to-Most prompting, No-Context QA, NoDecomp-Context QA. Comparisons are made across all evaluation tasks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Chain-of-Thought (Wei et al., 2022) and Least-to-Most (Zhou et al., 2023), which are contemporary state-of-the-art prompting methods at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section E and Appendix E compare alternative decomposition schemes (e.g., loop vs generate for letter concatenation, mid-split vs tail-split for reversal). Section 4.3 compares coarse vs fine-grained decomposition granularity. These function as ablation studies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Within each evaluation task, only a single metric is used: Exact Match for symbolic tasks and CommaQA, Answer F1 for open-domain QA. No task is evaluated with multiple metrics simultaneously."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated (Exact Match, Answer F1). No human evaluation of system outputs is performed, despite the paper making claims about the quality of decompositions and sub-task handling."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For open-domain QA, hyperparameters are tuned on 'a held out set of 100 questions for each dataset' and evaluated on '300 held-out dev questions' (Appendix A.2). Letter concatenation test examples use a separate word list from training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by input length (N=3,4,5 words for letter concatenation; N=4,6,8,10 for reversal), by dataset (3 open-domain QA datasets), by model size (Flan-T5-Large/XL/XXL, Codex), and by decomposition granularity (coarse/fine)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F provides detailed error analysis for letter concatenation (Section F.1) and CommaQA (Section F.2), showing specific examples of sub-task errors, incorrect letter extraction, and incorrect QA answers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports that 'CoT version of our decomposition strategy fails because the unrolled prompt becomes too long and convoluted.' Section 4.4 notes Decomp-Ctxt does not outperform NoDecomp-Ctxt on HotpotQA with Codex. Appendix C shows performance drops with smaller models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims — outperforming prior work on few-shot prompting, hierarchical decomposition for hard sub-tasks, recursive decomposition for length generalization, better QA performance, and incorporating symbolic retrieval — are all supported by experimental results in Sections 4.1-4.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'the separate prompts are more effective at teaching hard sub-tasks than a single CoT prompt' (Section 4.1) are supported by controlled comparisons where only the decomposition strategy varies. The ablation studies (Section E) use single-variable manipulation across decomposition schemes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims DECOMP is 'an effective few-shot paradigm for solving complex tasks' broadly, but experiments cover only specific symbolic manipulation tasks, synthetic QA, and multi-hop QA. The title 'Solving Complex Tasks' is broader than the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why DECOMP outperforms baselines. For example, it does not consider whether the improvement comes from effectively giving the model more tokens/context, prompt engineering effects, or task-specific factors rather than the decomposition principle itself."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Exact Match and Answer F1 on specific tasks and frames claims in terms of those metrics. There is no proxy gap — claims are about task accuracy on the tested benchmarks."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: 'text-davinci-002 InstructGPT3 model' (Section 4), 'code-davinci-002' (Section 4.4), 'davinci-001' (Section 4.2), 'text-curie-001' (Appendix C), 'Flan-T5-Large (0.7B), Flan-T5-XL (3B), and Flan-T5-XXL (11B)' (Section 4.4)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix G spanning approximately 25 pages, covering all decomposer prompts, sub-task handler prompts, CoT baselines, and Least-to-Most prompts for all tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'greedy search' is used for inference (Section 3.2), effectively specifying temperature=0. Retrieval hyperparameter K is tuned via grid search with explicit ranges stated (Appendix A.2). Number of in-context examples is specified per prompt."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The decomposer-handler framework is described in detail in Section 3, including the controller that routes sub-queries to handlers, the foreach and foreach_merge operators (Section 3.1), the inference procedure with EOQ markers (Section 3.2), and hierarchical/recursive capabilities (Section 3.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation data construction is documented: letter concatenation words from name lists (footnote 4), CommaQA-E dataset generation with context limit constraints (Section 4.3), open-domain QA corpus construction by combining paragraphs (Appendix A.1 with exact corpus sizes: 430,225 and 139,416 paragraphs)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated limitations section. The conclusion (Section 5) is a brief summary of contributions with no discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no consideration of how prompt selection, model choice, or task selection might affect the conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what settings were not tested, or what types of complex tasks DECOMP might not be suited for."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Datasets and code are released at https://github.com/allenai/DecomP (footnote 1). Evaluation datasets, prompts, and model outputs can be inspected."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are described: letter concatenation words from forebears.io name lists (footnote 4), CommaQA-E from Khot et al. (2022) with size reduction for context limits, open-domain QA from HotpotQA/MuSiQue/2WikiMultihopQA with corpus construction details (Appendix A.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and synthetically generated evaluation sets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw datasets to evaluation is documented: corpus construction with paragraph counts (Appendix A.1), hyperparameter tuning split (100 questions) vs evaluation split (300 questions), and random sampling from test sets with stated sample sizes."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'This work was supported in part by the National Science Foundation under grants IIS2007290.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Allen Institute for AI (AI2), Stony Brook University, University of Edinburgh. The work is done during an internship at AI2 (footnote on first page)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Science Foundation is an independent funding agency with no financial stake in whether Decomposed Prompting outperforms alternative methods."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-3 (text-davinci-002, code-davinci-002) and Flan-T5 models without stating their training data cutoff dates. This is necessary to assess whether evaluation benchmarks appeared in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The paper uses public benchmarks (HotpotQA, MuSiQue, GSM8K, MultiArith) with models that may have been trained on them."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No benchmark contamination analysis is performed despite using GPT-3 models on public benchmarks. HotpotQA (2018), GSM8K (2021), and MultiArith (2015) were all published before the GPT-3 training data was collected."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM performance on automated benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or API cost is reported. The paper mentions subsampling 'due to costs with API usage' (footnotes in Appendix B) but never quantifies the actual costs. DECOMP makes multiple API calls per example but the cost is not compared to baselines."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, number of tokens consumed, or computation time."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. While results are averaged across 3 prompt variants, there is no analysis of sensitivity to random seeds in the models themselves."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The number of prompt variants is stated: 'We create three different prompts for all our baselines and present the average' (Section 4.1). Per-prompt results are shown in Appendix D."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 specifies the search ranges: 'For NoDecomp-Ctxt, we search K ∈ {6, 8, 10} for GPT3 models and K ∈ {2, 4, 6, 8} for Flan-T5-* models. For Decomp-Ctxt, we search K ∈ {2, 4, 6}.' Selection is on a held-out set of 100 questions."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 describes configuration selection: 'We select it based on a grid search on a set of values to maximize performance on a held out set of 100 questions for each dataset.' Selection is done on a separate set from the evaluation set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both their own system and the baselines (CoT, Least-to-Most). No discussion of potential bias from authors implementing baselines, despite Lucic et al. (2018) showing systematic underperformance of author-implemented baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "DECOMP makes multiple API calls per example (decomposer + multiple sub-task handlers), substantially more than CoT (single call). Appendix E notes one decomposition uses O(n) vs O(log n) calls, but no systematic compute-performance comparison across methods is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the benchmarks (letter concatenation, list reversal, CommaQA) actually measure the claimed capabilities of 'solving complex tasks.' The synthetic tasks may not represent real-world complexity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "When comparing models (Flan-T5 variants, Codex), the same DECOMP scaffold is used. The main comparisons (DECOMP vs CoT vs L2M) are explicitly comparing different decomposition strategies, which is the intended variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Several benchmarks (HotpotQA 2018, MultiArith 2015, GSM8K 2021) were published before GPT-3's training data collection, meaning solutions could be in the training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The few-shot prompts contain examples with answers that could prime the model, but this is not analyzed as a potential confound."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation examples are independent of the training data. The letter concatenation task uses common first and last names that likely appear frequently in GPT-3's training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference tests, or decontamination procedures are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "DECOMP outperforms CoT and Least-to-Most prompting on kth letter concatenation, achieving near-100% accuracy where CoT achieves 6-23%",
       "evidence": "Figure 7 shows DECOMP at 97-98% vs CoT at 6-22.7% across N=3,4,5 words with k=3. Results averaged across 3 prompts. Per-prompt results in Fig 18 confirm consistency.",
       "supported": "strong"
     },
     {
       "claim": "Recursive DECOMP enables length generalization for list reversal, far exceeding CoT on longer sequences",
       "evidence": "Figure 8 shows DECOMP at 42% on N=10 vs CoT at 4.5% and CoT w/ rollout at 1%. Uses davinci-001 to demonstrate gains on a weaker model.",
       "supported": "moderate"
     },
     {
       "claim": "DECOMP outperforms CoT on CommaQA-E for both IID and compositional generalization settings",
       "evidence": "Figure 10 shows DECOMP(fine) at 59.7/64.2% vs CoT at 42.0/33.8% on IID/Comp.Gen. splits. Per-prompt results in Fig 19 confirm consistency across prompt choices.",
       "supported": "moderate"
     },
     {
       "claim": "DECOMP with retrieval significantly outperforms no-retrieval and simple-retrieval baselines on open-domain multi-hop QA",
       "evidence": "Figure 12 shows Decomp-Ctxt outperforms No-Ctxt and NoDecomp-Ctxt on MuSiQue (25.4 vs 18.3/21.4) and 2WikiMultihopQA (64.1 vs 38.1/47.1) with Codex. Exception: comparable to NoDecomp-Ctxt on HotpotQA.",
       "supported": "moderate"
     },
     {
       "claim": "Post-processing CoT with a GPT-3 answer extractor yields 14-17 point improvements on math QA",
       "evidence": "Figure 16 shows improvement from 78→95 on MultiArith and 36→50.6 on GSM8K. However, evaluation uses random subsamples (200 and 300 examples respectively) of the test sets.",
       "supported": "moderate"
     },
     {
       "claim": "The separate sub-task prompts are more effective at teaching hard sub-tasks than a single CoT prompt using the same reasoning procedure",
       "evidence": "Section 4.1 compares DECOMP to 'CoT w/ rollout' which describes the same decomposition steps in a single prompt. DECOMP (98%) outperforms CoT w/ rollout (70.5%) on N=4, k=3. The reasoning steps are identical but factored into separate prompts.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "Results are reported as point estimates averaged across 3 prompts. No confidence intervals, error bars, or significance tests are used despite making comparative claims. The reader cannot assess whether differences are statistically meaningful."
     },
     {
       "flag": "No contamination analysis",
       "detail": "GPT-3 models are evaluated on public benchmarks (HotpotQA, GSM8K, MultiArith) that predate the models' training data. No analysis of whether the models have seen evaluation data during training, which could inflate accuracy for all methods."
     },
     {
       "flag": "No limitations discussion",
       "detail": "The paper lacks any limitations section or discussion of threats to validity. Important limitations such as API cost overhead, sensitivity to prompt design, and generalizability to other task types are not addressed."
     },
     {
       "flag": "Compute cost not compared",
       "detail": "DECOMP makes multiple API calls per example (decomposer + multiple sub-task handlers), potentially much more expensive than CoT's single call. This cost is never quantified or compared, despite the paper acknowledging API costs forced them to subsample evaluation sets."
     },
     {
       "flag": "Small evaluation sample sizes without justification",
       "detail": "Evaluation uses 100-300 examples per dataset, subsampled due to API costs. No power analysis or justification for whether these sample sizes are sufficient to draw reliable conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": [
         "Jason Wei",
         "Xuezhi Wang",
         "Dale Schuurmans",
         "Maarten Bosma",
         "Ed Chi",
         "Quoc Le",
         "Denny Zhou"
       ],
       "year": 2022,
       "relevance": "Foundational chain-of-thought prompting method that DECOMP extends; key baseline for all experiments."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": [
         "Tom Brown",
         "Benjamin Mann",
         "Nick Ryder"
       ],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot prompting capabilities that underpin this work."
     },
     {
       "title": "Least-to-most prompting enables complex reasoning in large language models",
       "authors": [
         "Denny Zhou",
         "Nathanael Scharli",
         "Le Hou",
         "Jason Wei"
       ],
       "year": 2023,
       "relevance": "Closest prior work to DECOMP; generates sub-questions from easiest to hardest. Direct baseline comparison."
     },
     {
       "title": "Successive prompting for decomposing complex questions",
       "authors": [
         "Dheeru Dua",
         "Shivanshu Gupta",
         "Sameer Singh",
         "Matt Gardner"
       ],
       "year": 2022,
       "relevance": "Similar decomposition approach using sequential prompting for question answering; key comparison point."
     },
     {
       "title": "PAL: Program-aided language models",
       "authors": [
         "Luyu Gao",
         "Aman Madaan",
         "Shuyan Zhou"
       ],
       "year": 2022,
       "arxiv_id": "2211.10435",
       "relevance": "Combines LLMs with symbolic computation for reasoning tasks; related modular approach to LLM problem-solving."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": [
         "Timo Schick",
         "Jane Dwivedi-Yu",
         "Roberto Dessi"
       ],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Tool-augmented LLMs that learn to call external APIs; related approach to integrating external functions."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": [
         "Xuezhi Wang",
         "Jason Wei",
         "Dale Schuurmans",
         "Quoc Le",
         "Ed Chi",
         "Denny Zhou"
       ],
       "year": 2023,
       "relevance": "Alternative method for improving CoT reasoning via sampling consistency; complementary to decomposition approach."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": [
         "Karl Cobbe",
         "Vineet Kosaraju",
         "Mohammad Bavarian"
       ],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "GSM8K benchmark used for evaluation; training verifiers for mathematical reasoning."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": [
         "Aakanksha Chowdhery",
         "Sharan Narang",
         "Jacob Devlin"
       ],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "Large-scale LLM demonstrating chain-of-thought capabilities; related to the scaling of prompting approaches."
     },
     {
       "title": "Language model cascades",
       "authors": [
         "David Dohan",
         "Winnie Xu",
         "Aitor Lewkowycz"
       ],
       "year": 2022,
       "arxiv_id": "2207.10342",
       "relevance": "Theoretical framework for composing LLMs as probabilistic programs; formalizes the cascade structure DECOMP uses."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": [
         "Long Ouyang",
         "Jeff Wu",
         "Xu Jiang"
       ],
       "year": 2022,
       "relevance": "InstructGPT paper for the text-davinci-002 model used as the primary LLM in DECOMP experiments."
     },
     {
       "title": "Measuring and narrowing the compositionality gap in language models",
       "authors": [
         "Ofir Press",
         "Muru Zhang",
         "Sewon Min",
         "Ludwig Schmidt",
         "Noah A Smith",
         "Mike Lewis"
       ],
       "year": 2022,
       "arxiv_id": "2210.03350",
       "relevance": "Analyzes compositional reasoning limitations in LLMs; DECOMP addresses by decomposing compositional tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Decomposed prompting is a directly applicable technique for building LLM pipelines with modular sub-task handlers, relevant to prompt engineers and AI application developers."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that separate sub-task prompts outperform a single CoT using the same reasoning procedure is mildly surprising, but modular decomposition beating monolithic approaches is not counterintuitive."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised or relevant to this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper positions itself as a natural extension of CoT and least-to-most prompting rather than challenging them."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and prompts are released on GitHub (allenai/DecomP), allowing reproduction with moderate effort though it requires GPT-3 API access."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Allen Institute for AI (AI2) is well-respected in NLP research but not a household name in the broader tech community."
     }
   }
 }