ai-research-survey

scan.json (23147B)

---

 {
   "paper": {
     "title": "Are More LM Calls All You Need? Towards the Scaling Properties of Compound AI Systems",
     "authors": ["Lingjiao Chen", "Jared Quincy Davis", "Boris Hanin", "Peter Bailis", "Ion Stoica", "Matei Zaharia", "James Zou"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2403.02419",
     "doi": "10.48550/arXiv.2403.02419"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The performance of Vote and Filter-Vote compound AI systems is often non-monotonic in the number of LM calls: performance first increases then decreases (or vice versa). This is explained by the diversity of query difficulties — more calls help on easy queries but hurt on hard ones. The authors derive an analytical scaling model that accurately predicts performance and the optimal number of LM calls from a small number of samples.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We will release the code and datasets used in this paper' (Section 1) — this is a promise of future release, not an actual release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The real-world datasets used (MMLU Physics, TruthfulQA, GPQA, AVERITEC) are all publicly available standard benchmarks. The synthetic datasets are parameterized and fully described."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The experimental setup is described at a high level but not with reproducible commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates. While experiments are averaged over 1,000 runs, no confidence intervals or error bars are shown in figures or tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims about non-monotonic behavior and model fit quality are based on visual inspection and MSE values without formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Performance differences are shown visually in figures but not quantified with standardized effect measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 1,000 simulation runs were chosen, or why these particular dataset sizes were sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Although the paper averages over 1,000 runs, no standard deviation, IQR, or other spread measure is reported for any result."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Vote and Filter-Vote against each other and against single LM calls (K=1). The analytical scaling model is compared against empirical results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The systems studied (majority voting, filter-vote) are contemporary techniques used in state-of-the-art systems like Gemini CoT@32 and AlphaCode 2."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies parameters (K, α, p1, p2) and breaks down performance by easy/difficult queries, which serves as an ablation of the query difficulty distribution's effect."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is used as the evaluation metric across all experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant — the paper studies scaling properties of voting systems on objective multiple-choice benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The scaling model is fit on a training set of K values (2, 5, 10, 20, 50, 100) and used to predict performance for 100 randomly drawn K values, constituting a train/test separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by easy vs. difficult queries (Figures 2, 4b) and across four different datasets. Synthetic experiments vary α, p1, p2 systematically (Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses cases where more LM calls degrade performance (the core finding). Figure 4d gives concrete examples of easy and difficult queries."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The central finding is a negative result: more LM calls can hurt performance. The non-monotonic behavior is itself a cautionary finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about non-monotonic scaling, query difficulty explanation, and analytical model accuracy are all supported by the theoretical results and experiments in Sections 4-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that query difficulty diversity causes non-monotonic behavior is supported both by mathematical proof (Theorem 2) and controlled synthetic experiments where difficulty parameters are varied directly."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly limits scope to 'tasks with a fairly small number of possible responses (e.g., multiple-choice questions)' and notes 'tasks with many valid outputs, such as chat, remain under-explored' (Section 1). The Limitations section (Appendix B) further bounds generalization."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the non-monotonic behavior are discussed beyond the query difficulty framework. Other potential factors (prompt sensitivity, temperature effects, model stochasticity patterns) are not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on multiple-choice benchmarks and claims results about accuracy on multiple-choice benchmarks — the measurement matches the claims without proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'GPT-3.5-turbo-0125' (Section 5), which is an exact model version with snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for both the generator and filter are provided in Appendix D.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is reported as 0.1 (Appendix D.3). The number of queries per question (400) and simulation runs (1,000) are stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The systems are simple vote/filter-vote aggregation without agents, tools, or iterative workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how datasets are used (Appendix D.1), which partitions are selected, and how queries are formatted as multiple-choice questions. The simulation procedure (400 samples, random sampling with replacement for each K) is described in Appendix D.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B is a dedicated 'Limitations' section discussing scope boundaries."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: analysis is limited to two compound system types, experiments use only objective tasks, and predicting query difficulty without querying LMs remains open."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states boundaries: focuses on tasks with small answer spaces, only Vote and Filter-Vote designs, objective tasks only. Section 1 notes 'tasks with many valid outputs, such as chat, remain under-explored.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw LM responses (400 per query) are not released. Only aggregated performance curves are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: each query is sent to GPT-3.5-turbo-0125 400 times at temperature 0.1, and K responses are sampled with replacement, averaged over 1,000 runs (Appendix D.3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from query → 400 LM responses → random K-sample → vote/filter-vote → accuracy averaged over 1,000 runs is fully documented in Appendix D.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (Stanford, UC Berkeley, Princeton) are clearly listed on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is provided, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Several authors have known affiliations with AI companies (Databricks/Anyscale) which are not disclosed in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for GPT-3.5-turbo-0125 is not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-3.5 may have seen MMLU, TruthfulQA, or GPQA questions during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU was published in 2020 and TruthfulQA in 2021, both well before GPT-3.5's training. GPQA was published in late 2023. No contamination analysis is provided for any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes in the Conclusion that 'we do not discuss the cost of LM calls; this is an important dimension' but does not report any cost figures despite calling GPT-3.5 hundreds of times per query."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, API costs, or wall-clock times are reported, despite extensive API usage (400 calls × number of queries × 4 datasets)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While experiments are averaged over 1,000 simulation runs (random sampling from 400 pre-collected responses), no seed sensitivity analysis or variance across seeds is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All experiments are averaged over 1,000 runs' (Section 5) and '400 times' for initial query collection (Appendix D.3)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The temperature was set to 0.1 with no justification or search over alternatives. The scaling model fitting procedure uses specific K values with no discussion of sensitivity to this choice."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of temperature=0.1 and the specific K values used for fitting (2, 5, 10, 20, 50, 100) are not justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the analytical scaling model and evaluate it themselves without acknowledging potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about performance as a function of compute (number of LM calls). Figures 1, 2, 4, 5 all show performance curves across varying compute levels."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MMLU Physics, TruthfulQA, GPQA, or AVERITEC actually measure what they claim to measure, or whether voting behavior on multiple-choice questions generalizes to real-world compound AI system performance."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — the systems are simple vote/filter-vote aggregation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-3.5 may have been trained on benchmark data that predates its training cutoff."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the multiple-choice format or prompt structure leaks information about correct answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and benchmark questions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "The performance of Vote and Filter-Vote is often non-monotonic in the number of LM calls — first increasing then decreasing (or vice versa).",
       "evidence": "Empirically demonstrated across MMLU Physics, TruthfulQA, GPQA, and AVERITEC datasets with GPT-3.5 (Figure 1, Section 5).",
       "supported": "strong"
     },
     {
       "claim": "The non-monotonic behavior is caused by the diversity of query difficulties: more LM calls improve performance on easy queries but degrade performance on hard queries.",
       "evidence": "Theorem 2 provides mathematical conditions for when non-monotonicity emerges based on query difficulty distribution. Empirically validated with performance breakdowns (Figure 2, 4b) and systematic synthetic experiments (Figure 5).",
       "supported": "strong"
     },
     {
       "claim": "The analytical scaling model can accurately predict the performance of Vote and Filter-Vote and identify the optimal number of LM calls.",
       "evidence": "Scaling model predictions match empirical observations with MSE ranging from 1e-6 to 1e-4 (Figure 6). Predicted optimal K matches empirical optimal K exactly across all evaluated configurations (Table 2).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No cost analysis despite cost being central to the practical question",
       "detail": "The paper asks 'are more LM calls all you need?' but does not report any costs, despite the Conclusion acknowledging 'we do not discuss the cost of LM calls; this is an important dimension.' The practical implication is incomplete without cost data."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "Despite averaging over 1,000 runs, no error bars, confidence intervals, or standard deviations are reported for any empirical result. The reader cannot assess the statistical reliability of the performance curves."
     },
     {
       "flag": "Single model evaluation",
       "detail": "All experiments use only GPT-3.5-turbo-0125. The generality of the non-monotonic scaling behavior to other models is unknown."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Multiple benchmarks used (MMLU, TruthfulQA) predate GPT-3.5's training and could be contaminated, which would affect the easy/hard query distribution and potentially the scaling curves."
     }
   ],
   "cited_papers": [
     {
       "title": "The shift from models to compound ai systems",
       "authors": ["Matei Zaharia", "Omar Khattab", "Lingjiao Chen", "Jared Quincy Davis"],
       "year": 2024,
       "relevance": "Foundational blog post motivating compound AI systems as the new paradigm over monolithic models."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Key prior work on majority voting over multiple LLM reasoning paths to improve performance."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws work that this paper extends from training-time to inference-time scaling."
     },
     {
       "title": "Inverse scaling: When bigger isn't better",
       "authors": ["Ian R McKenzie", "Alexander Lyzhov", "Michael Pieler"],
       "year": 2023,
       "arxiv_id": "2306.09479",
       "relevance": "Demonstrates non-monotonic scaling in model size, parallel to this paper's findings on inference-time scaling."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba"],
       "year": 2023,
       "arxiv_id": "2305.14325",
       "relevance": "Multi-agent debate as a compound inference strategy using multiple LM calls."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Cost-efficient LLM inference strategies, directly relevant to compound system cost-performance tradeoffs."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM agents on real-world software engineering tasks."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Yupeng Chang", "Xu Wang", "Jindong Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey on LLM evaluation methodology and benchmarks."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Zhiheng Xi", "Wenxiang Chen", "Xin Guo"],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Survey of LLM-based agent architectures, relevant to understanding compound AI systems."
     },
     {
       "title": "GPQA: A graduate-level google-proof q&a benchmark",
       "authors": ["David Rein", "Betty Li Hou", "Asa Cooper Stickland"],
       "year": 2023,
       "arxiv_id": "2311.12022",
       "relevance": "Expert-level benchmark used in this paper's experiments, important for AI capability evaluation."
     }
   ]
 }