ai-research-survey

scan.json (26071B)

---

 {
   "paper": {
     "title": "More Bang for the Buck: Improving the Inference of Large Language Models at a Fixed Budget using Reset and Discard (ReD)",
     "authors": [
       "Sagi Meir",
       "Tommer D. Keidar",
       "Noam Levi",
       "Shlomi Reuveni",
       "Barak Hirshberg"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.21522",
     "doi": "10.48550/arXiv.2601.21522"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The paper connects pass@k to coverage@cost via renewal theory and shows that power-law pass@k with exponent α<1 leads to sublinear coverage growth under solve-to-completion. Reset-and-Discard (ReD) provably improves coverage@cost for any budget and any difficulty distribution, with τ=1 (reset every attempt) proven optimal. Experiments on three LLMs with HumanEval confirm ReD yields large savings in attempts, tokens, and USD, and enables efficient estimation of inference power-law exponents.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Sec. 6 states 'All the code used to generate the results of this paper will be posted on GitHub' — a promise of future release, not an actual release. No URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses HumanEval (Chen et al., 2021), a publicly available benchmark with 164 problems."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the Groq API and model names but provides no requirements.txt, Dockerfile, or library version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The experimental procedure is described narratively in Sec. 6 but there are no scripts, README, or commands to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1 and 2 show shaded regions representing standard deviation over 100 random realizations. Fig. 3 reports the power-law exponent as 0.34 ± 0.01."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparisons between ReD and solve-to-completion rely on visual inspection of curves and theoretical proofs, not formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented graphically in Figures 1-2 without explicit numerical effect sizes. The paper does not report, e.g., 'ReD achieves 80% coverage with X% fewer attempts than solve-to-completion.' Specific savings are not quantified in text."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval has 164 problems. No justification is provided for why this benchmark size is sufficient for the empirical claims. The choice of 100 random realizations is also not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation across 100 random realizations is shown as shaded regions in Figures 1 and 2. The power-law fit reports ±0.01 uncertainty."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares ReD against the standard solve-to-completion protocol throughout Sec. 6 and Figures 1-2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Solve-to-completion is the standard, prevailing practice for LLM inference allocation. It is the natural and appropriate baseline. The paper also cites related budget-aware work (Wang et al. 2024, FrugalGPT)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Theorem 4.1 proves τ=1 is optimal, effectively ablating the reset interval parameter. The paper also compares across three models and three cost metrics (attempts, tokens, USD), showing how different factors affect the comparison."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three cost metrics are used: cumulative attempts, cumulative tokens, and USD cost (Fig. 2). Coverage is the primary outcome metric."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant — the paper evaluates a budget allocation policy on automatically verifiable coding tasks. Correctness is determined by test suite execution."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "ReD is a parameter-free allocation policy (τ=1 is provably optimal, not tuned on data). The entire HumanEval benchmark serves as the evaluation set with no tuning phase."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-category or per-difficulty breakdown of HumanEval results is provided. Only aggregate coverage curves are shown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No specific failure cases are discussed. The paper does not analyze which types of problems ReD helps most or least, nor show examples where the method is less effective."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that power-law exponent inference does not work for the two larger models because they solve nearly all questions in very few rounds (Sec. 6). It also notes gpt-oss-20b's verbosity hurts its token efficiency (Sec. 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: the power-law → sublinear growth connection (Sec. 3), ReD improving coverage@cost (Theorem 4.1, Sec. 4), budget savings prediction (Sec. 4.3), power-law exponent inference (Sec. 5), and empirical validation (Sec. 6)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The central causal claim — 'ReD improves coverage@cost' — is justified by rigorous mathematical proof (Theorem 4.1, Appendix B) showing that resetting every attempt minimizes the mean attempts per question for any difficulty distribution. Empirical results confirm the theory."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The theoretical results are general (any pass@k with power-law behavior). Empirical claims are bounded: 'Experiments on three LLMs using HumanEval demonstrate...' (abstract). The limitations section explicitly bounds scope to 'perfectly verifiable tasks' and 'parallel sampling.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 7 discusses that sequential/correlated sampling may outperform i.i.d. sampling (an alternative to ReD's approach), that different verification methods (Best-of-N, majority vote, reward models) could change the dynamics, and that model verbosity affects token-based comparisons."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It measures coverage@cost (unique problems solved per budget) and frames results in exactly those terms, not broader claims about 'model capability' or 'intelligence.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: 'llama-3.1-8b-instant', 'llama-3.3-70b-versatile', and 'openai/gpt-oss-20b' with a citation to the model card (OpenAI, 2025)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full system prompt is provided verbatim in Sec. 6: 'You are an intelligent coding assistant. You will be provided with a function signature and docstring...' The paper also states the standard HumanEval prompt (function signature + docstring) was used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0.8 is stated (Sec. 6). k=100 attempts per question per model is stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is simple repeated sampling with regex extraction of output code."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 6 describes the full experimental procedure: evaluating pass@k up to k=100, saving results in a questions×attempts matrix, recording input/output tokens, shuffling rows and columns to generate realizations, and analyzing under both protocols."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 7 contains a 'Limitations and future work' subsection with substantive discussion of scope boundaries."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific limitations: only perfectly verifiable tasks are considered, only i.i.d. parallel sampling (not sequential/backtracking), and a perfect verifier is assumed. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 7 explicitly states: 'we considered only perfectly verifiable tasks,' identifies verification under imperfect verifiers as open, and notes that sequential sampling (correlated attempts) is not addressed but may be more beneficial."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The results matrix (per-question, per-attempt outcomes) is not released. Code and data are promised for future release but not yet available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 6 describes using the Groq API to evaluate each model on HumanEval for k=100 attempts, recording per-question per-attempt success and token counts in a results matrix."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is the standard HumanEval benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 6 documents the pipeline: query models via Groq API → save results matrix (questions×attempts) with token counts → shuffle rows/columns → analyze under both solve-to-completion and ReD protocols."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists funding: Israel Science Foundation (grants 1037/22, 1312/22), Pazy Foundation (grant 415-2023), ERC Horizon 2020 (grant 947731), TAD, VATAT, and EPFL AI4science."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Tel Aviv University (School of Chemistry, CPLS, CCMMS) and EPFL. No conflicts — they evaluate third-party models, not their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "All funding sources are academic (ISF, ERC, Pazy Foundation, TAD, VATAT, EPFL). None have a financial interest in the outcome of inference allocation policy research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three models (llama-3.1-8b-instant, llama-3.3-70b-versatile, gpt-oss-20b)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. HumanEval (2021) is very likely in the training data of all three models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021. All models used were trained after 2021 and likely saw HumanEval solutions. No contamination discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-million-token pricing for all three models. Figure 2 (right panel) shows cumulative USD cost. Cost is a central metric of the paper."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states k=100 attempts per question per model, 164 questions (HumanEval), and provides token pricing (Table 1). Cumulative token counts and USD costs are shown in Figure 2."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 100 random realizations (row/column shuffles of the results matrix), with standard deviations shown as shaded regions in Figures 1 and 2."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 6 states 'we plot the mean coverage over 100 random realizations' with standard deviation. k=100 attempts per question is also stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The method's only parameter (τ) is provably optimal at τ=1 (Theorem 4.1), so no hyperparameter search is needed. Temperature 0.8 is the only tunable parameter for the LLM calls, stated without search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "τ=1 is mathematically proven optimal for any difficulty distribution (Theorem 4.1, Appendix B). No selection from candidate configurations is needed."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not perform multiple statistical tests — comparisons are shown graphically and the main results are proved theoretically."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both ReD and the solve-to-completion baseline. No discussion of potential author-implementation bias, though the theoretical proofs mitigate this concern substantially."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's core contribution: coverage@cost plots show performance as a function of compute budget measured in attempts, tokens, and USD (Figures 1 and 2)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses HumanEval without discussing its construct validity or limitations as a benchmark. No analysis of whether HumanEval adequately represents the diversity of real coding tasks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models receive the same prompt and use simple repeated sampling with regex extraction."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021. All three models were trained after 2021 and may have seen solutions. No temporal leakage discussion is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The standard HumanEval prompt includes function signatures and docstrings, but no analysis of feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval problems are independent of training data or each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "Power-law pass@k with exponent 0 < α < 1 leads to sublinear growth of coverage@cost under solve-to-completion",
       "evidence": "Derived analytically via renewal theory and Z-transforms (Eq. 7, Sec. 3.3, Appendix A.2)",
       "supported": "strong"
     },
     {
       "claim": "ReD improves coverage@cost for any budget and any difficulty distribution, and τ=1 is the optimal resetting strategy",
       "evidence": "Proven in Theorem 4.1 and Appendix B using Chebyshev sum inequality. Corollary B.1 shows E[T] ≥ E[Tτ] for all τ.",
       "supported": "strong"
     },
     {
       "claim": "For α < 1, ReD changes coverage@cost growth from sublinear to linear",
       "evidence": "Shown analytically in Eq. 12 (Sec. 4.2). E[Tτ] is always finite for finite τ, making coverage@cost grow linearly.",
       "supported": "strong"
     },
     {
       "claim": "ReD substantially reduces attempts, tokens, and USD cost to reach a desired coverage on HumanEval",
       "evidence": "Figures 1 and 2 show ReD outperforms solve-to-completion across three models and three cost metrics. Mean ± std over 100 realizations shown.",
       "supported": "moderate"
     },
     {
       "claim": "The power-law exponent can be inferred from ReD runs via linear regression of ⟨Rn⟩/⟨Rn+1 - Rn⟩ vs n",
       "evidence": "Derived analytically (Eq. 22, Sec. 5.3). Demonstrated for llama-3.1-8b-instant: α = 0.34 ± 0.01 (R² = 0.96), matching direct estimation from pass@k (Fig. 3, Fig. S1).",
       "supported": "moderate"
     },
     {
       "claim": "ReD on a small model (llama-3.1-8b) can be more cost-effective than using larger models with solve-to-completion",
       "evidence": "Fig. 2 shows ReD on llama-8b outperforms llama-70b (solve-to-completion) in attempts up to ~90% coverage, and outperforms gpt-oss-20b in USD for nearly all coverage levels.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination discussion",
       "detail": "HumanEval was published in 2021; all three models were likely trained on it. Contamination could distort the pass@k curves that underpin the theoretical framework's empirical validation. The paper does not mention this."
     },
     {
       "flag": "Single benchmark validation",
       "detail": "All empirical results are on HumanEval (164 problems). While the theoretical results are general, the empirical demonstration is limited to one small benchmark. Results may not transfer to harder or more diverse benchmarks."
     },
     {
       "flag": "Code not released",
       "detail": "The paper promises 'All the code used to generate the results of this paper will be posted on GitHub' but provides no URL. Results cannot currently be independently verified."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and pass@k metric, which this paper builds upon for coverage@cost analysis."
     },
     {
       "title": "Large language monkeys: Scaling inference compute with repeated sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Documents inference-time scaling of pass@k over large sample budgets, a key empirical foundation for ReD."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Studies compute-optimal allocation of test-time compute, directly related to ReD's budget allocation problem."
     },
     {
       "title": "Inference scaling laws: An empirical analysis of compute-optimal inference for problem-solving with language models",
       "authors": ["Yangzhen Wu", "Zhiqing Sun", "Shanda Li"],
       "year": 2024,
       "arxiv_id": "2408.00724",
       "relevance": "Empirical analysis of inference scaling laws that ReD builds upon for budget-aware inference."
     },
     {
       "title": "Codemonkeys: Scaling test-time compute for software engineering",
       "authors": ["Ryan Ehrlich", "Bradley Brown", "Jordan Juravsky"],
       "year": 2025,
       "arxiv_id": "2501.14723",
       "relevance": "Scales test-time compute for SE tasks via repeated sampling, demonstrating the problem ReD addresses."
     },
     {
       "title": "How do large language monkeys get their power (laws)?",
       "authors": ["Rylan Schaeffer", "Joshua Kazdan", "John Hughes"],
       "year": 2025,
       "arxiv_id": "2502.17578",
       "relevance": "Proposes statistical explanations for power-law pass@k behavior, a key assumption underlying ReD's theory."
     },
     {
       "title": "Efficient prediction of pass@k scaling in large language models",
       "authors": ["Joshua Kazdan", "Rylan Schaeffer", "Aya Allouah"],
       "year": 2025,
       "arxiv_id": "2510.05197",
       "relevance": "Develops sample-efficient estimators for pass@k scaling, complementary to ReD's exponent inference method."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Proposes cost-optimized LLM routing strategies; ReD cites it as a related hierarchical routing approach."
     },
     {
       "title": "A simple model of inference scaling laws",
       "authors": ["Noam Levi"],
       "year": 2025,
       "relevance": "Proposes a difficulty-mixture model for pass@k power-law behavior that ReD's framework directly builds on."
     },
     {
       "title": "Reasoning in token economies: Budget-aware evaluation of LLM reasoning strategies",
       "authors": ["Junlin Wang", "Siddhartha Jain", "Dejiao Zhang"],
       "year": 2024,
       "relevance": "Argues for budget-aware evaluation of reasoning strategies, directly motivating coverage@cost over pass@k."
     },
     {
       "title": "Smaller, weaker, yet better: Training LLM reasoners via compute-optimal sampling",
       "authors": ["Hritik Bansal", "Arian Hosseini", "Rishabh Agarwal"],
       "year": 2025,
       "relevance": "Studies compute-optimal sampling for training data generation, related to ReD's coverage optimization."
     },
     {
       "title": "From decoding to meta-generation: Inference-time algorithms for large language models",
       "authors": ["Sean Welleck", "Amanda Bertsch", "Matthew Finlayson"],
       "year": 2024,
       "arxiv_id": "2406.16838",
       "relevance": "Comprehensive survey of inference-time algorithms for LLMs, providing context for ReD's contribution."
     }
   ]
 }