ai-research-survey

scan.json (26801B)

---

 {
   "paper": {
     "title": "AdaptEvolve: Improving Efficiency of Evolutionary AI Agents through Adaptive Model Selection",
     "authors": [
       "Pretam Ray",
       "Pratik Prabhanjan Brahma",
       "Zicheng Liu",
       "Emad Barsoum"
     ],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2602.11931"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is available at https://github.com/raypretam/adaptive_llm_selection', providing a working GitHub URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: LiveCodeBench v5 and MBPP. Both are standard public benchmarks that the authors did not modify."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The schema requires 'requirements.txt, Dockerfile, conda environment file, or a detailed Environment Setup section listing library versions.' The paper lists hyperparameters in Table 5 and mentions hardware (8x AMD MI250 GPUs), but does not provide a requirements.txt, Dockerfile, or library version listing. Citing scikit-learn and River without version numbers is insufficient. Hyperparameters are not the same as environment/dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and hyperparameters are listed in Table 5, the paper does not provide step-by-step reproduction instructions within the paper itself. No README commands or 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 and Table 2 are point estimates only (e.g., '73.6%', '35.4'). No confidence intervals, error bars, or uncertainty notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AdaptEvolve 'strictly outperforms' and 'significantly surpasses' baselines but provides no p-values, t-tests, or any statistical significance test to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The schema says 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper reports exactly this pattern: 'retains 97.9% of the 32B upper-bound accuracy (73.6% vs 75.2%) while reducing compute cost by 34.4%', '2.4 points (73.6% vs 71.2%)', and '41.5% while maintaining 97.1% of peak accuracy.' These provide percentage improvements with explicit baseline context, which satisfies the schema criterion."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The warm-up set of N=50 is used to train the decision tree router, but no justification or power analysis is provided for why 50 samples is sufficient to bootstrap a reliable classifier."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports standard deviations and skewness for confidence metrics in the calibration set analysis, but main results in Table 1 and Table 2 are single-run point estimates with no variance across seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: Pure Small model (4B Iterative), Pure Large model (32B Iterative), Random Routing, Static Decision Tree, and the Cascading baseline (Chen et al., 2023). See Table 1 and Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The Cascading baseline references Chen et al. (2023), which is recent. Random routing and pure-model bounds are appropriate reference points. The baselines represent reasonable comparisons for the setting."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 presents a pilot ablation comparing the Decision Tree router against a threshold-based linear switch on N=50 samples (Table 4), justifying the non-linear routing choice. The routing dynamics section also analyzes adaptation vs. static trees."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Accuracy, Compute Cost, and Efficiency (Acc/Cost) across all configurations in Tables 1 and 2, plus the Small:Large usage ratio — multiple metrics characterizing the cost-accuracy trade-off."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates code generation quality using automated pass/fail test cases on benchmarks. Human evaluation is clearly irrelevant to claims about routing efficiency in an automated system."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The warm-up calibration set (N=50) is distinct from the full evaluation. The main results in Table 1 use the full LiveCodeBench v5 (880 samples) and MBPP (974 samples), which are separate from the N=50 training set used to bootstrap the decision tree."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single aggregate numbers per benchmark (Table 1). No per-category or per-difficulty breakdown is provided (e.g., easy vs. hard problems within LiveCodeBench)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions that the method requires access to token-level log-probabilities, but no failure cases or error analysis of the router's incorrect routing decisions are presented."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments presented show the proposed method matching or improving efficiency relative to baselines. No configurations where AdaptEvolve failed or was worse are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 37.9% cost reduction and 97.5% accuracy retention. Table 1 and Section 3.6 support these numbers: LiveCodeBench shows 34.4% reduction at 97.9% accuracy, MBPP shows 41.5% at 97.1%, averaging to the stated figures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The schema states 'ablation studies (removing component X reduces performance by Y%) ARE causal claims — check whether the ablation design is adequate (controlled single-variable manipulation counts as YES).' The paper does controlled single-variable comparisons: static tree vs. adaptive tree (Table 1), threshold vs. tree (Table 4), and various routing strategies vs. pure baselines. Each comparison changes one variable while holding the benchmark, models, and other settings constant. This is adequate controlled experimental design for the causal claims made. The lack of statistical power (single run) is captured by other checklist items (significance_tests, variance_reported)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'intelligent resource allocation is a viable pathway for scalable agentic reasoning' — a broad generalization beyond the tested setting of two Qwen3 models (4B and 32B) on two Python coding benchmarks with OpenEvolve. The paper does not bound this claim to its tested models, framework, or benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed results, such as whether the efficiency gains could stem from benchmark-specific properties, the particular model family (Qwen3), or the warm-up set selection. No threats-to-validity section addresses confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 specifies 'Qwen3-4B (Small/Router Target) and Qwen3-32B (Large/Escalation Target)' from the Qwen3 family (Yang et al., 2025), with Table 5 confirming model names. These are specific enough given the paper's publication date and the Qwen3 technical report citation."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the framework as using few-shot exemplars from previous high-scoring generations, but provides no actual prompt text sent to the models. Section 3.1 mentions the framework but does not show any prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix A.4) provides a comprehensive hyperparameters table: temperature=0.6, top-p=0.95, max tokens=20000, timeout=3600s, decision tree max_depth=5, warm-up N=50, plus all evolutionary framework parameters (population size, archive size, etc.)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the adaptive routing scaffold in detail: Section 2 explains the decision tree router Φ, confidence metric computation, warm-up phase, Hoeffding Adaptive Tree for online adaptation, and Figure 1 provides a workflow diagram. The escalation logic (Table 6) and confidence computation (Appendix A.1) are fully specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document any preprocessing of the benchmark datasets. Section 3.3 states sample counts (880 for LiveCodeBench, 974 for MBPP) but provides no details on filtering, preprocessing, or any transformations applied to benchmark inputs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations' and discusses that the method requires access to token-level log-probabilities, restricting applicability to open-weight models or APIs exposing logprobs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses only one specific constraint (logprob access requirement). It does not address threats to validity such as single-run results without confidence intervals, potential benchmark contamination, or the generalizability of findings to other model families or frameworks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the logprob limitation implies some scope boundary, the paper does not explicitly state what the results do NOT show — e.g., that results apply only to Qwen3 models, only to Python coding tasks, only to OpenEvolve-style frameworks, or only to the two tested benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregated results are presented in tables. No raw per-sample predictions, routing decisions, or intermediate outputs are released alongside the code."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the benchmarks used (LiveCodeBench v5 with 880 samples, MBPP with 974 samples), and the warm-up phase collection (N=50) is described in Section 3.5 with the labeling logic in Table 6."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data comes from standard public benchmarks (LiveCodeBench, MBPP), so recruitment does not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The confidence computation pipeline is described (Appendix A.1), but the full data pipeline from benchmark input to final evaluation output — including how the evolutionary iterations proceed, how outputs are collected, and how accuracy is computed — is only partially described through references to OpenEvolve."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The footnote on the first page states 'Work done during Internship at Advanced Micro Devices, Inc. (AMD).' This explicitly identifies AMD as the institution that supported the work. While there is no formal 'Acknowledgments' section with grant numbers, the funding source (AMD via internship) is clearly disclosed to the reader. The schema asks 'Is the funding source disclosed?' and it is."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the title page: Pretam Ray (IIT Kharagpur) and three AMD employees. The footnote explicitly states the work was done during an internship at AMD."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work is performed at AMD and three of four authors are AMD employees. AMD produces the MI250 GPUs used in the experiments and has commercial interest in demonstrating efficient inference on AMD hardware. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears anywhere in the paper. The AMD affiliation is noted but no formal declaration of financial interests or competing interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen3-4B and Qwen3-32B but does not state the training data cutoff date for either model. The Qwen3 technical report (Yang et al., 2025) is cited but the cutoff date is not mentioned in the paper."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap appears in the paper. MBPP (2021) is particularly at risk of being in Qwen3 training data, but this is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LiveCodeBench is cited as 'contamination free' by design (Jain et al., 2024), but MBPP (Austin et al., 2021) predates Qwen3 by several years and is widely used, making contamination a real concern that is not addressed. The paper does not discuss contamination risk for either benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 and Table 1 explicitly report inference cost in normalized compute units (one 32B call = 1.0, one 4B call = 0.125), with total costs per configuration reported across both benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The schema asks 'Is the total computational budget stated? Look for: GPU hours, total API spend, hardware used, training time.' While Section 3.1 states the hardware (8x AMD MI250 GPUs) and Table 5 lists configuration parameters, the paper never states the TOTAL compute consumed — no GPU-hours, no wall-clock time for experiments, no total training/inference time. Listing hardware is necessary but not sufficient; the total budget (how long the hardware was used) is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "AdaptEvolve reduces total inference compute by an average of 37.9% across benchmarks while retaining 97.5% of the upper-bound accuracy of static large-model baselines.",
       "evidence": "Table 1 shows LiveCodeBench: 34.4% cost reduction, 97.9% accuracy retention (73.6/75.2); MBPP: 41.5% cost reduction, 97.1% accuracy retention (91.3/94.0). Section 3.6 reports these averages.",
       "supported": "moderate"
     },
     {
       "claim": "The Adaptive Hoeffding Tree strictly outperforms static decision trees by dynamically adjusting decision boundaries as the population evolves.",
       "evidence": "Table 1 shows HAT achieves 73.6% accuracy vs. 71.2% for static Decision Tree on LiveCodeBench (2.4 point improvement). Section 3.6 ('Adaptation to Drift') discusses this result.",
       "supported": "moderate"
     },
     {
       "claim": "Intrinsic confidence metrics (LGC, MC, TC, BWC) provide a reliable signal for predicting model solvability, with statistically separable distributions between solved and unsolved instances.",
       "evidence": "Table 3 shows mean LGC of 5.86 for unsolved vs. 7.98 for solved instances, with similar shifts across all four metrics. Section A.2 provides distributional analysis.",
       "supported": "moderate"
     },
     {
       "claim": "A Decision Tree router outperforms simple linear threshold-based routing in the pilot study.",
       "evidence": "Table 4 shows Decision Tree achieves 74.12% accuracy at 2.02 cost (Eff=36.7) vs. Threshold Switch at 70.07% accuracy at 2.08 cost (Eff=33.7) on a stratified N=50 subset.",
       "supported": "weak"
     },
     {
       "claim": "AdaptEvolve achieves a superior Pareto frontier compared to the Cascading baseline, with similar accuracy but lower compute cost.",
       "evidence": "Table 2 shows HAT: 73.6% accuracy at 2.08 cost (Eff=35.4) vs. Cascading baseline: 73.8% accuracy at 2.81 cost (Eff=26.3).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AdaptEvolve proposes a confidence-driven model selection framework for evolutionary agentic coding systems that routes between a small (4B) and large (32B) Qwen3 model based on token-level entropy signals. Evaluated on LiveCodeBench v5 and MBPP using the OpenEvolve framework, the method reduces inference compute by an average of 37.9% while retaining 97.5% of the large-model accuracy ceiling. An Adaptive Hoeffding Tree that handles distribution shift outperforms static decision tree routing. The approach requires access to token-level log-probabilities, limiting applicability to open-weight models or logprob-exposing APIs.",
   "red_flags": [
     {
       "flag": "No variance or error bars on main results",
       "detail": "All results in Tables 1 and 2 are single-run point estimates. With an evolutionary stochastic process (random seed=42 fixed), it is impossible to assess result stability without multiple seeds. The claimed improvements (e.g., 2.4 accuracy points on LiveCodeBench) may not be statistically significant."
     },
     {
       "flag": "AMD affiliation conflict of interest",
       "detail": "Three of four authors are AMD employees, experiments run on AMD MI250 GPUs, and the work was performed during an AMD internship. AMD has commercial interest in demonstrating efficient inference on its hardware. No competing interests statement is provided."
     },
     {
       "flag": "MBPP benchmark contamination risk unaddressed",
       "detail": "MBPP was published in 2021 and is widely used in LLM training. The paper uses Qwen3 models without stating training data cutoffs or addressing whether MBPP examples appear in Qwen3 training data."
     },
     {
       "flag": "Overgeneralized conclusions",
       "detail": "The conclusion claims 'intelligent resource allocation is a viable pathway for scalable agentic reasoning' based solely on two Qwen3 models on two Python coding benchmarks within one framework (OpenEvolve). This generalization substantially outstrips the evidence."
     },
     {
       "flag": "Tiny pilot study used for architectural decisions",
       "detail": "The decision to use a Decision Tree over linear thresholding (Appendix A.3) is based on N=50 samples — the same 50 samples used to train the router. This circular validation does not provide independent evidence for the architectural choice."
     },
     {
       "flag": "No prompts provided",
       "detail": "The paper describes using few-shot exemplars from prior high-scoring generations but does not provide any actual prompt text sent to the models, making full reproduction dependent entirely on the code repository."
     }
   ],
   "cited_papers": [
     {
       "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery",
       "authors": [
         "Alexander Novikov",
         "Ngân Vũ",
         "Marvin Eisenberger"
       ],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "Foundational evolutionary coding agent that AdaptEvolve builds upon; directly relevant to agentic AI coding system evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": [
         "Naman Jain",
         "King Han",
         "Alex Gu"
       ],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-aware code generation benchmark used to evaluate AdaptEvolve; relevant to LLM coding capability evaluation methodology."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": [
         "Lingjiao Chen",
         "Matei Zaharia",
         "James Zou"
       ],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Model cascading baseline that AdaptEvolve compares against; represents a key approach in cost-efficient LLM inference."
     },
     {
       "title": "RouteLLM: Learning to route LLMs with preference data",
       "authors": [
         "Isaac Ong",
         "Amjad Almahairi",
         "Vincent Wu"
       ],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "LLM routing system for cost-quality trade-offs; directly related to the routing methodology studied in AdaptEvolve."
     },
     {
       "title": "Deep think with confidence",
       "authors": [
         "Yichao Fu",
         "Xuewei Wang",
         "Yuandong Tian",
         "Jiawei Zhao"
       ],
       "year": 2025,
       "arxiv_id": "2508.15260",
       "relevance": "Source of the entropy-based confidence metrics (LGC, MC, TC, BWC) adopted by AdaptEvolve for model selection."
     },
     {
       "title": "ShinkaEvolve: Towards open-ended and sample-efficient program evolution",
       "authors": [
         "Robert Tjarko Lange",
         "Yuki Imajuku",
         "Edoardo Cetin"
       ],
       "year": 2025,
       "arxiv_id": "2509.19349",
       "relevance": "Related evolutionary agentic coding framework; relevant to the survey's scope on agentic AI programming systems."
     },
     {
       "title": "Qwen3 technical report",
       "authors": [
         "An Yang"
       ],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Describes the Qwen3 model family (4B and 32B) used in AdaptEvolve experiments; relevant to LLM capability and efficiency evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": [
         "Jacob Austin",
         "Augustus Odena",
         "Maxwell Nye"
       ],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used for evaluation in AdaptEvolve; foundational LLM code generation evaluation paper."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": [
         "Xuezhi Wang",
         "Jason Wei",
         "Dale Schuurmans"
       ],
       "year": 2022,
       "relevance": "Self-consistency mechanism referenced as improving small model capabilities; relevant to LLM reasoning methodology."
     },
     {
       "title": "Speculative decoding with big little decoder",
       "authors": [
         "Sehoon Kim",
         "Karttikeya Mangalam",
         "Suhong Moon"
       ],
       "year": 2023,
       "relevance": "Speculative decoding approach for efficient LLM inference; compared conceptually to model cascade approaches in AdaptEvolve."
     },
     {
       "title": "AlphaResearch: Accelerating new algorithm discovery with language models",
       "authors": [
         "Zhaojian Yu",
         "Kaiyue Feng",
         "Yilun Zhao"
       ],
       "year": 2025,
       "arxiv_id": "2511.08522",
       "relevance": "Related work on accelerating agentic systems with improved planning; cited in context of evolutionary AI agent optimization."
     }
   ]
 }