ai-research-survey

scan.json (27855B)

---

 {
   "paper": {
     "title": "Evolving Excellence: Automated Optimization of LLM-based Agents",
     "authors": [
       "Paul Brookes",
       "Vardan Voskanyan",
       "Rafail Giavrimis",
       "Matthew Truscott",
       "Mina Ilieva",
       "Chrystalla Pavlou",
       "Alexandru Staicu",
       "Manal Adham",
       "Will Evers-Hood",
       "Jingzhi Gong",
       "Kejia Zhang",
       "Matvey Fedoseev",
       "Vishal Sharma",
       "Roman Bauer",
       "Zheng Wang",
       "Hema Nair",
       "Wei Jie",
       "Tianhua Xu",
       "Aurora Constantin",
       "Carmine Ventre",
       "Leslie Kanthan",
       "Michail Basios"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.09108",
     "doi": "10.48550/arXiv.2512.09108"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Artemis, an evolutionary optimization platform, achieves 13.6% improvement on competitive programming (ALE Agent), 10.1% on code optimization (Mini-SWE), 36.9% token cost reduction on math reasoning (CrewAI), and 22% accuracy improvement on primary math (MathTales-Teacher with Qwen2.5-7B). Results suggest automated agent configuration tuning is most effective for under-optimized agents with clear metrics, while already well-tuned systems show limited accuracy gains but may benefit from cost optimization.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states 'we are going to open source the code for all four case study agents' as future intent. No repository URL is provided. The Artemis platform itself is proprietary."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used (AtCoder Heuristic Contest, SWE-Perf, Math Odyssey, GSM8K) are publicly available. The paper references public benchmark sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided. The paper does not describe hardware or software environment beyond mentioning Claude 3.5 Sonnet and Qwen2.5-7B as base models."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Section 7 acknowledges 'the complete Artemis platform setup cannot be shared.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 reports 95% confidence intervals for ALE Agent results: baseline 0.660 (95% CI: [0.594, 0.726]), prompt optimized 0.750 (95% CI: [0.689, 0.811])."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Multiple significance tests are reported: Mann-Whitney U test for Mini-SWE (p < 0.005), p-values for ALE (p = 0.10), CrewAI accuracy (p = 0.478), CrewAI tokens (p < 10^-6), MathTales (p < 0.001)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are reported throughout: 13.6% improvement from 66.0% to 75.0% (ALE), 10.1% gain (Mini-SWE), 36.9% token reduction (CrewAI), 22% accuracy improvement from 0.59 to 0.81 (MathTales)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes used. The ALE benchmark has 40 problems, CrewAI uses stratified samples of 30 problems, MathTales uses 50 validation and 300 evaluation problems, but no power analysis or justification for these sizes is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While confidence intervals are provided for ALE, no standard deviations or variance across experimental runs are reported. MathTales reports averages over 3 runs but without spread measures. The paper does not report variance across optimization runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each agent system is compared against its unoptimized baseline configuration. Table 3 summarizes baseline vs. optimized results for all four agents."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baselines are the agents' own unoptimized configurations. No comparison against other optimization methods (DSPy, ADAS, AFlow, PromptBreeder) is included despite Table 1 listing them as related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted to determine which components of Artemis (semantic mutations, crossover, hierarchical evaluation, Bayesian optimization) contribute to the improvements. The ALE Agent compares prompt vs. search optimization strategies, but this is not a component ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: ALE uses acceptance rate, average performance, and average rank. Mini-SWE reports apply rate, correctness, and performance score. CrewAI reports accuracy and token cost. MathTales reports accuracy and completeness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the optimized configurations or outputs is included. All evaluation is automated through benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "MathTales uses a separate validation set (50 problems) for optimization and a larger evaluation set (300 problems) for final testing. CrewAI tests both 30×10 and 50×6 configurations against the full 387-problem corpus."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Mini-SWE provides per-project breakdowns (Figure 8) showing performance across 9 Python projects. CrewAI provides per-run breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.6 discusses when optimization does not work. Section 7 notes Mini-SWE had -0.1% for pylint, and CrewAI showed a slight accuracy decrease. The paper discusses conditions under which optimization provides limited benefit."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CrewAI showed a non-significant 3.7% accuracy decrease (p = 0.478). ALE results did not reach statistical significance (p = 0.10). Mini-SWE showed minimal/negative gains for some projects (pylint -0.1%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 13.6% ALE improvement, 10.1% Mini-SWE gain, 36.9% CrewAI cost reduction, and 22% MathTales accuracy improvement are all supported by results in Section 6 and Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Artemis 'delivers substantial improvements' and optimization 'can substantially enhance under-optimized systems,' but the study design does not control for confounds. Improvements could be due to simply trying more configurations rather than Artemis's specific evolutionary approach. No comparison with random search or other optimization baselines."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automated Optimization of LLM-based Agents' broadly, but results are on only 4 specific agents. Section 7 discusses limitations but the abstract and introduction present results as generalizable ('making sophisticated optimization accessible to practitioners')."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the improvements could be achieved by simpler methods (random search, manual prompt engineering by a domain expert). No consideration of whether the evolutionary approach specifically contributes versus simply trying many configurations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific metrics (acceptance rate, performance score, token cost, accuracy) and frames claims at the same granularity. It does not inflate benchmark scores into broader claims about 'agent intelligence' or similar."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Mini-SWE states 'Claude 3.5 Sonnet' without a snapshot date or API version. MathTales uses 'Qwen2.5-7B.' ALE Agent and CrewAI do not specify which LLM they use. The LLMs used in Artemis's mutation/crossover operators are not specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Before/after prompt text is provided for all four agents in Figures 5, 7, 13, and 14, showing the actual prompt content used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other LLM sampling parameters are reported. Evolutionary algorithm parameters (population size, generations) are mentioned only for MathTales (2 generations, population size 3). Other agents lack these details."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The Artemis platform's workflow is described in Section 4 with three stages (project setup, component discovery, optimization strategies). The hierarchical evaluation strategy is described. Agent pipelines are described at a high level for each case study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "For CrewAI, the stratified sampling method for selecting 30 problems from 387 is mentioned but not described in detail. No documentation of how benchmark data was prepared or filtered for any agent."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Conclusion, Limitations, and Future Work' includes a dedicated paragraph discussing limitations including optimization effectiveness varying with initial configuration quality, generalizability concerns, and computational costs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 identifies specific threats: optimization effectiveness varies with initial configuration quality, ALE did not reach statistical significance, CrewAI showed accuracy decrease, and overfitting to benchmarks is acknowledged as a risk."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses that well-tuned agents may not benefit, it does not explicitly state what types of agents, tasks, or domains the results do NOT apply to. The broad framing of the title and introduction exceeds the tested scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-problem results, full optimization logs, individual run outputs) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Each experiment describes the benchmark source, number of problems, and evaluation procedure. Section 5 details the experimental setup for all four agents."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluations use automated benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from optimization runs to final reported results is not documented. For CrewAI, it's unclear how the 30-problem stratified samples were drawn. The relationship between optimization (validation) and final evaluation runs is not fully described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section states: 'This work was supported by EU Horizon 2020 Grant 101008280 (DIOR).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed. Most authors are affiliated with TurinTech AI, the company that developed Artemis."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "TurinTech AI, the company that developed and commercially offers Artemis, employs most of the authors. The EU grant supports the project but the primary conflict is that the authors are evaluating their own commercial product."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Multiple authors work for TurinTech AI which commercially sells Artemis, but this conflict is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training cutoff dates are stated for Claude 3.5 Sonnet or Qwen2.5-7B. The LLMs used in Artemis's operators are not specified at all."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark problems (GSM8K published 2021, AtCoder problems, SWE-Perf) could have appeared in the training data of the LLMs used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) is widely known to be contaminated in many models. AtCoder problems are public. No contamination analysis is conducted for any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports $24-26 per evaluation run for ALE, Section 5.2 reports $30-60 for Mini-SWE. Total optimization time is reported: 411.2 hours for ALE prompt optimization, 260.5 hours for search, 9 hours for Mini-SWE."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total optimization hours are reported: 671.7 hours for ALE Agent, 9 hours for Mini-SWE. Section 5.2 mentions '20-30 hours evaluation time for a full benchmarking.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results are not shown across multiple random seeds for any agent."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "CrewAI states 12 evaluation runs and tests 30×10 and 50×6 configurations. MathTales reports 3 evaluation runs. ALE evaluates on all 40 problems per run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Only MathTales states its evolutionary parameters (2 generations, population size 3). Other agents do not specify the number of configurations explored, generations, or total evolutionary search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "MathTales explicitly selects the best configuration from validation set performance and evaluates on a separate 300-problem evaluation set. CrewAI tests generalization with stratified sampling."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple statistical tests are performed across 4 agents and multiple metrics, but no multiple comparison correction (Bonferroni, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own commercial product (Artemis) against unoptimized baselines without acknowledging self-evaluation bias. No independent evaluation is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance curves as a function of compute budget are shown. It is unclear how performance scales with optimization time. ALE required 671.7 hours vs Mini-SWE's 9 hours, but no analysis of diminishing returns."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks (AtCoder, SWE-Perf, Math Odyssey, GSM8K) validly measure the capabilities the paper claims to optimize. GSM8K in particular is known to have saturation and contamination issues."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper varies agent configurations through Artemis while keeping the scaffold constant per agent, but does not discuss whether scaffold choice confounds results. Different agents use different scaffolds, making cross-agent comparisons unreliable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. GSM8K (2021) predates all models used. AtCoder problems are public. No analysis of whether models have seen benchmark solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, whether optimized prompts effectively encode benchmark-specific patterns rather than general strategies."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether validation and evaluation sets share structural similarities that could inflate generalization claims."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied to any benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "Artemis achieves 13.6% improvement in acceptance rate for the ALE Agent on AtCoder Heuristic Contest through prompt optimization.",
       "evidence": "Section 6.1, Table 3: acceptance rate rose from 0.660 (95% CI: [0.594, 0.726]) to 0.750 (95% CI: [0.689, 0.811]). p = 0.10.",
       "supported": "moderate"
     },
     {
       "claim": "Mini-SWE Agent shows a statistically significant 10.1% performance improvement on SWE-Perf.",
       "evidence": "Section 6.2, Table 3: performance score improved from 0.891 to 0.981, p < 0.005 via Mann-Whitney U test. Apply rate and correctness maintained at 92.1% and 87.9%.",
       "supported": "strong"
     },
     {
       "claim": "CrewAI Agent achieves 36.9% reduction in token cost with only a non-significant 3.7% accuracy decrease.",
       "evidence": "Section 6.3, Table 3: tokens reduced from 12033 to 7329 (p < 10^-6), accuracy dropped from 0.82 to 0.78 (p = 0.478).",
       "supported": "strong"
     },
     {
       "claim": "MathTales-Teacher Agent achieves 22% accuracy improvement on GSM8K primary-level mathematics.",
       "evidence": "Section 6.4, Table 3: accuracy improved from 0.59 to 0.81 (p < 0.001), completeness from 0.796 to 0.917 (p < 0.001). Results evaluated on 300 problems across 3 runs.",
       "supported": "strong"
     },
     {
       "claim": "Artemis is a general-purpose framework that works across agent architectures and task types.",
       "evidence": "Results on 4 agents spanning competitive programming, code optimization, and math reasoning. However, only 4 agents tested, all by the same team, and the platform is proprietary.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Most authors are affiliated with TurinTech AI, which commercially develops and sells the Artemis platform. The paper evaluates Artemis without independent validation or comparison against competing optimization methods. No competing interests statement is included."
     },
     {
       "flag": "No comparison with alternative optimization methods",
       "detail": "Despite Table 1 listing APE, PromptBreeder, ADAS, AFlow, GEPA, and ShinkaEvolve as related work, no empirical comparison against any of these methods is conducted. The only baseline is the unoptimized agent configuration, making it impossible to assess whether Artemis's evolutionary approach adds value over simpler alternatives like random search."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "GSM8K (published 2021) is widely known to be contaminated in many LLMs. AtCoder problems are publicly available. No contamination analysis is performed, which is especially concerning because optimized prompts could be encoding benchmark-specific patterns."
     },
     {
       "flag": "Proprietary platform limits reproducibility",
       "detail": "The Artemis platform is proprietary and 'the complete Artemis platform setup cannot be shared' (Section 7). Agent code is promised for future release but not yet available. Results cannot be independently verified."
     },
     {
       "flag": "Statistical significance not reached for headline result",
       "detail": "The ALE Agent's 13.6% improvement (the largest absolute gain, featured prominently in the abstract) did not reach statistical significance (p = 0.10). This is acknowledged but the result is still presented as a headline finding."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models are human-level prompt engineers",
       "authors": ["Yongchao Zhou", "Andrei Ioan Muresanu", "Ziwen Han"],
       "year": 2023,
       "arxiv_id": "2211.01910",
       "relevance": "APE framework for automated prompt optimization, baseline approach for LLM-based prompt engineering."
     },
     {
       "title": "PromptBreeder: Self-referential self-improvement via prompt evolution",
       "authors": ["Chrisantha Fernando"],
       "year": 2023,
       "arxiv_id": "2309.16797",
       "relevance": "Evolutionary algorithms applied to prompt optimization, direct predecessor approach."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating code generation agents on real-world software engineering tasks."
     },
     {
       "title": "Automated design of agentic systems",
       "authors": ["Shengran Hu", "Cong Lu", "Jeff Clune"],
       "year": 2024,
       "arxiv_id": "2408.08435",
       "relevance": "ADAS framework for code-based agent workflow optimization."
     },
     {
       "title": "AFlow: Automating agentic workflow generation",
       "authors": ["Jiayi Chen"],
       "year": 2024,
       "arxiv_id": "2410.10762",
       "relevance": "MCTS-based approach to automating agentic workflow design."
     },
     {
       "title": "Code generation with AlphaCodium: From prompt engineering to flow engineering",
       "authors": ["Tal Ridnik", "Dedy Kredo", "Itamar Friedman"],
       "year": 2024,
       "arxiv_id": "2401.08500",
       "relevance": "Flow engineering approach to code generation, demonstrating workflow-level optimization over prompt-only methods."
     },
     {
       "title": "Why do multi-agent LLM systems fail? MAST: Multi-agent system failure taxonomy",
       "authors": ["Weize Chen"],
       "year": 2024,
       "arxiv_id": "2503.13657",
       "relevance": "Taxonomy of failure modes in multi-agent LLM systems, motivating need for systematic agent optimization."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agent reasoning framework combining reasoning traces with actions."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn"],
       "year": 2023,
       "relevance": "Memory-based self-feedback framework for LLM agents without weight updates."
     },
     {
       "title": "SWE-Perf: can language models optimize code performance on real-world repositories?",
       "authors": ["Xinyi He"],
       "year": 2025,
       "relevance": "Benchmark used in this paper for evaluating code performance optimization agents."
     },
     {
       "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery",
       "authors": ["Alexander Novikov"],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "Closed-loop LLM generation and verification pipeline for algorithmic improvement using evolutionary methods."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez"],
       "year": 2024,
       "relevance": "Agent framework for autonomous software engineering tasks, context for agent evaluation methods."
     }
   ]
 }