ai-research-survey

scan-v5.json (28361B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Evolving Excellence: Automated Optimization of LLM-based Agents",
     "authors": [
       "Paul Brookes",
       "Vardan Voskanyan",
       "Rafail Giavrimis",
       "Matthew Truscott",
       "Mina Ilieva",
       "Chrystalla Pavlou",
       "Alexandru Staicu",
       "Manal Adham",
       "Will Evers-Hood",
       "Jingzhi Gong",
       "Kejia Zhang",
       "Matvey Fedoseev",
       "Vishal Sharma",
       "Roman Bauer",
       "Zheng Wang",
       "Hema Nair",
       "Wei Jie",
       "Tianhua Xu",
       "Aurora Constantin",
       "Carmine Ventre",
       "Leslie Kanthan",
       "Michail Basios"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.09108",
     "doi": "10.48550/arXiv.2512.09108"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's specific claims (13.6% ALE, 10.1% Mini-SWE, 36.9% CrewAI token reduction, 22% MathTales accuracy) are all substantiated by empirical results in Section 6 with reported statistics.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes improvements to evolutionary optimization but never ablates whether the evolutionary mechanism itself (vs. any prompt variation) is causal; no comparison against random search or manual prompt engineering is included.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly bounds claims: 'automated optimization is not universally beneficial' and conditions for success (poorly-tuned agents, well-defined metrics) are spelled out; project-level variance in Mini-SWE is also acknowledged.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The central alternative — that any prompt rewrite (manual or random) would yield similar gains — is never considered. The paper does not discuss whether the evolutionary mechanism adds value over simpler baselines.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims map directly to measured outcomes: acceptance rate is directly counted, performance score comes from benchmark execution, token cost is directly measured; no conflation of proxy metrics with broader constructs.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.6 'Key Insights and Limitations' and Section 8 'Conclusion, Limitations, and Future Work' both contain dedicated limitations discussions with specific examples beyond a single concluding sentence.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: benchmark overfitting ('improvements on benchmarks may not translate to real-world usage'), project-level variance in Mini-SWE (pylint -0.1%), non-significant ALE results (p=0.10), and small sample sizes limiting validation.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states Artemis 'works best for tasks with objective solving approaches, well-defined success criteria, and measurable outcomes' and that 'well-tuned agents may offer limited room for further optimization.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgment section explicitly discloses EU Horizon 2020 Grant 101008280 (DIOR) as funding source with a CORDIS project URL.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list institutional affiliations in the paper header; multiple authors are identified as affiliated with TurinTech AI, the commercial developer of Artemis.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "EU Horizon 2020 is an independent public research funder with no financial stake in Artemis's commercial success.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "TurinTech AI employees evaluate their own commercial Artemis platform. No competing interests statement or financial interests declaration is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Definition 1 formally defines 'agent configuration' as C = (P, T, M, Θ) covering prompts, tool descriptions, model assignments, and continuous parameters; the optimization objective is formalized in Equation 1.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four numbered contributions are explicitly enumerated in the introduction: the Artemis platform, novel semantic mutation/crossover operators, systematic experiments with statistical validation, and analysis of optimization success factors.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 organizes related work into four paradigms and Table 1 provides structured comparative analysis across five dimensions, clearly positioning Artemis relative to APE, PromptBreeder, ADAS, AFlow, AlphaEvolve, and DSPy.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "Section 7 states 'we are going to open source the code for all four case study agents as supplementary material' — a future promise only; the complete Artemis platform itself cannot be shared.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation benchmarks are standard public resources: AtCoder Heuristic Contest, SWE-Perf, Math Odyssey, and GSM8K are publicly available.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or dependency specifications are provided. Model names (Claude 3.5 Sonnet, Qwen2.5-7B) are given without snapshot dates or API version identifiers.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided. The Artemis platform is proprietary, and the promised agent code has not yet been released at time of publication.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "95% CIs are explicitly reported for ALE results (baseline [0.594, 0.726], optimized [0.689, 0.811]); other experiments report p-values from hypothesis tests supporting variance estimates.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Mann-Whitney U test used for Mini-SWE (p<0.005); p-values reported throughout for all comparisons including CrewAI cost (p<10^-6) and MathTales (p<0.001).",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Percentage improvements with baseline values are given for all experiments (e.g., 66.0%→75.0% for ALE, 12033→7329 tokens for CrewAI), providing effect size context throughout.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No power analysis or statistical justification is provided for sample sizes (40 ALE problems, 140 Mini-SWE functions, 30 CrewAI problems per run, 50/300 MathTales problems). Sizes appear benchmark-determined.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Variance is captured via 95% CIs for ALE, 12 evaluation runs for CrewAI with run-level comparison, and 3 repeated runs for MathTales; non-parametric testing for Mini-SWE incorporates distributional spread.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Pre-optimization baselines are included for all four agents with reported metrics: ALE 0.660, Mini-SWE 0.891, CrewAI accuracy 0.82 and 12033 tokens, MathTales accuracy 0.59.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": false,
           "justification": "The 'baselines' are unoptimized agent configurations only; no empirical comparison against other optimization methods (DSPy, APE, PromptBreeder, random search) is conducted. Table 1 comparison is purely conceptual.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "No ablation isolates the evolutionary mechanism from alternatives. The ALE prompt-vs-search comparison tests two optimization strategies but does not ablate whether the genetic algorithm itself adds value over simpler approaches.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics are used across experiments: acceptance rate, performance score, accuracy, completeness, token cost, and per-project breakdowns for Mini-SWE.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "All evaluations use automated benchmark scoring (acceptance rate, performance score, token count, mathematical correctness); human evaluation is not relevant for these tasks.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "MathTales-Teacher uses a separate validation set (50 problems for optimization selection) and evaluation set (300 problems for final assessment). CrewAI uses stratified sampling with held-out test problems.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Mini-SWE reports per-project results across 9 Python libraries (requests +20%, scikit-learn +29%, astropy +62%, pylint -0.1%). MathTales reports accuracy and completeness separately.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 7 explicitly discusses failure cases: pylint -0.1%, CrewAI accuracy slight decrease, ALE non-significance, and the principle that well-tuned agents show limited improvement potential.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "CrewAI accuracy decreased 4% (p=0.277); ALE improvements did not reach significance (p=0.10); Mini-SWE showed project-level negative variance (pylint). All are reported honestly.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Mini-SWE uses 'Claude 3.5 Sonnet' without snapshot date. The LLM ensemble used internally by Artemis is never specified. Only 'Qwen2.5-7B' provides a version identifier without weights checkpoint.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Before/after prompts are shown verbatim in Figures 5, 7, 13, and 14, covering all four agent case studies with full prompt text.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "GA parameters are sparsely reported: population size 3 and 2 generations mentioned only for MathTales. Temperature, top-p, and LLM inference parameters are never reported for any experiment.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The Artemis optimization pipeline (component discovery, local/global optimization, hierarchical evaluation with LLM scoring then benchmark execution) is described in Section 4 with figures. Each agent's architecture is described.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "Stratified sampling is mentioned for CrewAI (30 from 387 problems) and MathTales (50 from GSM8K), but the stratification criteria and problem selection procedures are not documented sufficiently to reproduce.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No raw evaluation outputs, optimization logs, or per-run results are released. The Artemis platform output cannot be independently verified.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses established public benchmarks (AtCoder Heuristic Contest, SWE-Perf, Math Odyssey, GSM8K) with citations, and evaluation metric definitions are provided for each benchmark.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "Standard public benchmarks are used; no participant recruitment is involved.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "The optimization pipeline is described conceptually but no code is released. The full path from problem input to fitness evaluation cannot be independently verified or reproduced.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for Claude 3.5 Sonnet and Qwen2.5-7B are not stated anywhere in the paper, despite using benchmarks (especially GSM8K from 2021) that are almost certainly in training data.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Potential overlap between training data and benchmarks (particularly GSM8K published 2021, likely memorized) is never discussed; the paper does not acknowledge that models may have seen these problems.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "GSM8K (2021) was almost certainly included in Claude 3.5 Sonnet's and Qwen2.5-7B's training data, directly undermining validity of the MathTales-Teacher results. This is never addressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Per-evaluation costs are explicitly reported: $24-26 per ALE run, $30-60 per Mini-SWE run, with per-problem token costs for CrewAI (12033 vs 7329 average tokens).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Total optimization time is reported: 671.7 hours for ALE (411.2h prompt + 260.5h search), 9 hours for Mini-SWE, with variable time noted for others.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Artemis achieves a 13.6% improvement in ALE Agent acceptance rate on AtCoder Heuristic Contest",
       "evidence": "Acceptance rate increased from 0.660 (95% CI: [0.594, 0.726]) to 0.750 (95% CI: [0.689, 0.811]); p=0.10, not statistically significant",
       "supported": "moderate"
     },
     {
       "claim": "Artemis achieves a statistically significant 10.1% performance improvement for Mini-SWE Agent on SWE-Perf",
       "evidence": "Performance score increased from 0.891 to 0.981 using Mann-Whitney U test (p<0.005); apply rate and correctness maintained at 92.1% and 87.9%",
       "supported": "strong"
     },
     {
       "claim": "Artemis achieves a 36.9% reduction in token cost for CrewAI Agent on Math Odyssey with only a non-significant 4% accuracy decrease",
       "evidence": "Average token count reduced from 12033 to 7329 (p<10^-6); accuracy decrease -3.7% (p=0.277) over 12 evaluation runs",
       "supported": "strong"
     },
     {
       "claim": "Artemis improves MathTales-Teacher Agent accuracy by 22% on GSM8K across a 300-problem evaluation set",
       "evidence": "Accuracy 0.59→0.81 and completeness 0.796→0.917 (p<0.001) across 3 repeated runs; however GSM8K contamination is unaddressed",
       "supported": "moderate"
     },
     {
       "claim": "Joint multi-component optimization captures interdependencies that isolated component optimization misses",
       "evidence": "Stated as motivation in the introduction but no empirical ablation compares joint vs. isolated optimization",
       "supported": "unsupported"
     },
     {
       "claim": "Evolutionary optimization outperforms manual trial-and-error for LLM agent configuration tuning",
       "evidence": "Only compared against unoptimized baselines; no comparison against manual prompt engineering, random search, or other automated methods",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "Artemis, a commercial evolutionary optimization platform by TurinTech AI, demonstrates statistically significant improvements for three of four evaluated agent systems: 10.1% for Mini-SWE code optimization (p<0.005), 36.9% token cost reduction for CrewAI math reasoning (p<10^-6), and 22% accuracy gain for a small Qwen2.5-7B model on GSM8K (p<0.001). The ALE competitive programming improvement (13.6%) did not reach statistical significance (p=0.10). Optimization effectiveness depends on initial configuration quality — poorly-tuned agents benefit most — and on task characteristics favoring well-defined objective metrics. Critical limitations include the absence of comparison against other optimization methods, no code release, potential GSM8K benchmark contamination, and an undisclosed commercial conflict of interest from the authors evaluating their own product.",
   "red_flags": [
     {
       "flag": "Undisclosed commercial conflict of interest",
       "detail": "TurinTech AI employees evaluate their own commercial Artemis platform. No competing interests statement is present despite the obvious financial interest in positive results."
     },
     {
       "flag": "Non-significant primary result prominently featured",
       "detail": "The 13.6% ALE improvement (p=0.10) fails the α=0.05 threshold yet is highlighted in the abstract, introduction, and conclusion as a key result."
     },
     {
       "flag": "No comparison against other optimization methods",
       "detail": "No empirical comparison against DSPy, APE, PromptBreeder, random search, or manual prompt engineering. The claim of superiority rests entirely on comparison to unoptimized baselines."
     },
     {
       "flag": "Code not released at publication",
       "detail": "Source code is promised as future open source but is unavailable at time of publication. The core Artemis platform cannot be shared, making reproduction impossible."
     },
     {
       "flag": "GSM8K contamination unaddressed",
       "detail": "GSM8K (published 2021) was almost certainly in training data for both Claude 3.5 Sonnet and Qwen2.5-7B. The 22% accuracy improvement may partly reflect prompts that better elicit memorized answers rather than improved reasoning."
     },
     {
       "flag": "Trivially small evolutionary search",
       "detail": "MathTales uses only population size 3 and 2 generations — equivalent to evaluating 6 total prompt variants. This is functionally indistinguishable from exhaustive search over a tiny candidate set."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Models are Human-Level Prompt Engineers (APE)",
       "relevance": "Foundational automated prompt optimization; direct prior work that Artemis builds upon and claims to extend beyond with full agent configuration optimization"
     },
     {
       "title": "PromptBreeder: Self-Referential Self-Improvement via Prompt Evolution",
       "relevance": "Evolutionary prompt optimization approach; closest prior work to Artemis's genetic algorithm methodology for natural language component optimization"
     },
     {
       "title": "Automated Design of Agentic Systems (ADAS)",
       "relevance": "Code-structure-based agent workflow optimization; Artemis explicitly positions itself as superior by being architecture-agnostic rather than requiring code-level access"
     },
     {
       "title": "AFlow: Automating Agentic Workflow Generation",
       "relevance": "MCTS-based workflow optimization; second key comparison target in Table 1 for positioning Artemis's generality"
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail? MAST: Multi-Agent System Failure Taxonomy",
       "relevance": "Taxonomy of 14 failure modes motivating the need for systematic agent configuration analysis and optimization"
     },
     {
       "title": "SWE-Bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Primary code agent benchmark; cited to contextualize Artemis's 57% resolution rate claim and as standard of comparison for code agent capability"
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems (GSM8K)",
       "relevance": "Evaluation benchmark used for MathTales-Teacher experiment; one of the most widely used math reasoning benchmarks in LLM research"
     },
     {
       "title": "AlphaEvolve: A Coding Agent for Scientific and Algorithmic Discovery",
       "relevance": "Related evolutionary LLM coding agent demonstrating novel algorithmic improvements through closed-loop generation and verification"
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "relevance": "Core reasoning-action architecture used in MathTales-Teacher agent; foundational agent framework being optimized by Artemis"
     },
     {
       "title": "SWE-Perf: Can Language Models Optimize Code Performance on Real-World Repositories?",
       "relevance": "Benchmark used for Mini-SWE Agent evaluation across 9 Python libraries; establishes the evaluation protocol for code performance optimization"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners facing LLM agent configuration overhead can directly apply the optimization concept, though Artemis is commercial and not open-source."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that cost optimization (36.9% token reduction) can be achieved with negligible accuracy loss is mildly surprising; most other results confirm expected behavior."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns raised; the work focuses entirely on prompt optimization for benchmark performance."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Undisclosed commercial conflict of interest and prominent reporting of a non-significant result could attract critical methodological scrutiny."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Artemis has a commercial web interface, but it is not publicly available or open-source; practitioners cannot easily try it without engaging TurinTech AI."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "TurinTech AI is a lesser-known commercial startup; use of Claude 3.5 Sonnet and GSM8K provides name recognition for the benchmarks but not the lab."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "25471098",
         "title": "Causality Is Graphically Simple",
         "points": 90,
         "comments": 7,
         "url": "https://news.ycombinator.com/item?id=25471098",
         "created_at": "2020-12-18T19:48:36Z"
       },
       {
         "hn_id": "45574705",
         "title": "StreamingVLM: Real-Time Understanding for Infinite Video Streams",
         "points": 33,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45574705",
         "created_at": "2025-10-14T00:02:18Z"
       },
       {
         "hn_id": "45591789",
         "title": "StreamingVLM: Real-Time Understanding for Infinite Video Streams",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45591789",
         "created_at": "2025-10-15T13:02:15Z"
       },
       {
         "hn_id": "42362464",
         "title": "RoboHanger: Learning Generalizable Robotic Hanger Insertion for Diverse Garments",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42362464",
         "created_at": "2024-12-09T02:05:35Z"
       }
     ],
     "top_points": 90,
     "total_points": 125,
     "total_comments": 7
   }
 }