scan-v4.json (33294B)
1 { 2 "scan_version": 4, 3 "paper_type": "empirical", 4 "paper": { 5 "title": "Evolving Excellence: Automated Optimization of LLM-based Agents", 6 "authors": [ 7 "Paul Brookes", 8 "Vardan Voskanyan", 9 "Rafail Giavrimis", 10 "Matthew Truscott", 11 "Mina Ilieva", 12 "Chrystalla Pavlou", 13 "Alexandru Staicu", 14 "Manal Adham", 15 "Will Everhood", 16 "Jingzhi Gong", 17 "Kejia Zhang", 18 "Matvey Fedoseev", 19 "Vishal Sharma", 20 "Roman Bauer", 21 "Zheng Wang", 22 "Hema Nair", 23 "Wei Jie", 24 "Tianhua Xu", 25 "Aurora Constantin", 26 "Carmine Ventre", 27 "Leslie Kanthan", 28 "Michail Basios" 29 ], 30 "year": 2025, 31 "venue": "arXiv.org", 32 "arxiv_id": "2512.09108", 33 "doi": "10.48550/arXiv.2512.09108" 34 }, 35 "checklist": { 36 "claims_and_evidence": { 37 "abstract_claims_supported": { 38 "applies": true, 39 "answer": true, 40 "justification": "Abstract claims of 13.6% ALE improvement, 10.1% Mini-SWE gain, 36.9% CrewAI cost reduction, and 22% MathTales accuracy improvement are all supported by results in Section 6 and Table 3.", 41 "source": "opus" 42 }, 43 "causal_claims_justified": { 44 "applies": true, 45 "answer": false, 46 "justification": "The paper claims Artemis 'delivers substantial improvements' and optimization 'can substantially enhance under-optimized systems,' but the study design does not control for confounds. Improvements could be due to simply trying more configurations rather than Artemis's specific evolutionary approach. No comparison with random search or other optimization baselines.", 47 "source": "opus" 48 }, 49 "generalization_bounded": { 50 "applies": true, 51 "answer": false, 52 "justification": "The title claims 'Automated Optimization of LLM-based Agents' broadly, but results are on only 4 specific agents. Section 7 discusses limitations but the abstract and introduction present results as generalizable ('making sophisticated optimization accessible to practitioners').", 53 "source": "opus" 54 }, 55 "alternative_explanations_discussed": { 56 "applies": true, 57 "answer": false, 58 "justification": "The paper does not discuss whether the improvements could be achieved by simpler methods (random search, manual prompt engineering by a domain expert). No consideration of whether the evolutionary approach specifically contributes versus simply trying many configurations.", 59 "source": "opus" 60 }, 61 "proxy_outcome_distinction": { 62 "applies": true, 63 "answer": true, 64 "justification": "The paper measures specific metrics (acceptance rate, performance score, token cost, accuracy) and frames claims at the same granularity. It does not inflate benchmark scores into broader claims about 'agent intelligence' or similar.", 65 "source": "opus" 66 } 67 }, 68 "limitations_and_scope": { 69 "limitations_section_present": { 70 "applies": true, 71 "answer": true, 72 "justification": "Section 8 'Conclusion, Limitations, and Future Work' includes a dedicated paragraph discussing limitations including optimization effectiveness varying with initial configuration quality, generalizability concerns, and computational costs.", 73 "source": "opus" 74 }, 75 "threats_to_validity_specific": { 76 "applies": true, 77 "answer": true, 78 "justification": "Section 8 identifies specific threats: optimization effectiveness varies with initial configuration quality, ALE did not reach statistical significance, CrewAI showed accuracy decrease, and overfitting to benchmarks is acknowledged as a risk.", 79 "source": "opus" 80 }, 81 "scope_boundaries_stated": { 82 "applies": true, 83 "answer": false, 84 "justification": "While the paper discusses that well-tuned agents may not benefit, it does not explicitly state what types of agents, tasks, or domains the results do NOT apply to. The broad framing of the title and introduction exceeds the tested scope.", 85 "source": "opus" 86 } 87 }, 88 "conflicts_of_interest": { 89 "funding_disclosed": { 90 "applies": true, 91 "answer": true, 92 "justification": "Acknowledgment section states: 'This work was supported by EU Horizon 2020 Grant 101008280 (DIOR).'", 93 "source": "opus" 94 }, 95 "affiliations_disclosed": { 96 "applies": true, 97 "answer": true, 98 "justification": "Author affiliations are listed. Most authors are affiliated with TurinTech AI, the company that developed Artemis.", 99 "source": "opus" 100 }, 101 "funder_independent_of_outcome": { 102 "applies": true, 103 "answer": false, 104 "justification": "TurinTech AI, the company that developed and commercially offers Artemis, employs most of the authors. The EU grant supports the project but the primary conflict is that the authors are evaluating their own commercial product.", 105 "source": "opus" 106 }, 107 "financial_interests_declared": { 108 "applies": true, 109 "answer": false, 110 "justification": "No competing interests or financial interests statement is present. Multiple authors work for TurinTech AI which commercially sells Artemis, but this conflict is not explicitly declared.", 111 "source": "opus" 112 } 113 }, 114 "scope_and_framing": { 115 "key_terms_defined": { 116 "applies": true, 117 "answer": true, 118 "justification": "Definition 1 formally defines 'agent configuration' as C=(P,T,M,Θ). 'Fitness function,' 'semantic mutation,' and 'crossover operators' are defined in context. Key terms are operationalized adequately.", 119 "source": "haiku" 120 }, 121 "intended_contribution_clear": { 122 "applies": true, 123 "answer": true, 124 "justification": "Four numbered contributions are explicitly listed in the introduction: the platform itself, novel mutation/crossover operators, systematic experiments with statistical validation, and analysis of optimization success factors.", 125 "source": "haiku" 126 }, 127 "engagement_with_prior_work": { 128 "applies": true, 129 "answer": true, 130 "justification": "Section 2 surveys four paradigms (prompt engineering, evolutionary algorithms, workflow optimization, multi-agent systems) and Table 1 directly compares Artemis to APE, PromptBreeder, ADAS, AFlow, AlphaCodium, GEPA, and ShinkaEvolve on five dimensions.", 131 "source": "haiku" 132 } 133 } 134 }, 135 "type_checklist": { 136 "empirical": { 137 "artifacts": { 138 "code_released": { 139 "applies": true, 140 "answer": false, 141 "justification": "Section 7 states 'we are going to open source the code for all four case study agents' as future intent. No repository URL is provided. The Artemis platform itself is proprietary.", 142 "source": "opus" 143 }, 144 "data_released": { 145 "applies": true, 146 "answer": true, 147 "justification": "The benchmarks used (AtCoder Heuristic Contest, SWE-Perf, Math Odyssey, GSM8K) are publicly available. The paper references public benchmark sources.", 148 "source": "opus" 149 }, 150 "environment_specified": { 151 "applies": true, 152 "answer": false, 153 "justification": "No environment specifications, requirements files, or dependency details are provided. The paper does not describe hardware or software environment beyond mentioning Claude 3.5 Sonnet and Qwen2.5-7B as base models.", 154 "source": "opus" 155 }, 156 "reproduction_instructions": { 157 "applies": true, 158 "answer": false, 159 "justification": "No step-by-step reproduction instructions are provided. Section 7 acknowledges 'the complete Artemis platform setup cannot be shared.'", 160 "source": "opus" 161 } 162 }, 163 "statistical_methodology": { 164 "confidence_intervals_or_error_bars": { 165 "applies": true, 166 "answer": true, 167 "justification": "Section 6.1 reports 95% confidence intervals for ALE Agent results: baseline 0.660 (95% CI: [0.594, 0.726]), prompt optimized 0.750 (95% CI: [0.689, 0.811]).", 168 "source": "opus" 169 }, 170 "significance_tests": { 171 "applies": true, 172 "answer": true, 173 "justification": "Multiple significance tests are reported: Mann-Whitney U test for Mini-SWE (p < 0.005), p-values for ALE (p = 0.10), CrewAI accuracy (p = 0.478), CrewAI tokens (p < 10^-6), MathTales (p < 0.001).", 174 "source": "opus" 175 }, 176 "effect_sizes_reported": { 177 "applies": true, 178 "answer": true, 179 "justification": "Percentage improvements with baseline context are reported throughout: 13.6% improvement from 66.0% to 75.0% (ALE), 10.1% gain (Mini-SWE), 36.9% token reduction (CrewAI), 22% accuracy improvement from 0.59 to 0.81 (MathTales).", 180 "source": "opus" 181 }, 182 "sample_size_justified": { 183 "applies": true, 184 "answer": false, 185 "justification": "No justification is given for the sample sizes used. The ALE benchmark has 40 problems, CrewAI uses stratified samples of 30 problems, MathTales uses 50 validation and 300 evaluation problems, but no power analysis or justification for these sizes is provided.", 186 "source": "opus" 187 }, 188 "variance_reported": { 189 "applies": true, 190 "answer": false, 191 "justification": "While confidence intervals are provided for ALE, no standard deviations or variance across experimental runs are reported. MathTales reports averages over 3 runs but without spread measures. The paper does not report variance across optimization runs or seeds.", 192 "source": "opus" 193 } 194 }, 195 "evaluation_design": { 196 "baselines_included": { 197 "applies": true, 198 "answer": true, 199 "justification": "Each agent system is compared against its unoptimized baseline configuration. Table 3 summarizes baseline vs. optimized results for all four agents.", 200 "source": "opus" 201 }, 202 "baselines_contemporary": { 203 "applies": true, 204 "answer": false, 205 "justification": "The only baselines are the agents' own unoptimized configurations. No comparison against other optimization methods (DSPy, ADAS, AFlow, PromptBreeder) is included despite Table 1 listing them as related work.", 206 "source": "opus" 207 }, 208 "ablation_study": { 209 "applies": true, 210 "answer": false, 211 "justification": "No ablation study is conducted to determine which components of Artemis (semantic mutations, crossover, hierarchical evaluation, Bayesian optimization) contribute to the improvements. The ALE Agent compares prompt vs. search optimization strategies, but this is not a component ablation.", 212 "source": "opus" 213 }, 214 "multiple_metrics": { 215 "applies": true, 216 "answer": true, 217 "justification": "Multiple metrics are reported: ALE uses acceptance rate, average performance, and average rank. Mini-SWE reports apply rate, correctness, and performance score. CrewAI reports accuracy and token cost. MathTales reports accuracy and completeness.", 218 "source": "opus" 219 }, 220 "human_evaluation": { 221 "applies": true, 222 "answer": false, 223 "justification": "No human evaluation of the optimized configurations or outputs is included. All evaluation is automated through benchmarks.", 224 "source": "opus" 225 }, 226 "held_out_test_set": { 227 "applies": true, 228 "answer": true, 229 "justification": "MathTales uses a separate validation set (50 problems) for optimization and a larger evaluation set (300 problems) for final testing. CrewAI tests both 30×10 and 50×6 configurations against the full 387-problem corpus.", 230 "source": "opus" 231 }, 232 "per_category_breakdown": { 233 "applies": true, 234 "answer": true, 235 "justification": "Mini-SWE provides per-project breakdowns (Figure 8) showing performance across 9 Python projects. CrewAI provides per-run breakdowns.", 236 "source": "opus" 237 }, 238 "failure_cases_discussed": { 239 "applies": true, 240 "answer": true, 241 "justification": "Section 6.6 discusses when optimization does not work. Section 7 notes Mini-SWE had -0.1% for pylint, and CrewAI showed a slight accuracy decrease. The paper discusses conditions under which optimization provides limited benefit.", 242 "source": "opus" 243 }, 244 "negative_results_reported": { 245 "applies": true, 246 "answer": true, 247 "justification": "CrewAI showed a non-significant 3.7% accuracy decrease (p = 0.478). ALE results did not reach statistical significance (p = 0.10). Mini-SWE showed minimal/negative gains for some projects (pylint -0.1%).", 248 "source": "opus" 249 } 250 }, 251 "setup_transparency": { 252 "model_versions_specified": { 253 "applies": true, 254 "answer": false, 255 "justification": "Mini-SWE states 'Claude 3.5 Sonnet' without a snapshot date or API version. MathTales uses 'Qwen2.5-7B.' ALE Agent and CrewAI do not specify which LLM they use. The LLMs used in Artemis's mutation/crossover operators are not specified.", 256 "source": "opus" 257 }, 258 "prompts_provided": { 259 "applies": true, 260 "answer": true, 261 "justification": "Before/after prompt text is provided for all four agents in Figures 5, 7, 13, and 14, showing the actual prompt content used in experiments.", 262 "source": "opus" 263 }, 264 "hyperparameters_reported": { 265 "applies": true, 266 "answer": false, 267 "justification": "No temperature, top-p, or other LLM sampling parameters are reported. Evolutionary algorithm parameters (population size, generations) are mentioned only for MathTales (2 generations, population size 3). Other agents lack these details.", 268 "source": "opus" 269 }, 270 "scaffolding_described": { 271 "applies": true, 272 "answer": true, 273 "justification": "The Artemis platform's workflow is described in Section 4 with three stages (project setup, component discovery, optimization strategies). The hierarchical evaluation strategy is described. Agent pipelines are described at a high level for each case study.", 274 "source": "opus" 275 }, 276 "data_preprocessing_documented": { 277 "applies": true, 278 "answer": false, 279 "justification": "For CrewAI, the stratified sampling method for selecting 30 problems from 387 is mentioned but not described in detail. No documentation of how benchmark data was prepared or filtered for any agent.", 280 "source": "opus" 281 } 282 }, 283 "data_integrity": { 284 "raw_data_available": { 285 "applies": true, 286 "answer": false, 287 "justification": "No raw experimental data (per-problem results, full optimization logs, individual run outputs) is made available.", 288 "source": "opus" 289 }, 290 "data_collection_described": { 291 "applies": true, 292 "answer": true, 293 "justification": "Each experiment describes the benchmark source, number of problems, and evaluation procedure. Section 5 details the experimental setup for all four agents.", 294 "source": "opus" 295 }, 296 "recruitment_methods_described": { 297 "applies": false, 298 "answer": false, 299 "justification": "No human participants. All evaluations use automated benchmarks.", 300 "source": "opus" 301 }, 302 "data_pipeline_documented": { 303 "applies": true, 304 "answer": false, 305 "justification": "The pipeline from optimization runs to final reported results is not documented. For CrewAI, it's unclear how the 30-problem stratified samples were drawn. The relationship between optimization (validation) and final evaluation runs is not fully described.", 306 "source": "opus" 307 } 308 }, 309 "contamination": { 310 "training_cutoff_stated": { 311 "applies": true, 312 "answer": false, 313 "justification": "No training cutoff dates are stated for Claude 3.5 Sonnet or Qwen2.5-7B. The LLMs used in Artemis's operators are not specified at all.", 314 "source": "opus" 315 }, 316 "train_test_overlap_discussed": { 317 "applies": true, 318 "answer": false, 319 "justification": "No discussion of whether the benchmark problems (GSM8K published 2021, AtCoder problems, SWE-Perf) could have appeared in the training data of the LLMs used.", 320 "source": "opus" 321 }, 322 "benchmark_contamination_addressed": { 323 "applies": true, 324 "answer": false, 325 "justification": "GSM8K (2021) is widely known to be contaminated in many models. AtCoder problems are public. No contamination analysis is conducted for any benchmark.", 326 "source": "opus" 327 } 328 }, 329 "human_studies": { 330 "pre_registered": { 331 "applies": false, 332 "answer": false, 333 "justification": "No human participants in this study.", 334 "source": "opus" 335 }, 336 "irb_or_ethics_approval": { 337 "applies": false, 338 "answer": false, 339 "justification": "No human participants in this study.", 340 "source": "opus" 341 }, 342 "demographics_reported": { 343 "applies": false, 344 "answer": false, 345 "justification": "No human participants in this study.", 346 "source": "opus" 347 }, 348 "inclusion_exclusion_criteria": { 349 "applies": false, 350 "answer": false, 351 "justification": "No human participants in this study.", 352 "source": "opus" 353 }, 354 "randomization_described": { 355 "applies": false, 356 "answer": false, 357 "justification": "No human participants in this study.", 358 "source": "opus" 359 }, 360 "blinding_described": { 361 "applies": false, 362 "answer": false, 363 "justification": "No human participants in this study.", 364 "source": "opus" 365 }, 366 "attrition_reported": { 367 "applies": false, 368 "answer": false, 369 "justification": "No human participants in this study.", 370 "source": "opus" 371 } 372 }, 373 "cost_and_practicality": { 374 "inference_cost_reported": { 375 "applies": true, 376 "answer": true, 377 "justification": "Section 5.1 reports $24-26 per evaluation run for ALE, Section 5.2 reports $30-60 for Mini-SWE. Total optimization time is reported: 411.2 hours for ALE prompt optimization, 260.5 hours for search, 9 hours for Mini-SWE.", 378 "source": "opus" 379 }, 380 "compute_budget_stated": { 381 "applies": true, 382 "answer": true, 383 "justification": "Total optimization hours are reported: 671.7 hours for ALE Agent, 9 hours for Mini-SWE. Section 5.2 mentions '20-30 hours evaluation time for a full benchmarking.'", 384 "source": "opus" 385 } 386 }, 387 "experimental_rigor": { 388 "seed_sensitivity_reported": { 389 "applies": true, 390 "answer": false, 391 "justification": "No seed sensitivity analysis is reported. Results are not shown across multiple random seeds for any agent.", 392 "source": "opus" 393 }, 394 "number_of_runs_stated": { 395 "applies": true, 396 "answer": true, 397 "justification": "CrewAI states 12 evaluation runs and tests 30×10 and 50×6 configurations. MathTales reports 3 evaluation runs. ALE evaluates on all 40 problems per run.", 398 "source": "opus" 399 }, 400 "hyperparameter_search_budget": { 401 "applies": true, 402 "answer": false, 403 "justification": "Only MathTales states its evolutionary parameters (2 generations, population size 3). Other agents do not specify the number of configurations explored, generations, or total evolutionary search budget.", 404 "source": "opus" 405 }, 406 "best_config_selection_justified": { 407 "applies": true, 408 "answer": true, 409 "justification": "MathTales explicitly selects the best configuration from validation set performance and evaluates on a separate 300-problem evaluation set. CrewAI tests generalization with stratified sampling.", 410 "source": "opus" 411 }, 412 "multiple_comparison_correction": { 413 "applies": true, 414 "answer": false, 415 "justification": "Multiple statistical tests are performed across 4 agents and multiple metrics, but no multiple comparison correction (Bonferroni, etc.) is applied.", 416 "source": "opus" 417 }, 418 "self_comparison_bias_addressed": { 419 "applies": true, 420 "answer": false, 421 "justification": "The authors evaluate their own commercial product (Artemis) against unoptimized baselines without acknowledging self-evaluation bias. No independent evaluation is conducted.", 422 "source": "opus" 423 }, 424 "compute_budget_vs_performance": { 425 "applies": true, 426 "answer": false, 427 "justification": "No performance curves as a function of compute budget are shown. It is unclear how performance scales with optimization time. ALE required 671.7 hours vs Mini-SWE's 9 hours, but no analysis of diminishing returns.", 428 "source": "opus" 429 }, 430 "benchmark_construct_validity": { 431 "applies": true, 432 "answer": false, 433 "justification": "No discussion of whether the benchmarks (AtCoder, SWE-Perf, Math Odyssey, GSM8K) validly measure the capabilities the paper claims to optimize. GSM8K in particular is known to have saturation and contamination issues.", 434 "source": "opus" 435 }, 436 "scaffold_confound_addressed": { 437 "applies": true, 438 "answer": false, 439 "justification": "The paper varies agent configurations through Artemis while keeping the scaffold constant per agent, but does not discuss whether scaffold choice confounds results. Different agents use different scaffolds, making cross-agent comparisons unreliable.", 440 "source": "opus" 441 } 442 }, 443 "data_leakage": { 444 "temporal_leakage_addressed": { 445 "applies": true, 446 "answer": false, 447 "justification": "No discussion of temporal leakage. GSM8K (2021) predates all models used. AtCoder problems are public. No analysis of whether models have seen benchmark solutions.", 448 "source": "opus" 449 }, 450 "feature_leakage_addressed": { 451 "applies": true, 452 "answer": false, 453 "justification": "No discussion of whether the evaluation setup leaks information. For example, whether optimized prompts effectively encode benchmark-specific patterns rather than general strategies.", 454 "source": "opus" 455 }, 456 "non_independence_addressed": { 457 "applies": true, 458 "answer": false, 459 "justification": "No discussion of whether validation and evaluation sets share structural similarities that could inflate generalization claims.", 460 "source": "opus" 461 }, 462 "leakage_detection_method": { 463 "applies": true, 464 "answer": false, 465 "justification": "No concrete leakage detection or prevention method is applied to any benchmark.", 466 "source": "opus" 467 } 468 } 469 } 470 }, 471 "claims": [ 472 { 473 "claim": "Artemis achieves 13.6% improvement in ALE Agent competitive programming acceptance rate (66.0%→75.0%) through prompt optimization", 474 "evidence": "40-problem AtCoder Heuristic Contest benchmark; 95% CI [0.689, 0.811] for optimized vs [0.594, 0.726] baseline; p=0.10 — not statistically significant", 475 "supported": "moderate" 476 }, 477 { 478 "claim": "Mini-SWE Agent shows statistically significant 10.1% performance improvement on SWE-Perf code optimization (p<0.005)", 479 "evidence": "Mann-Whitney U test across 140 optimization instances in 9 Python projects; apply rate maintained at 92.1% and correctness at 87.9%", 480 "supported": "strong" 481 }, 482 { 483 "claim": "CrewAI Agent achieves 36.9% reduction in token cost (12,033→7,329 tokens) with a non-significant 4% accuracy decrease", 484 "evidence": "12 evaluation runs of 30 problems each on Math Odyssey; p<10^-6 for cost reduction, p=0.277 for accuracy change", 485 "supported": "strong" 486 }, 487 { 488 "claim": "MathTales-Teacher Agent (Qwen2.5-7B) achieves 22% accuracy improvement on GSM8K, generalizing from 50-problem validation to 300-problem evaluation set", 489 "evidence": "3 repeated evaluation runs on held-out 300-problem set; accuracy 0.59→0.81, completeness 0.796→0.917, both p<0.001", 490 "supported": "strong" 491 }, 492 { 493 "claim": "Evolutionary optimization generalizes across architecturally distinct agent systems without requiring code modification", 494 "evidence": "Four agents tested spanning competitive programming, code optimization, and math reasoning with different frameworks (ReAct, CrewAI, mini-SWE-agent, ALE pipeline)", 495 "supported": "moderate" 496 }, 497 { 498 "claim": "Initial configuration quality is the primary determinant of optimization success: vague prompts yield large gains, already-tuned agents yield minimal gains", 499 "evidence": "Qualitative comparison of CrewAI (already-tuned, <4% accuracy gain) vs ALE/Mini-SWE (vague prompts, 10-14% gain); no quantitative measure of 'configuration quality' defined", 500 "supported": "weak" 501 } 502 ], 503 "methodology_tags": [ 504 "benchmark-eval", 505 "case-study" 506 ], 507 "key_findings": "Artemis, a proprietary evolutionary optimization platform from TurinTech AI, achieves statistically significant improvements on 3 of 4 tested LLM agents: 10.1% on code performance optimization (Mini-SWE, p<0.005), 36.9% token cost reduction (CrewAI, p<10^-6), and 22% accuracy improvement on primary-school math (MathTales, p<0.001). A 13.6% competitive programming improvement (ALE) did not reach statistical significance (p=0.10). The most credible finding is that vaguely-specified baseline configurations benefit substantially from automated prompt refinement, while already-tuned agents show diminishing returns. Critical limitations include the proprietary platform (not independently reproducible), the complete absence of comparisons against any contemporary optimization method, and no discussion of benchmark contamination for established datasets like GSM8K.", 508 "red_flags": [ 509 { 510 "flag": "Developer self-evaluation", 511 "detail": "TurinTech AI employees evaluate their own commercial Artemis platform with no independent replication; all four case studies use the company's proprietary tool." 512 }, 513 { 514 "flag": "No competing method comparison", 515 "detail": "Table 1 lists DSPy, APE, PromptBreeder, GEPA as direct alternatives, yet none are tested as baselines. Only unoptimized agents serve as comparison, making it impossible to assess whether Artemis outperforms simpler approaches." 516 }, 517 { 518 "flag": "Platform not reproducible", 519 "detail": "'The complete Artemis platform setup cannot be shared.' Only case study agent code is promised for future release, making independent validation structurally impossible." 520 }, 521 { 522 "flag": "ALE result not statistically significant", 523 "detail": "The largest prominently featured improvement (13.6% for ALE) has p=0.10 — not significant at α=0.05 — yet appears in the abstract alongside statistically significant results without clear differentiation." 524 }, 525 { 526 "flag": "Benchmark contamination ignored", 527 "detail": "GSM8K (2021), AtCoder problems, and Math Odyssey are used with Claude 3.5 Sonnet and Qwen2.5-7B without any discussion of whether these well-known datasets appeared in pretraining corpora." 528 }, 529 { 530 "flag": "No competing interests declaration", 531 "detail": "TurinTech AI is a commercial entity marketing Artemis; no competing interests or financial interests statement is present in the paper." 532 }, 533 { 534 "flag": "GA hyperparameters mostly unreported", 535 "detail": "MathTales reports population size 3 and 2 generations, but ALE (671.7 hours), Mini-SWE, and CrewAI optimization hyperparameters (population, crossover rates, mutation rates, generations) are not disclosed." 536 } 537 ], 538 "cited_papers": [ 539 { 540 "title": "Automated Design of Agentic Systems (ADAS)", 541 "relevance": "Direct competitor: code-based workflow optimization system compared against Artemis in Table 1 on scope, generality, and architectural agnosticism" 542 }, 543 { 544 "title": "AFlow: Automating Agentic Workflow Generation", 545 "relevance": "MCTS-based workflow optimization; key alternative approach situated as narrower (workflow-only, not architecture-agnostic) compared to Artemis" 546 }, 547 { 548 "title": "PromptBreeder: Self-referential self-improvement via prompt evolution", 549 "relevance": "Foundational evolutionary prompt optimization paper that Artemis extends from isolated prompts to full multi-component agent pipelines" 550 }, 551 { 552 "title": "MAST: Multi-Agent System Failure Taxonomy", 553 "relevance": "Motivating evidence: first empirical taxonomy of 14 failure modes in multi-agent LLM systems, justifying need for systematic optimization" 554 }, 555 { 556 "title": "Large Language Models are Human-Level Prompt Engineers (APE)", 557 "relevance": "Foundational LLM-driven prompt optimization work; established LLMs can improve their own prompts, foundational prior for Artemis" 558 }, 559 { 560 "title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning", 561 "relevance": "Contemporary Pareto-based evolutionary prompt optimization; closest algorithmic competitor not empirically compared against" 562 }, 563 { 564 "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", 565 "relevance": "Standard coding benchmark; paper uses Artemis's 57% SWE-bench resolution rate as a credibility signal for the platform" 566 }, 567 { 568 "title": "Code Generation with AlphaCodium: From Prompt Engineering to Flow Engineering", 569 "relevance": "Domain-specialized workflow optimization achieving 19%→44% on CodeContests; illustrates limits of domain-specific vs. general approaches" 570 } 571 ], 572 "engagement_factors": { 573 "practical_relevance": { 574 "score": 2, 575 "justification": "The evolutionary prompt optimization idea is immediately actionable for practitioners, even if Artemis itself is proprietary and not publicly available." 576 }, 577 "surprise_contrarian": { 578 "score": 1, 579 "justification": "The finding that already-well-tuned agents show diminishing optimization returns is mildly counterintuitive, but the overall narrative (optimization helps underspecified agents) is expected." 580 }, 581 "fear_safety": { 582 "score": 0, 583 "justification": "No AI safety or risk concerns raised; the paper is purely about performance optimization." 584 }, 585 "drama_conflict": { 586 "score": 1, 587 "justification": "Self-evaluation by the platform's developer creates implicit conflict-of-interest tension, but the paper does not frame the work as controversial." 588 }, 589 "demo_ability": { 590 "score": 1, 591 "justification": "Artemis exists as a commercial product at turintech.ai but is not openly accessible; readers cannot independently try it." 592 }, 593 "brand_recognition": { 594 "score": 1, 595 "justification": "TurinTech AI is a small UK startup with no broad recognition; the use of Claude 3.5 Sonnet adds mild Anthropic association." 596 } 597 }, 598 "hn_data": { 599 "threads": [ 600 { 601 "hn_id": "25471098", 602 "title": "Causality Is Graphically Simple", 603 "points": 90, 604 "comments": 7, 605 "url": "https://news.ycombinator.com/item?id=25471098", 606 "created_at": "2020-12-18T19:48:36Z" 607 }, 608 { 609 "hn_id": "45574705", 610 "title": "StreamingVLM: Real-Time Understanding for Infinite Video Streams", 611 "points": 33, 612 "comments": 0, 613 "url": "https://news.ycombinator.com/item?id=45574705", 614 "created_at": "2025-10-14T00:02:18Z" 615 }, 616 { 617 "hn_id": "45591789", 618 "title": "StreamingVLM: Real-Time Understanding for Infinite Video Streams", 619 "points": 1, 620 "comments": 0, 621 "url": "https://news.ycombinator.com/item?id=45591789", 622 "created_at": "2025-10-15T13:02:15Z" 623 }, 624 { 625 "hn_id": "42362464", 626 "title": "RoboHanger: Learning Generalizable Robotic Hanger Insertion for Diverse Garments", 627 "points": 1, 628 "comments": 0, 629 "url": "https://news.ycombinator.com/item?id=42362464", 630 "created_at": "2024-12-09T02:05:35Z" 631 } 632 ], 633 "top_points": 90, 634 "total_points": 125, 635 "total_comments": 7 636 } 637 }