ai-research-survey

scan.json (28790B)

---

 {
   "paper": {
     "title": "Let the Barbarians In: How AI Can Accelerate Systems Performance Research",
     "authors": [
       "Audrey Cheng",
       "Shu Liu",
       "Melissa Pan",
       "Zhifei Li",
       "Shubham Agarwal",
       "Mert Cemri",
       "Bowen Wang",
       "Alexander Krentsel",
       "Tian Xia",
       "Jongseok Park",
       "Shuo Yang",
       "Jeff Chen",
       "Lakshya Agrawal",
       "Ashwin Naren",
       "Shulu Li",
       "Ruiying Ma",
       "Aditya Desai",
       "Jiarong Xing",
       "Koushik Sen",
       "Matei Zaharia",
       "Ion Stoica"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.14806",
     "doi": "10.48550/arXiv.2512.14806"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Three open-source ADRS frameworks (OpenEvolve, GEPA, ShinkaEvolve) can generate algorithms that match or exceed human state-of-the-art for systems performance problems across ten case studies, including 13x faster MoE load balancing and 35% greater spot instance savings. The paper identifies best practices along three axes (specification, evaluation, feedback) and finds that LLMs can discover cross-domain techniques (e.g., Hamilton's Apportionment from political science for GPU load balancing). OpenEvolve showed the most consistent performance across models, while GEPA and ShinkaEvolve showed stronger model-specific preferences.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper references open-source ADRS frameworks (OpenEvolve, GEPA, ShinkaEvolve) with GitHub links, but does not release the code for its own case studies, simulators, evaluation setups, or configuration files beyond what is shown in the appendix."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses simulators and workload traces from prior work but does not release a unified dataset or artifact package. No download link for the evaluation data is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper mentions using specific models (GPT-5, Gemini-3.0-Pro) and frameworks but does not specify library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Configuration files are shown in Appendix D but without instructions on how to run the experiments end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports mean ± standard deviation but no confidence intervals or error bars. Standard deviation is reported but CIs are not."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims frameworks outperform SOTA baselines but uses no statistical significance tests. Comparisons are based solely on mean values from three runs."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: '13× faster', '35% greater savings', '6% higher average cost savings', '17% cost savings', '3× runtime speedup', '60% better than greedy' — all with baseline reference points."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each experiment is repeated three times with no justification for why three runs is sufficient. No power analysis or discussion of whether three runs provides adequate statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports mean ± standard deviation across three runs for each framework-model combination on all tasks."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each case study includes baselines: SOTA human-designed solutions and initial/greedy programs. Table 2 lists SOTA baselines for all ten tasks. Table 3 includes Human SOTA row."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are from recent, top-venue publications: NSDI '24 (CBL), VLDB '24 (TXN), MLSys '25 (LLM-SQL), and state-of-the-art implementations from DeepSeek."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies are conducted: seed diversity (Section 5.1, LLM-SQL), feedback granularity (Section 5.3, CBL with minimal/moderate/detailed), training set coverage (Appendix C.1), and hint refinement effects."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: performance score, runtime, cost, LOC. For EPLB: balance factor and rearrangement runtime. For LLM-SQL: prefix hit rate and algorithm runtime. Combined scores are also defined."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated solutions' quality is reported. The authors mention they 'readily understood the generated solutions' (Section 2) but no systematic human evaluation was conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For CBL, 30% of traces are sampled for feedback during search, with final results reported on the full evaluation set (Section 4.1). Appendix C.1 discusses training/test split effects."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-task breakdowns across all ten tasks for all framework-model combinations. Per-workload results are shown in figures (e.g., Figure 8 for CBL)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed extensively: Cloudcast shows no improvement (Table 2), reward hacking examples (MAS, EPLB, Prism in Section 5.2), and Appendix C.3 categorizes failure patterns from 420 LLM-judged traces."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cloudcast failed to improve over baseline. TXN online setting couldn't surpass SMF. MAS v3 degraded performance when verification was removed. Section 6.1 discusses problems ADRS is not suited for."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '13× faster load balancing' and '35% greater savings' are supported by Table 2 and Table 3 results. The claim that ADRS-generated solutions 'can match or even outperform human state-of-the-art' is supported across multiple case studies."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are mostly about ablation effects (e.g., 'providing diverse seeds improves scores', 'moderate feedback outperforms minimal and detailed'). These are supported by controlled single-variable ablation experiments described in Section 5."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly discusses which problems are best suited (isolated changes, reliable evaluations, efficient evaluations) and which are not. The paper states 'we do not yet have a universal recipe' in the abstract and limits claims to systems performance problems."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses training data contamination as an alternative explanation for TXN online results (Section 4.5: 'likely due to training data contamination from the SMF paper'). Section 6 discusses limitations and alternative framings."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific system metrics (cost savings, runtime, load balance factor, makespan) and claims improvements in those specific metrics rather than making broader proxy claims. The metrics directly measure what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-5' and 'Gemini-3.0-Pro-Preview' and 'Gemini-3.0' without snapshot dates or API versions. The ShinkaEvolve config lists 'gpt-4.1-mini', 'gpt-4.1-nano', 'o4-mini' without versions. No model version identifiers beyond marketing names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language (three-part template: problem, evaluation criteria, context) but does not provide the actual prompt text used in experiments. Section 5.1 discusses prompt design principles without showing prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Configuration files are provided in Appendix D for all three frameworks, including temperature (0.7), top_p (0.95), max_tokens (32000), population sizes, migration rates, island counts, and iteration limits."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The ADRS architecture is described in detail in Section 3.2 and Figure 1b, with five components (Prompt Generator, Solution Generator, Evaluator, Storage, Solution Selector). Table 1 compares framework designs. Each framework's parent selection, evolution context, and evaluation mechanisms are documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "For each case study, the evaluator setup, trace sampling, and data pipeline are described. For CBL: '30% of the traces as a feedback subset' (Section 4.1). Simulator configurations are specified per case study."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Open Challenges' is a dedicated section discussing which problems ADRS is suited for (Section 6.1) and open challenges (Section 6.2)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 identifies specific limitations: ADRS struggles with problems requiring coordinated multi-module changes, problems where semantic equivalence is undecidable, and problems requiring expensive evaluations. Training data contamination is discussed for TXN."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly states what ADRS does NOT handle well: multi-module changes, expensive evaluations, and undecidable correctness verification. The abstract notes 'we do not yet have a universal recipe for applying ADRS across all of systems research.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (evolution logs, per-iteration scores, generated programs) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Each case study describes the data source: existing simulators from published papers (CBL from NSDI '24, TXN from VLDB '24), public benchmarks (ShareGPT, GSM8K), and specific trace configurations."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The pilot survey (Appendix A, Figure 7) of 31 PhD students does not describe how participants were recruited, what their specific research areas are, or whether recruitment could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: each framework runs 100 iterations, solutions are evaluated against predefined workloads, scores are collected with mean ± std over 3 runs. The flow from generation to evaluation is described in Section 3.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with UC Berkeley. The affiliation is clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making independence impossible to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Several authors may have connections to companies whose products are referenced (e.g., Ion Stoica co-founded Databricks and Anyscale), but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-5 and Gemini-3.0-Pro but does not state training data cutoff dates for these models. This is relevant because the paper's own case study baselines (NSDI '24, VLDB '24) could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges contamination risk for TXN: 'OpenEvolve rediscovered this algorithm from a random baseline, likely due to training data contamination from the SMF paper' (Section 4.5). However, this is only discussed for one case study."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Beyond the TXN contamination note, the paper does not systematically address whether the LLMs' training data includes the baseline algorithms, simulator code, or published solutions being compared against. The fact that LLMs may have memorized SOTA solutions could confound results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper's main contribution is benchmark evaluation of ADRS frameworks, not a human subjects study. The pilot survey in Appendix A is minor."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study; the pilot survey of PhD students is a minor supplementary element."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study in the main contribution."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study in the main contribution."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports cost per case study: most under $15-$30. Time is also reported (1-8 hours for 100 iterations). Section 4 states 'most of our case studies required only a few hours and cost less than several tens of dollars.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports time and cost for each case study. Iteration counts (100) are stated. Total API spend is bounded (e.g., '≤$15', '≤$30'). Number of LLM calls per iteration is stated per framework in Table 1."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports mean ± standard deviation across three runs for each framework-model combination, showing result variation across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'We repeat each experiment three times.' Table 3 reports results over three runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses default configurations for each framework (Section 4: 'using their default configurations') but does not report any hyperparameter search budget for tuning these defaults or the evaluator scoring functions."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results for all three frameworks with both models (Table 3), not just the best configuration. Best framework per task is identified but all results are shown."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares three frameworks × two models × ten tasks with no statistical tests at all, let alone multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors design the case studies, evaluators, and scoring functions, then evaluate their own ADRS approach. This author-evaluation bias is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "All experiments are capped at 100 iterations, but no performance-vs-compute curves are shown. The paper does not report how performance scales with iteration budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 discusses why systems performance problems are well-suited for ADRS (verifiable solutions, preserving correctness, small algorithmic code, simulator-based evaluation). Section 6.1 discusses which problems are and aren't well-suited."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates three different frameworks (scaffolds) and two different models, separating the framework effect from the model effect. Table 3 shows results by framework × model. Table 1 describes design differences between frameworks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether GPT-5 or Gemini-3.0 were trained on the baseline algorithms' source code or published papers. The TXN contamination note is an exception but is not systematic."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information that would not be available in a real research scenario (e.g., the ADRS prompts may implicitly encode knowledge of the SOTA solution)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs' training data includes the exact simulators, baseline implementations, or published papers being used in the evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. The TXN contamination is noted post-hoc as a suspicion but no detection method was applied."
       }
     }
   },
   "claims": [
     {
       "claim": "ADRS-generated solutions can match or outperform human state-of-the-art designs across ten systems performance case studies.",
       "evidence": "Table 2 shows improvements over SOTA in 8 of 10 tasks. Table 3 provides quantitative results with mean ± std over 3 runs. Specific improvements: 13× faster EPLB, 35% better CBL savings, 60% better offline TXN.",
       "supported": "moderate"
     },
     {
       "claim": "OpenEvolve achieves the highest success rate across frameworks, delivering the best solution in 9 out of 20 total cases.",
       "evidence": "Table 3 aggregate results across all tasks. Section 4 states framework comparison: OpenEvolve 9, ShinkaEvolve 8, GEPA 6 top results.",
       "supported": "moderate"
     },
     {
       "claim": "ADRS frameworks can discover cross-domain techniques that human experts might overlook.",
       "evidence": "Table 4 shows five examples: Hamilton's Apportionment for EPLB, Borda Count for TXN, Kirchhoff's Current Law for telemetry, UCB for CBL-Multi, gradient-based power control for NS3.",
       "supported": "moderate"
     },
     {
       "claim": "Moderate feedback granularity outperforms both minimal and detailed feedback for ADRS.",
       "evidence": "Section 5.3 ablation on CBL: moderate feedback achieves 13.0% cost reduction vs minimal (7.7%) and detailed (10.2%).",
       "supported": "weak"
     },
     {
       "claim": "Diverse seed programs improve ADRS evolution outcomes compared to uniform seeding.",
       "evidence": "Section 5.1 ablation on LLM-SQL: uniform seeding caps at 0.74, diverse seeds reach 0.7755. Only diverse-seed runs exceeded 0.74.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Only 3 runs per experiment",
       "detail": "Each experiment is repeated only three times, which is insufficient for reliable statistical inference. Standard deviations in Table 3 are sometimes very large relative to differences between methods (e.g., NS3 ShinkaEvolve GPT-5: 89.50 ± 18.73), suggesting high variance that 3 runs cannot adequately characterize."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Despite claiming one framework or model outperforms another, no significance tests are reported. Given the high variance in some tasks, many claimed differences may not be statistically significant."
     },
     {
       "flag": "Contamination risk largely unaddressed",
       "detail": "The LLMs used (GPT-5, Gemini-3.0) were almost certainly trained on the published SOTA algorithms being compared against. The paper acknowledges this for TXN ('likely due to training data contamination') but does not systematically address it for other case studies. The cross-domain 'discoveries' could be memorized techniques."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors designed both the evaluation frameworks and the scoring functions, then reported results of their own approach. No independent evaluation was conducted. The evaluator design choices (scoring weights, test set composition) could favor ADRS outcomes."
     },
     {
       "flag": "Selective case study presentation",
       "detail": "Cloudcast, where ADRS failed to improve over baseline, receives much less attention than successful cases. The paper presents 5 detailed case studies in the main text, all showing positive results, with the failure case relegated to the appendix."
     },
     {
       "flag": "Draft comments left in paper",
       "detail": "Internal comments ('/* accheng: @Ion, do we want to keep this still? */', '/* shu: do we still need this table? */') are left in the paper, suggesting incomplete revision and potentially premature conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "OpenEvolve: an open-source evolutionary coding agent",
       "authors": ["Asankhaya Sharma"],
       "year": 2025,
       "relevance": "Open-source implementation of AlphaEvolve's evolutionary LLM-based code generation approach, central to this paper's evaluation."
     },
     {
       "title": "GEPA: Reflective prompt evolution can outperform reinforcement learning",
       "authors": ["Lakshya A Agrawal", "Shangyin Tan"],
       "year": 2025,
       "arxiv_id": "2507.19457",
       "relevance": "One of three ADRS frameworks evaluated; uses natural-language reflection to mutate prompts with Pareto filtering."
     },
     {
       "title": "ShinkaEvolve: Towards open-ended and sample-efficient program evolution",
       "authors": ["Robert Tjarko Lange", "Yuki Imajuku", "Edoardo Cetin"],
       "year": 2025,
       "arxiv_id": "2509.19349",
       "relevance": "Third ADRS framework evaluated; emphasizes structured introspection and correctness-gated evolution."
     },
     {
       "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery",
       "authors": ["Alexander Novikov"],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "Proprietary Google DeepMind ADRS framework using MAP-Elites and island models for LLM-based algorithm discovery."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian"],
       "year": 2024,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent LLM system for software development, used as baseline in the MAS case study."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework evaluated in the MAS optimization case study."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Proposes MAST taxonomy of multi-agent system failure modes, used in the MAS case study evaluation."
     },
     {
       "title": "Glia: A human-inspired AI for automated systems design and optimization",
       "authors": ["Pouya Hamadanian"],
       "year": 2025,
       "arxiv_id": "2510.27176",
       "relevance": "Agentic workflow for systems design with strong results on distributed LLM inference."
     },
     {
       "title": "Darwin Godel Machine: Open-ended evolution of self-improving agents",
       "authors": ["Jenny Zhang", "Shengran Hu", "Cong Lu", "Robert Lange", "Jeff Clune"],
       "year": 2025,
       "arxiv_id": "2505.22954",
       "relevance": "Demonstrates open-ended self-referential code improvement and self-evolving AI agents."
     },
     {
       "title": "MLGym: A new framework and benchmark for advancing AI research agents",
       "authors": ["Deepak Nathani"],
       "year": 2025,
       "arxiv_id": "2502.14499",
       "relevance": "Benchmark for AI research agents, related to automated research evaluation."
     },
     {
       "title": "DAPO: An open-source LLM reinforcement learning system at scale",
       "authors": ["Qiying Yu"],
       "year": 2025,
       "arxiv_id": "2503.14476",
       "relevance": "Open-source LLM RL system; paper's finding that excessive length signals unreliability is cited."
     },
     {
       "title": "AlgoTune: Can language models speed up general-purpose numerical programs?",
       "authors": ["Ori Press"],
       "year": 2025,
       "arxiv_id": "2507.15887",
       "relevance": "Benchmark evaluating LLM program optimization abilities, directly related to AI-driven code generation."
     }
   ]
 }