ai-research-survey

scan.json (30038B)

---

 {
   "paper": {
     "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
     "authors": [
       "Chan Jun Shern",
       "Neil Chowdhury",
       "Oliver Jaffe",
       "James Aung",
       "Dane Sherburn",
       "Evan Mays",
       "Giulio Starace",
       "Kevin Liu",
       "Leon Maksin",
       "Tejal Patwardhan",
       "Lilian Weng",
       "Aleksander Mądry"
     ],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2410.07095",
     "doi": "10.48550/arXiv.2410.07095"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We open-source our benchmark code (github.com/openai/mle-bench/)' in the abstract and conclusion, providing a working URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses publicly available Kaggle competitions, and the Reproducibility Statement says 'code is provided to allow users to reproduce datasets in a way that complies with relevant licenses.' The leaderboard snapshots and curation details are also provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.5 specifies the exact setup: Ubuntu 20.04 Docker container, Microsoft Azure Standard_NV36ads_A10_v5 VM (36 vCPUs, 440GB RAM, Nvidia A10 GPU), sysbox runtime, Python virtual environment with necessary packages."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The Reproducibility Statement says 'We provide all necessary details for reproducing our results, including dataset curation, evaluation metrics, and experimental setup. Our codebase is publicly available, including code for reproducing the full benchmark and experiments.' They also provide examples for running agents."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 and other results tables report '± one standard error of the mean' for all metrics across seeds."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims o1-preview 'significantly outperforms all other models' (Section 3.1) but uses no formal significance tests — it compares means with standard errors without p-values, t-tests, or other statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute performance numbers are reported with context: o1-preview 16.9% vs GPT-4o 8.7% medals (Table 2), pass@1 16.9% doubling to pass@8 34.1% (Figure 3). Baseline and treatment numbers are both provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 75 competitions were selected (vs. more or fewer), no power analysis, and no justification for why 3 seeds per experiment (or 16/36 for some) is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors across seeds are reported in all main tables (Table 2, 3, 4, 9, 10). Table 2 notes 'Each experiment is repeated with 3 seeds, except o1-preview (AIDE) and GPT-4o (AIDE) which use 16 and 36 seeds respectively.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: three scaffolds (AIDE, MLAB, OpenHands) and four models (o1-preview, GPT-4o, Claude 3.5 Sonnet, Llama 3.1 405B). Human baselines are provided via Kaggle leaderboards."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All models are frontier LLMs from 2024: o1-preview, GPT-4o-2024-08-06, Claude 3.5 Sonnet, Llama 3.1 405B. All scaffolds are contemporary open-source frameworks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations: scaffolding choice (Table 2), hardware configuration (Table 3: CPU-only, standard, extra GPU), time available (Figure 4: 24h vs 100h), number of attempts (Figure 3: pass@k), and obfuscated descriptions (Table 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports Made Submission %, Valid Submission %, Above Median %, Bronze %, Silver %, Gold %, and Any Medal %. Per-competition raw scores are also discussed."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated through competition grading code. While the authors manually inspect agent logs for rule-breaking (Appendix A.3) and analyze failure modes (Section 3.1), there is no systematic human evaluation of the agents' ML engineering solutions or processes."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 states 'We include an additional 7 competitions as a development split, for developing agents without over-fitting to the test set.' The 75 main competitions form the test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 breaks down results by complexity level (Low/Medium/High). Table 10 breaks down by 19 task categories. Figure 9 shows performance by competition date."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 Discussion details failure modes: agents not using the validation server, ending runs early, filling context windows with large files, overloading disk/RAM, and not considering time limitations. Figure 2 shows real agent trajectories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows extra GPU has no meaningful benefit. MLAB achieves only 0.8% medal rate. Agents often fail to create valid submissions (Section 3.1). Figure 4 shows medals sometimes decreasing over time due to AIDE's imperfect selection method."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match results: 'best-performing setup — OpenAI's o1-preview with AIDE scaffolding — achieves at least the level of a Kaggle bronze medal in 16.9% of competitions' is directly supported by Table 2. Pass@8 doubling claim matches Figure 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made through controlled ablations: varying only the scaffold (Table 2, top vs bottom), varying only the model (Table 2, within AIDE), varying compute (Table 3), varying time (Figure 4). Each varies a single factor while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly bounds scope: 'the tasks included in MLE-bench don't cover the full spectrum of capabilities required for AI R&D,' noting Kaggle tasks have 'clear problem statements, datasets that are clean and well-documented' unlike real-world R&D."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 investigates contamination as an alternative explanation for performance, testing via familiarity analysis (Section 4.1) and obfuscated descriptions (Section 4.2). Section 6 discusses algorithmic progress giving agents advantages over original competitors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly distinguishes between what MLE-bench measures (performance on well-specified Kaggle competitions) and the broader claim (ML engineering capabilities for AI R&D), noting 'real-world AI R&D often may not even have a clear problem statement, and figuring out the dataset and metrics is part of the problem.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Footnotes and tables specify: gpt-4o-2024-08-06 (footnote 6), claude-3-5-sonnet-20240620 (footnote 8), meta-llama/llama-3.1-405b-instruct (footnote 9), gpt-4o-mini-2024-07-18 (footnote 14). o1-preview lacks a snapshot date but was a distinct single release at the time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 provides the full verbatim prompt used for all agents, including overall instructions, competition-specific details, submission validation, and rules. Appendix A.6 describes scaffold-specific prompt modifications."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 reports scaffold-level hyperparameters (max_steps, time_limit, etc.) for all three frameworks. However, LLM API inference parameters (temperature, top-p, sampling settings) are not stated for any model, and the schema requires these."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and Appendix A.6 describe all three scaffolds in detail: AIDE's tree search over solutions, MLAB's tool-based approach, OpenHands' CodeActAgent. All modifications are documented (A.6.1-A.6.3). Figure 2 shows real trajectories."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents curation from 5673 Kaggle competitions through screening criteria to 75 selected. Appendix A.1 lists all filtering criteria. Table 8 documents how each competition's train-test split was created."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' provides substantial discussion across four areas: contamination/plagiarism risks, coverage of AI R&D capabilities, differences to real competitions, and accessibility/cost."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses threats specific to this study: models may have trained on Kaggle materials (contamination), new train-test splits may not perfectly match original distributions, algorithmic progress gives agents advantages over historical competitors, and high compute cost limits reproducibility (1800 GPU hours per run)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states specific boundaries: 'the tasks included in MLE-bench don't cover the full spectrum of capabilities required for AI R&D,' listing that real R&D involves unclear problem statements, messy data, and choosing metrics — none of which MLE-bench tests."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying Kaggle competition data is publicly available, leaderboard snapshots are included, and benchmark code for reproducing datasets is open-sourced. Raw agent logs were used for analysis."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the full curation process: starting from 5673 Meta Kaggle competitions, excluding Community Competitions (leaving 586), manual screening for relevance, filtering by reproducibility criteria, arriving at 75 competitions."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The data source is publicly available Kaggle competitions, a standard well-known platform."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from 5673 competitions through screening criteria (Appendix A.1) to 75 is documented with the counts at each step. Table 8 documents each competition's split procedure. Grading code reimplementation is described in Section 2.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments section. All authors are from OpenAI, implying OpenAI funded the work, but this is not explicitly stated as a funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed under 'OpenAI' affiliation on the paper's first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI funds this research and has direct commercial interest in their models (o1-preview, GPT-4o) performing well on ML engineering tasks. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement in the paper. OpenAI employees are evaluating OpenAI models without explicit disclosure of the financial conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the models evaluated (o1-preview, GPT-4o, Claude 3.5 Sonnet, Llama 3.1 405B), despite investigating contamination effects through other means."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 extensively discusses potential overlap: Section 4.1 measures GPT-4o's familiarity with competition pages and winning solutions using token-level probability analysis. Section 6 notes 'GPT-4's base model could reproduce several rows from the dataset of the Titanic competition.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 comprehensively addresses contamination: familiarity analysis (Section 4.1), obfuscated descriptions experiment (Section 4.2), plagiarism detection with Dolos (Section 2.3.1), and log analysis tools (Appendix A.3). They find 'no systematic effect of contamination for GPT-4o.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The benchmark evaluates AI agents, not humans."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports 'o1-preview with AIDE used 127.5M input tokens and 15.0M output tokens on average for one seed of 75 competitions.' Total GPU hours (1800 per run) are also stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states 'A single run of our main experiment setup of 24 hours per competition attempt requires 24 hours × 75 competitions = 1800 GPU hours of compute.' Hardware is specified in Appendix A.5 (Azure VMs with A10 GPUs)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across multiple seeds with standard error: 3 seeds for most experiments, 16 for o1-preview (AIDE), and 36 for GPT-4o (AIDE). Standard errors in Table 2 show variation across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 caption states 'Each experiment is repeated with 3 seeds, except o1-preview (AIDE) and GPT-4o (AIDE) which use 16 and 36 seeds respectively.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Scaffold parameters in Table 7 (e.g., AIDE max 2000 steps, MLAB max 2000 steps) appear fixed without justification or search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports all model-scaffold combinations tried, not just the best. The selection of AIDE as the best scaffold for subsequent experiments is transparent and based on the full comparison results."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical significance tests are performed in the paper, so multiple comparison correction does not apply."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI employees created the benchmark and evaluated OpenAI models (o1-preview, GPT-4o) which achieved the highest scores. The paper does not acknowledge or discuss the bias of evaluating their own models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares performance across three hardware configurations (CPU-only, standard GPU, extra GPU). Figure 4 shows performance as a function of time (up to 100 hours). These directly relate compute to performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses construct validity: MLE-bench tests well-specified Kaggle tasks, which differ from real AI R&D. Section 5 compares to alternative benchmarks (MLAgentBench, DSBench, ML-Bench) and discusses how their design differs."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 separates scaffold and model effects: GPT-4o is tested across three scaffolds (AIDE, MLAB, OpenHands), and four models are compared within AIDE. This design isolates the scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 investigates whether familiarity with competitions correlates with performance (Figure 5). Section 6 notes 'algorithmic progress may result in older competitions being easier.' Figure 9 plots performance vs competition date."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation design prevents information leakage: the validation server checks submission validity but does not provide scores (Section 2.3). Rules forbid agents from viewing solutions online (Section 2.3.1). New train-test splits prevent direct answer leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes creating new train-test splits to ensure independence, checking that 'the distributions of the original and reconstructed test sets are similar.' Section 4 addresses whether model training data is independent of test data through contamination analysis."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Multiple detection methods: token-level familiarity analysis (Section 4.1), obfuscated description experiment (Section 4.2), Dolos plagiarism detection against top 50 Kaggle notebooks (Section 2.3.1), and GPT-4o-based log analysis for rule-breaking (Appendix A.3)."
       }
     }
   },
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MLE-bench evaluates AI agents on 75 Kaggle ML engineering competitions, finding that o1-preview with AIDE scaffolding achieves medals in 16.9% of competitions (pass@1), which doubles to 34.1% with 8 attempts. Scaffolding choice matters substantially (AIDE 8.7% vs MLAB 0.8% for GPT-4o), while hardware scaling shows minimal benefit — the extra GPU did not improve performance. Contamination analysis via familiarity testing and obfuscated descriptions finds no evidence of systematically inflated scores from model memorization.",
   "claims": [
     {
       "claim": "o1-preview with AIDE achieves at least the level of a Kaggle bronze medal in 16.9% of competitions",
       "evidence": "Table 2 shows 16.9% ± 1.1% Any Medal rate for o1-preview (AIDE) across 16 seeds, with breakdown: 3.4% Bronze, 4.1% Silver, 9.4% Gold.",
       "supported": "strong"
     },
     {
       "claim": "Performance roughly doubles from pass@1 to pass@8 for both o1-preview and GPT-4o",
       "evidence": "Figure 3 shows o1-preview goes from 16.9% (pass@1) to ~34.1% (pass@8), and GPT-4o from 8.7% to ~17% (pass@6 achieves comparable score to o1-preview pass@1).",
       "supported": "strong"
     },
     {
       "claim": "AIDE scaffolding significantly outperforms MLAB and OpenHands for GPT-4o",
       "evidence": "Table 2 shows GPT-4o (AIDE) 8.7% vs MLAB 0.8% vs OpenHands 4.4% medal rates. However, no formal significance test is used to support the word 'significantly.'",
       "supported": "moderate"
     },
     {
       "claim": "Hardware scaling (CPU-only vs GPU vs 2 GPUs) has minimal effect on agent performance",
       "evidence": "Table 3 shows CPU-only 9.1%, Standard 8.7%, Extra GPU 10.2% — all within standard error. However, only 3 seeds for non-standard configs.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o's familiarity with competitions does not correlate with performance, suggesting contamination does not systematically inflate scores",
       "evidence": "Figure 5 shows Pearson's correlation -0.24 (p=0.04) between familiarity and performance — actually slightly negative. Table 4 shows obfuscated descriptions yield equivalent performance (8.4% vs 8.5%).",
       "supported": "moderate"
     },
     {
       "claim": "Agents improve with more time, achieving more medals at 100 hours than 24 hours",
       "evidence": "Figure 4 shows GPT-4o (AIDE) reaches ~8.7% at 24h and ~11.8% at 100h. However, results are from a single run configuration.",
       "supported": "moderate"
     },
     {
       "claim": "Agents can score well on competitions solvable with well-known approaches but struggle to debug issues and recover from missteps",
       "evidence": "Section 3.1 Discussion describes failure modes: not using validation server, ending runs early, overloading resources, not considering time constraints. Table 10 shows 0% medals on harder categories like Image Segmentation, Object Detection.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All authors are from OpenAI and evaluate OpenAI models (o1-preview, GPT-4o) which achieve the top results. No explicit acknowledgment of this conflict of interest or self-comparison bias. While third-party models are included, the benchmark was designed by the team whose model leads."
     },
     {
       "flag": "No formal significance tests despite comparative claims",
       "detail": "The paper claims o1-preview 'significantly outperforms all other models' (Section 3.1) based solely on comparing means with standard errors, without performing any formal statistical test. The word 'significantly' is used in a statistical context without statistical backing."
     },
     {
       "flag": "Missing LLM inference hyperparameters",
       "detail": "Temperature, top-p, and other LLM API sampling parameters are not reported for any model, despite these settings meaningfully affecting output quality and consistency."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational LLM code evaluation benchmark (HumanEval/Codex) and pass@k metric used in MLE-bench."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Early coding challenge benchmark showing LLM capabilities on competitive programming problems."
     },
     {
       "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",
       "authors": ["Qian Huang", "Jian Vora", "Percy Liang", "Jure Leskovec"],
       "year": 2024,
       "relevance": "Closest prior benchmark for ML agent evaluation using 13 Kaggle and bespoke tasks; MLE-bench extends this with 75 competitions."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Leading software engineering benchmark for AI agents; MLE-bench positions as complementary for ML engineering specifically."
     },
     {
       "title": "AI Agents That Matter",
       "authors": ["Sayash Kapoor", "Benedikt Stroebl", "Zachary S. Siegel"],
       "year": 2024,
       "arxiv_id": "2407.01502",
       "relevance": "Analysis of agent evaluation methodology including cost-performance tradeoffs and evaluation pitfalls."
     },
     {
       "title": "OpenDevin: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Open-source agent platform (OpenHands) used as one of three scaffolds evaluated in MLE-bench."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Multi-turn agent evaluation benchmark with diverse environments; foundational work for AI agent evaluation."
     },
     {
       "title": "GAIA: a benchmark for General AI Assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Craig Swift"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "Benchmark for general-purpose AI agents on real-world tasks requiring tool use and reasoning."
     },
     {
       "title": "ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code",
       "authors": ["Xiangru Tang", "Yuliang Liu", "Zefan Cai"],
       "year": 2024,
       "arxiv_id": "2311.09835",
       "relevance": "Evaluates agents on understanding and using existing ML codebases; complementary to MLE-bench's open-ended task formulation."
     },
     {
       "title": "DSBench: How Far Are Data Science Agents to Becoming Data Science Experts?",
       "authors": ["Liqiang Jing", "Zhehui Huang", "Xiaoyang Wang"],
       "year": 2024,
       "arxiv_id": "2409.07703",
       "relevance": "Concurrent Kaggle-based benchmark focused on data science tasks; MLE-bench argues their competition selection is more diverse and challenging."
     },
     {
       "title": "ConStat: Performance-Based Contamination Detection in Large Language Models",
       "authors": ["Jasper Dekoninck", "Mark Niklas Müller", "Martin Vechev"],
       "year": 2024,
       "arxiv_id": "2405.16281",
       "relevance": "Defines contamination framework used to motivate MLE-bench's contamination analysis methodology."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code evaluation benchmark addressing temporal leakage concerns relevant to MLE-bench."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Demonstrated LLM capabilities at competition-level code generation; early milestone for autonomous coding agents."
     }
   ]
 }