ai-research-survey

scan.json (31066B)

---

 {
   "paper": {
     "title": "ResearchCodeBench: Benchmarking LLMs on Implementing Novel Machine Learning Research Code",
     "authors": [
       "Tianyu Hua",
       "Harper Hua",
       "Violet Xiang",
       "Benjamin Klieger",
       "Sang T. Truong",
       "Weixin Liang",
       "Fan-Yun Sun",
       "Nick Haber"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.02314",
     "doi": "10.48550/arXiv.2506.02314"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ResearchCodeBench evaluates 32 LLMs on 212 coding challenges from 20 recent ML papers, finding that even the best model (Gemini-2.5-Pro-Preview at 37.3% Scaled Pass@1) implements less than 40% of tasks correctly. Reasoning-oriented models consistently outperform standard models, and higher-performing models benefit substantially from paper context while LLaMA-based models actually perform worse with paper access. The dominant error type is functional/semantic errors (59%), indicating models struggle with algorithmic reasoning rather than syntax.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data are available: https://researchcodebench.github.io/' in the abstract (footnote 2), providing a URL for the benchmark framework and data."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Same URL provides access to the benchmark data (212 coding challenges, test cases, paper metadata). The abstract footnote says 'Code and data are available.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 states 'the full benchmark executes locally in a single Docker or Conda environment' and specifies it runs on 'a standard personal laptop such as an M1 MacBook Pro.' The released code framework includes environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the full benchmark execution pipeline (Section 2.2), provides exact prompt templates (Appendix E), and releases an 'open-source benchmarking framework that is easy to use' with code and data at the project URL."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Figures 2, 3, 6) report only point estimates (e.g., '37.3% success rate'). No confidence intervals, error bars, or uncertainty quantification for any model's performance."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Gemini-2.5-Pro-Preview to perform best') based solely on comparing raw pass rate numbers. No statistical significance tests are used to validate any model comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute scores with context for comparison (e.g., '37.3% success rate' vs '32.3%' and '30.8%'), and relative improvements (e.g., 'relative improvements of up to 30% over their original scores without the paper' in Section 3.3). This provides sufficient magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 20 papers and 212 tasks were chosen. No power analysis or discussion of whether this sample size is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All models are evaluated using greedy decoding with a single run. No variance, standard deviation, or spread measures are reported across any experimental conditions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 32 models from 10 different companies (Figure 2), including both open-source and proprietary models, providing extensive cross-model comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated are highly contemporary (Gemini-2.5-Pro-Preview-05-06, GPT-4.1, O3, O4-Mini, Claude-3.7-Sonnet, DeepSeek-R1), representing the state of the art at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 presents a paper reliance ablation comparing model performance with and without paper context. Section 3.2 provides a contamination-safe subset analysis isolating the effect of potential data exposure."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are reported: Scaled Pass@1 (primary metric, weighted by executable lines of code) in Figure 2, and standard (vanilla) Pass@1 in Figure 6 (Appendix F)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using equivalence tests and unit tests. No human evaluation of model outputs is performed. The paper notes human baseline is absent due to cost constraints (Section 5)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark serves as a held-out test set — models are not tuned on any portion of it. Models are evaluated with greedy decoding on the full set of 212 tasks with no development or tuning split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 (right panel) provides per-paper performance breakdowns. Results are broken down by contamination-safe vs full benchmark, and by with/without paper context (Figure 4). Per-model error breakdowns are provided (Appendix G)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 provides detailed error analysis with a taxonomy of 7 error categories (functional, name, type, syntax, import, attribute, index/key errors), with per-model breakdowns in Appendix G."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that LLaMA-based models perform worse when given the paper (Section 3.3, 'suggesting that long academic documents may introduce context dilution or confusion'), and that smaller models show no benefit from paper context."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (Gemini-2.5-Pro at 37.3%, O3 at 32.3%, O4-mini at 30.8%, best models below 40%) are directly supported by Figure 2's results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims about paper context improving performance are supported by a controlled ablation (Section 3.3) comparing identical models with and without paper access, which is an adequate design for this claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper restricts claims to ML papers specifically, acknowledges 'the current benchmark includes only 20 papers and focuses exclusively on machine learning' (Section 5), and explicitly discusses plans for broader coverage as a future direction."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1-3.2 discusses contamination as an alternative explanation for high scores. Section 3.3 explores paper reliance vs pure code completion. The paper also considers 'context dilution or confusion' for LLaMA's degraded performance with paper access."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Scaled Pass@1 on fill-in-the-blank code completion with hints and surrounding context, but claims to evaluate 'ability to translate cutting-edge ML contributions from recent research papers into executable code.' The gap between fill-in-the-blank snippet completion (with visible surrounding code and hints) and the actual task of implementing novel research from scratch is not acknowledged or discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Many model names are marketing names without specific API versions or snapshot dates: 'GPT-4.1', 'Claude-3.7-Sonnet', 'O3 (High)', 'DeepSeek-R1', 'Grok-3-Beta'. While some include dates (e.g., 'Gemini-2.5-Pro-Preview-05-06', 'DeepSeek-Chat-V3-0324'), most lack the snapshot-level specificity required for reproducibility."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.1 provides the exact prompt templates used for both with-paper and without-paper settings, including the complete system instructions and formatting guidelines given to models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states 'All models are evaluated using greedy decoding' which specifies the key inference parameter (temperature=0, deterministic generation). This is consistently applied across all 32 models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive a single prompt with paper context and code, and generate a direct completion. There is no tool use, retry logic, or multi-step workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents the full pipeline: paper selection criteria (top venues, well-documented contributions, author collaboration), LaTeX processing via latexpand, PDF-to-Markdown conversion via OCR, snippet annotation with XML tags, and contextual dependency identification via import analysis."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('Discussion') includes a dedicated 'Limitations and future directions' subsection discussing specific constraints of the benchmark."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: limited scope to 20 ML papers only, manually written test cases limiting scalability, no human baseline making it impossible to contextualize LLM performance. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states 'the current benchmark includes only 20 papers and focuses exclusively on machine learning' and notes 'we do not include a human baseline,' clearly delineating what the results do and do not show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark code, data, coding challenges, and test cases are released at https://researchcodebench.github.io/, allowing independent verification and re-evaluation of results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the data collection procedure in detail: paper selection from top venues, identification of core contributions, contextual dependency analysis, snippet annotation with XML tags, and test case co-development with paper authors or domain experts."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes paper selection: 'We begin by selecting 20 recent ML papers from top-tier conferences... We prioritize papers whose core contributions are both well-documented in the paper and cleanly implemented in their open-source repositories. Additionally, we consider whether the authors may be interested in collaborating.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents each pipeline stage: paper selection → core contribution identification → contextual code extraction → snippet annotation → task construction → test creation. The pipeline from paper to benchmark task is fully described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states 'This work was supported in part by the National Science Foundation under Grant No. 2302701, and by the Stanford HAI Hoffman-Yee Research Grant.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Stanford University. The paper does not evaluate any Stanford-affiliated AI product, so there is no undisclosed conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from NSF and Stanford HAI, neither of which has a financial interest in which LLM performs best on the benchmark. The funders are independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. There is no declaration of financial interests, patents, or equity related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses model knowledge cutoff dates extensively, with Figure 3 (left panel) showing 'the most recent known cutoff date for each company's models' as horizontal colored lines ranging from August 2023 to January 2025."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1-3.2 extensively analyzes potential train/test overlap by comparing model cutoff dates with repository commit dates. They identify that '13 out of 20 have their first commits in 2025, after the most recent Gemini-2.5-Pro-Preview's knowledge cutoff date.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 creates a 'contamination-safe subset of 13 papers whose initial commits occurred after the latest known model cutoff (January 2025)' and reports separate results for this subset, demonstrating concrete contamination mitigation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates LLMs on automated coding tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or evaluators requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports evaluation time ('each task takes just 1.25 seconds to evaluate') but does not report API inference costs, total tokens consumed, or per-model inference costs for running 32 models across 212 tasks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, total inference time across all models, or hardware requirements for running the full evaluation suite of 32 models."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All models are evaluated with a single greedy decoding pass. No analysis of sensitivity to random seeds, sampling temperature, or other sources of variance."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All models are evaluated using greedy decoding' (Section 3) and reports 'scaled pass@1... computed over the first (i.e., top-1) model completion' (Section 2.2), clearly indicating a single deterministic run per model."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses greedy decoding without discussing whether other decoding strategies (temperature sampling, nucleus sampling, multiple completions) were explored. No hyperparameter search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a single fixed configuration (greedy decoding, pass@1) for all models, explicitly described in Section 2.2 and Section 3. There is no selection from multiple configurations, eliminating cherry-picking concerns."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 32 models and makes ranking claims without any statistical tests or corrections for multiple comparisons. All comparisons are based on raw pass rate differences."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate all models on it without acknowledging potential bias in benchmark design (e.g., whether task construction or test design might favor certain model families). The concern from Lucic et al. about author-evaluation bias is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Reasoning models like O3 (High) and O4-Mini (High) likely consume significantly more inference-time compute than standard models like GPT-4o, but performance is not compared at matched compute budgets. The compute disparity is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether fill-in-the-blank code completion with hints and surrounding context actually measures the claimed capability of 'translating novel ML contributions into executable code.' The gap between the task design and the broader research implementation skill is not analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. All models receive the same direct prompt and generate a single completion. There is no scaffolding confound to address."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.2 extensively analyze temporal leakage by comparing model training cutoff dates with repository first-commit dates, and creating a contamination-safe subset of 13 papers with commits after all model cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides hints and surrounding code context to models but does not analyze whether these features leak information about the expected solution. The interplay between visible context code and the masked snippet is not discussed as a potential leakage source."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 212 tasks come from only 20 papers, with multiple tasks per paper. Tasks from the same paper likely share structural similarities and dependencies, but this non-independence is not discussed or addressed in the evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a concrete temporal-based detection method: comparing model knowledge cutoff dates with repository first commit dates (Figure 3 left panel), and creating a contamination-safe subset for comparison (Section 3.2)."
       }
     }
   },
   "claims": [
     {
       "claim": "Even the best LLMs correctly implement less than 40% of the benchmark tasks (Gemini-2.5-Pro-Preview at 37.3% Scaled Pass@1).",
       "evidence": "Figure 2 shows Scaled Pass@1 results for 32 models, with Gemini-2.5-Pro-Preview-05-06 achieving the highest score of 37.3%.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning-oriented models (Gemini-2.5-Pro, O3, O4-mini) consistently outperform standard models, even on the contamination-safe subset.",
       "evidence": "Figure 2 shows reasoning models ranked highest. Figure 3 (right panel) shows the advantage persists on the contamination-safe 13-paper subset.",
       "supported": "strong"
     },
     {
       "claim": "All models show a noticeable performance drop on the contamination-safe subset, suggesting most tasks are genuinely novel.",
       "evidence": "Figure 3 (right panel) shows consistent performance drops across all 32 models when evaluated on the 13-paper contamination-safe subset vs. the full 20-paper benchmark.",
       "supported": "strong"
     },
     {
       "claim": "Higher-performing models benefit substantially from paper context, with relative improvements up to 30%.",
       "evidence": "Figure 4 (left panel) shows per-model differences in Scaled Pass@1 with vs. without paper context. Top models like O4-Mini and Gemini-2.5-Pro show the largest positive deltas.",
       "supported": "moderate"
     },
     {
       "claim": "LLaMA-based models perform worse when given the research paper, suggesting long academic documents introduce context dilution.",
       "evidence": "Figure 4 (left panel) shows all LLaMA models (Llama-4-Maverick, Llama-4-Scout, Llama-3.3-70B) have negative deltas, performing worse with paper context.",
       "supported": "moderate"
     },
     {
       "claim": "Functional/semantic errors dominate at 58.6% of all failures, indicating the primary challenge is algorithmic reasoning rather than syntax.",
       "evidence": "Figure 5 shows the error distribution. Functional errors account for 59%, followed by name errors (9%), type errors (9%), and syntax errors (8%).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All 32 models are compared using single-run point estimates with greedy decoding. No confidence intervals, error bars, or significance tests support the model rankings. Small differences between closely-ranked models cannot be distinguished from noise."
     },
     {
       "flag": "Error classification using LLM",
       "detail": "The paper uses GPT-4o-Mini to classify errors (Section 3.4) while arguing against LLM-based evaluation in Section 2.1 ('LLM judges for code are often evaluated using tests themselves'). The reliability of this LLM-based error classification is not validated."
     },
     {
       "flag": "Selection bias in paper choice",
       "detail": "Papers were selected partly based on 'whether the authors may be interested in collaborating on the benchmark' (Section 2.1). This introduces selection bias: papers whose authors collaborate may have cleaner, better-documented code, potentially making tasks easier or harder in non-representative ways."
     },
     {
       "flag": "Small and narrow benchmark",
       "detail": "Only 20 ML papers and 212 tasks. The paper draws broad conclusions about LLM research code capability from a very limited sample that covers only machine learning, with no diversity in scientific domain."
     },
     {
       "flag": "No human baseline",
       "detail": "Without a human baseline, it is impossible to assess whether 37.3% represents a meaningful gap from human capability or whether the tasks are inherently difficult even for expert humans. The paper acknowledges this limitation but draws conclusions about LLM capability regardless."
     },
     {
       "flag": "Non-independence of tasks",
       "detail": "212 tasks drawn from only 20 papers means roughly 10 tasks per paper on average. Tasks from the same paper share code context, difficulty characteristics, and domain. This clustering is not accounted for in the evaluation, potentially inflating or deflating performance estimates."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Major code generation benchmark evaluating LLMs on resolving real-world GitHub issues, directly comparable benchmark in the LLM coding evaluation space."
     },
     {
       "title": "MLE-bench: Evaluating machine learning agents on machine learning engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark evaluating LLM agents on ML engineering tasks, closely related benchmark for assessing AI capability in research-adjacent coding."
     },
     {
       "title": "PaperBench: Evaluating AI's ability to replicate AI research",
       "authors": ["Giulio Starace", "Oliver Jaffe", "Dane Sherburn"],
       "year": 2025,
       "relevance": "Benchmark for AI replication of ML research papers end-to-end, the most directly comparable benchmark to ResearchCodeBench but using LLM-based judging rather than execution-based tests."
     },
     {
       "title": "SciCode: A research coding benchmark curated by scientists",
       "authors": ["Minyang Tian", "Luyu Gao", "Shizhuo Dylan Zhang"],
       "year": 2024,
       "relevance": "Scientific coding benchmark emphasizing domain-specific research code generation, directly relevant to evaluating LLM capability in research contexts."
     },
     {
       "title": "SUPER: Evaluating agents on setting up and executing tasks from research repositories",
       "authors": ["Ben Bogin", "Kejuan Yang", "Shashank Gupta"],
       "year": 2024,
       "arxiv_id": "2409.07440",
       "relevance": "Benchmark evaluating LLM agents on setting up and executing research code from repositories, addressing computational reproducibility of research."
     },
     {
       "title": "MLAgentBench: Evaluating language agents on machine learning experimentation",
       "authors": ["Qian Huang", "Jian Vora", "Percy Liang", "Jure Leskovec"],
       "year": 2024,
       "relevance": "Benchmark for LLM agents performing ML experimentation tasks, relevant to understanding agentic AI capability in research settings."
     },
     {
       "title": "CORE-Bench: Fostering the credibility of published research through a computational reproducibility agent benchmark",
       "authors": ["Zachary S Siegel", "Sayash Kapoor", "Nitya Nagdir", "Benedikt Stroebl", "Arvind Narayanan"],
       "year": 2024,
       "arxiv_id": "2409.11363",
       "relevance": "Benchmark focused on computational reproducibility of research, directly relevant to research code quality and methodology assessment."
     },
     {
       "title": "MLGym: A new framework and benchmark for advancing AI research agents",
       "authors": ["Deepak Nathani", "Lovish Madaan", "Nicholas Roberts"],
       "year": 2025,
       "relevance": "Framework and benchmark for AI research agents performing ML tasks, relevant to evaluating agentic AI capability in research workflows."
     },
     {
       "title": "The AI Scientist: Towards fully automated open-ended scientific discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "System for automated scientific discovery including code generation, directly relevant to AI capability in research code implementation."
     },
     {
       "title": "CodeJudge: Evaluating code generation with large language models",
       "authors": ["Weixi Tong", "Tianyi Zhang"],
       "year": 2024,
       "doi": "10.18653/v1/2024.emnlp-main.1118",
       "relevance": "LLM-as-judge approach for code evaluation, relevant to methodology of evaluating AI-generated code quality."
     },
     {
       "title": "Can LLMs generate novel research ideas? A large-scale human study with 100+ NLP researchers",
       "authors": ["Chenglei Si", "Diyi Yang", "Tatsunori Hashimoto"],
       "year": 2025,
       "relevance": "Large-scale evaluation of LLM capability in research idea generation, directly relevant to understanding AI's role in scientific research."
     },
     {
       "title": "Towards an AI co-scientist",
       "authors": ["Juraj Gottweis", "Wei-Hung Weng", "Alexander Daryin"],
       "year": 2025,
       "arxiv_id": "2502.18864",
       "relevance": "System exploring AI as a research collaborator, relevant to the broader context of AI capability in scientific research workflows."
     }
   ]
 }