ai-research-survey

scan.json (21201B)

---

 {
   "paper": {
     "title": "CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark",
     "authors": ["Zachary S. Siegel", "Sayash Kapoor", "Nitya Nadgir", "Benedikt Stroebl", "Arvind Narayanan"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.11363"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/siegelz/core-bench (footnote 3, Section 2.1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is released alongside the code at the GitHub repository. Tasks are based on publicly available CodeOcean capsules."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation harness specifies Azure VM types (Standard_E2as_v5, Standard_NC4as_T4_v3), Ubuntu Linux, 80 GB disk (Appendix B). Agents run in isolated VMs with specified configurations."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes an evaluation harness with detailed instructions for running agents on VMs, including resume flags and resource management (Appendix B). The GitHub repository provides the harness code."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals reported for CORE-Agent accuracy across three runs in Table A3 (e.g., '60.60% ± 4.51%'). Error bars shown in Figure 8."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CORE-Agent outperforms AutoGPT and GPT-4o outperforms GPT-4o-mini, but no statistical significance tests are reported for these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute accuracy differences with baseline context are reported throughout (e.g., AutoGPT 6.7% vs CORE-Agent 21.48% on Hard; GPT-4o-mini 8.9% → 44.44% with task-specific modifications)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 90 papers (45 train/45 test) and CORE-Agent was run 3 times, but no justification for why 90 papers or 3 runs is sufficient. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported from three trials in Figure 8 error bars, and confidence intervals in Table A3. AutoGPT was only run once due to cost constraints, which is acknowledged."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines compared: unmodified AutoGPT (general-purpose) and CORE-Agent (task-specific), each with two LLM backends (Section 3, Table 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "AutoGPT is a contemporary general-purpose agent. This is a new benchmark so there are no prior baselines to compare against; the chosen baselines are reasonable for establishing initial results."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between AutoGPT and CORE-Agent across difficulty levels effectively serves as an ablation of task-specific modifications. Table 4 details incremental modifications. The cost limit analysis (Figure 7) ablates the budget constraint."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports task accuracy (pass@1), average cost, pass@k, pass^k reliability metric, and breakdowns by vision vs. text questions (Sections 4, C.3, C.4)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper measuring automated agent accuracy against ground-truth reproductions. Human evaluation of agent outputs is not relevant — correctness is objectively verifiable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Explicit 45/45 train/test split. 'We report all results in this section on the test split unless otherwise mentioned, since we used the train split while developing the agent' (Section 4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by difficulty level (Easy/Medium/Hard), discipline (CS/Social Science/Medicine), programming language (Python/R), and question modality (vision/text) in Section 4 and Figure 8."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 qualitatively analyzes common failure cases at each difficulty level: retrieval from multiple files, dependency installation loops, competing instructions. Appendices D.3.1-D.3.4 provide detailed examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Increasing cost limit from $4 to $10 did not significantly improve GPT-4o-mini performance (Section 4.3, Figure 7). Section 4.7 reports an agent attempting unsafe behavior (creating online accounts)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (best agent 21% on hardest task, 270 tasks from 90 papers, three disciplines) are all supported by results in Tables 5 and A3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about task-specific modifications improving accuracy are supported by controlled comparisons (same LLM, same benchmark, only modifications differ). The ablation design is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested setting. The paper acknowledges limitations of CodeOcean sourcing and specific disciplines. Title and abstract appropriately scope to 'computational reproducibility' rather than broader claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: Python vs R performance differences may be due to output format rather than language difficulty (Section 4.5), CS tasks being easier may be confounded with Python prevalence, cost limits may constrain performance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions specified: 'GPT-4o-2024-05-13 and GPT-4o-mini-2024-07-18' (Section 3)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full CORE-Agent prompts provided in Appendix D.2, including the --ai-role and --best-practice arguments for each difficulty level."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, or other API sampling parameters for the LLM calls. Only the $4 cost limit per task is specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "AutoGPT architecture described including tools (execute_shell, query_vision_language_model), auto-summarization of past actions, and CORE-Agent modifications (Table 4, Appendix D)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Capsule selection process documented with 10 filtering criteria (Table 2), filtering pipeline from 5,090 capsules to 90 (Figure 3), and task question construction process (Appendix A.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. Some limitations are scattered throughout (e.g., CodeOcean sourcing, cost constraints) but there is no substantive dedicated discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity discussed. The paper does not address construct validity threats from using CodeOcean (pre-verified reproducible capsules) vs. real-world papers, or selection bias from the 10 filtering criteria."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not discuss that the benchmark only covers papers already known to be reproducible, limiting generalization to real-world reproducibility checking."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data and evaluation harness are released at the GitHub repository. CodeOcean capsules are publicly accessible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection via CodeOcean webscraping described (Appendix A.1), with 10 filtering criteria (Table 2) and manual verification of each capsule's local reproducibility."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sourced from CodeOcean public repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: 5,090 capsules scraped → filtered by discipline/language/10 criteria → 90 capsules selected → 3 manual reproductions per capsule → 181 task questions created (Figure 3, Appendix A)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section mentions compute support from Princeton University's Center for Statistics and Machine Learning and OpenAI's researcher access program."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with Princeton University, clearly stated. No authors are affiliated with OpenAI or AutoGPT."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI provided compute support via researcher access program, and the paper exclusively evaluates OpenAI models (GPT-4o, GPT-4o-mini). OpenAI has a financial interest in demonstrating model capabilities."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for GPT-4o or GPT-4o-mini, despite evaluating them on tasks derived from public CodeOcean repositories."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges contamination concerns: 'CORE-Bench's foundation in public repositories enables periodic updates of the benchmark tasks, which could mitigate concerns about contamination and saturation' (Section 2.1)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions the possibility of periodic updates to mitigate contamination, it does not analyze whether the CodeOcean capsules used were available before GPT-4o's training cutoff. Many capsules likely predate the model's training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Average API costs reported per task and per difficulty level (Figure 6, Table A3). Successful tasks averaged $0.54 vs $2.59 for failed tasks with CORE-Agent GPT-4o."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Azure VM types specified (Appendix B), $4 per-task cost limit stated, total evaluation time (~2 hours with parallelization vs 20+ days sequential). Cost limit sensitivity analysis in Figure 7."
       }
     }
   },
   "claims": [
     {
       "claim": "The best agent (CORE-Agent with GPT-4o) achieves 21% accuracy on CORE-Bench-Hard, 58% on Medium, and 60% on Easy.",
       "evidence": "Table 5 and Table A3 report these results on the test set across three runs with confidence intervals.",
       "supported": "strong"
     },
     {
       "claim": "Task-specific modifications to general-purpose agents yield significant performance improvements, especially for weaker models.",
       "evidence": "Section 4.2: AutoGPT GPT-4o-mini went from 8.9% to 44.44% on Easy with minimal modifications. Table 5 shows consistent improvements across all settings.",
       "supported": "strong"
     },
     {
       "claim": "Stronger models (GPT-4o) outperform weaker models (GPT-4o-mini) despite having a lower token budget due to higher per-token costs.",
       "evidence": "Section 4.3 and Table 5: GPT-4o outperforms GPT-4o-mini across all agents and difficulty levels despite the same $4 cost limit.",
       "supported": "strong"
     },
     {
       "claim": "Increasing the cost limit from $4 to $10 does not significantly improve accuracy because agents get stuck rather than needing more budget.",
       "evidence": "Section 4.3 and Figure 7: GPT-4o-mini unchanged, GPT-4o modest increase from 26% to 31% on train set Hard tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Computational reproducibility is a widespread problem across scientific fields, with significant proportions of studies failing despite available code and data.",
       "evidence": "Table 1 summarizes 18 studies across 15+ fields showing reproducibility failure rates. The ML Reproducibility Challenge 2022 data shows 10/28 papers had errors.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CORE-Bench introduces 270 computational reproducibility tasks from 90 papers across CS, social science, and medicine at three difficulty levels. The best baseline agent (CORE-Agent with GPT-4o) achieves only 21% on the hardest tasks, showing substantial room for improvement. Task-specific modifications to general-purpose agents yield large gains (e.g., 8.9% to 44.4% for GPT-4o-mini on Easy), and stronger models outperform weaker ones even with lower token budgets. Agents primarily fail due to dependency installation loops and difficulty retrieving results from multiple output files.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section, which is notable for a benchmark paper making claims about real-world utility."
     },
     {
       "flag": "Selection bias in benchmark construction",
       "detail": "All 90 papers are pre-verified as reproducible on CodeOcean, with 10 filtering criteria that exclude many real-world scenarios (e.g., >45 min runtime, >10GB, complex bash commands). This limits construct validity claims about real-world reproducibility, but this is not discussed."
     },
     {
       "flag": "OpenAI compute support with exclusive OpenAI model evaluation",
       "detail": "OpenAI provided researcher access program support, and only OpenAI models were evaluated. No non-OpenAI models tested despite the benchmark being model-agnostic."
     },
     {
       "flag": "AutoGPT only run once",
       "detail": "Due to cost constraints, AutoGPT was only run once on the test set, so no confidence intervals are available for the baseline comparison. This weakens the comparative claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational code generation benchmark (HumanEval) that CORE-Bench aims to complement with more realistic tasks."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Key benchmark for evaluating AI agents on real-world software engineering tasks, directly compared to CORE-Bench's approach."
     },
     {
       "title": "AI Agents That Matter",
       "authors": ["Sayash Kapoor", "Benedikt Stroebl", "Zachary S. Siegel"],
       "year": 2024,
       "arxiv_id": "2407.01502",
       "relevance": "Companion paper on agent evaluation methodology, addressing cost-accuracy tradeoffs and benchmark design principles."
     },
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "authors": ["Chris Lu"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Proposes automating the full research pipeline; CORE-Bench argues reproducibility is a necessary prerequisite step."
     },
     {
       "title": "SWE-AGENT: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez"],
       "year": 2024,
       "relevance": "Demonstrates importance of task-specific agent modifications for software engineering, paralleling CORE-Agent's approach."
     },
     {
       "title": "Lessons from the Trenches on Reproducible Evaluation of Language Models",
       "authors": ["Stella Biderman"],
       "year": 2024,
       "arxiv_id": "2405.14782",
       "relevance": "Discusses challenges in reproducible LLM evaluation, directly relevant to benchmarking methodology."
     },
     {
       "title": "Benchmarking Large Language Models As AI Research Agents",
       "authors": ["Qian Huang"],
       "year": 2023,
       "arxiv_id": "2310.03302",
       "relevance": "Benchmark for LLMs as research agents conducting ML experiments, complementary to CORE-Bench's reproducibility focus."
     },
     {
       "title": "SciCode: A Research Coding Benchmark Curated by Scientists",
       "authors": ["Minyang Tian"],
       "year": 2024,
       "arxiv_id": "2407.13168",
       "relevance": "Research coding benchmark for scientific tasks, a related benchmark in the AI-for-science agent space."
     },
     {
       "title": "Security of AI Agents",
       "authors": ["Yifeng He"],
       "year": 2024,
       "arxiv_id": "2406.08689",
       "relevance": "Addresses safety concerns for AI agents, relevant to CORE-Bench's finding about agents attempting unsafe web actions."
     },
     {
       "title": "DiscoveryBench: Towards Data-Driven Discovery with Large Language Models",
       "authors": ["Bodhisattwa Prasad Majumder"],
       "year": 2024,
       "arxiv_id": "2407.01725",
       "relevance": "Benchmark for LLM scientific discovery capabilities, complementary to CORE-Bench's reproducibility evaluation."
     }
   ]
 }