ai-research-survey

scan.json (32628B)

---

 {
   "paper": {
     "title": "The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search",
     "authors": [
       "Yutaro Yamada",
       "Robert Tjarko Lange",
       "Cong Lu",
       "Shengran Hu",
       "Chris Lu",
       "Jakob Foerster",
       "Jeff Clune",
       "David Ha"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.08066",
     "doi": "10.48550/arXiv.2504.08066"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'We have open-sourced the code at https://github.com/SakanaAI/AI-Scientist-v2' (abstract/Section 1). Additionally, links to experiment-specific GitHub repositories are provided for each of the three workshop submissions (Appendix C)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states they 'open-source the full codebase for The AI Scientist-v2 and the ICLR 2025 workshop experiment data' (Section 1, contribution 4). The workshop experiment repositories are linked in Appendix C. The datasets used in the AI-generated papers are public benchmarks (synthetic arithmetic, CIFAR-10, MNIST, Kaggle pest dataset)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment specification is provided in the paper. The supplementary material lists models and hyperparameters (Tables 2-3) but not software dependencies or library versions needed to reproduce the system."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released on GitHub, the paper itself does not contain step-by-step reproduction instructions. There is no 'Reproducing Results' section or detailed README-style instructions in the paper for running the system end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main evaluation is based on three workshop submissions, with one accepted. No confidence intervals or error bars are reported for the system-level evaluation. The AI-generated papers internally use standard deviations across runs (Section 3.2.2 mentions replication nodes for mean and std), but the paper's own evaluation of the AI Scientist-v2 system provides no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the system can produce 'workshop-level' papers based on one out of three submissions being accepted (score 6.33 vs. acceptance threshold). No statistical test is used to assess whether this result is significantly different from chance or to compare against baselines."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports specific reviewer scores (6, 6, 7 for the accepted paper, average 6.33), contextualizes this as 'roughly in the top 45% of submissions,' and compares against the workshop acceptance threshold. While not a formal effect size metric, it provides enough context (baseline of average human acceptance threshold) to understand magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only three manuscripts were submitted to the workshop, and only one was accepted. The paper acknowledges this is a small sample ('Our current study aims to see whether The AI Scientist-v2 can produce at least one paper that survives peer review, and not what fraction of the time it can do so') but does not provide a formal justification for why N=3 is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance across multiple system runs is reported. The paper mentions running multiple seeds per idea and selecting the best, but does not report the distribution of quality across seeds. Only the selected best outputs are described."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares The AI Scientist-v2 against its predecessor, The AI Scientist-v1, via a feature comparison table (Table 1). The key baseline is that v1 was 'Not Submitted' to peer review while v2 achieved workshop acceptance. The internal author reviews also compare against workshop and conference quality standards."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "While Section 6 (Related Work) discusses contemporary systems like AIDE, Agent Laboratory, CycleResearcher, and AI-Researcher, no direct experimental comparison is made against any of these systems. The only comparison is with the predecessor v1, and it is qualitative (Table 1 feature comparison), not quantitative."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple novel components (agentic tree search, experiment manager, VLM reviewer, template-free codebase). No ablation study is conducted to determine which components contribute most to the quality improvement over v1. There is no experiment removing the VLM reviewer, the tree search, or the experiment manager individually."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses multiple dimensions: (1) official peer review scores from the workshop, (2) detailed internal author reviews with multiple scoring dimensions (soundness, presentation, contribution, overall workshop/conference scores), and (3) code reviews of the generated papers. Each of the three papers is assessed along these multiple criteria."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The core evaluation involves blind peer review at the ICLR 2025 ICBINB workshop (Section 4.1). Three manuscripts were reviewed by workshop reviewers who did not know which were AI-generated. Additionally, the authors conducted their own detailed internal review of all three manuscripts (Section 4.2)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a benchmark evaluation paper in the traditional sense. The evaluation is via peer review submission, which is inherently a held-out evaluation (reviewers had not seen the papers before), but the concept of train/test split does not apply to the system-level evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per paper: each of the three submissions receives individual reviewer scores, detailed internal reviews (Appendix C.1-C.3), and code reviews. The internal reviews provide per-dimension scores (soundness, presentation, contribution, overall). Per-reviewer scores are also reported for the accepted paper."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failure cases: two of three submissions were rejected, and the authors provide detailed analyses of weaknesses in all three papers, including inaccurate figure captions, hallucinated citations, dataset overlap issues (57% train-test overlap), contradictions between figures and text claims, and missing references. The code reviews (C.1.2, C.2.3, C.3.3) are particularly thorough in documenting failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple negative findings: two of three papers were rejected; the system produces inaccurate citations and hallucinations; figure captions sometimes contradict the actual figures; the system 'does not yet consistently reach the rigorous standard required for top-tier conference publications' (Section 5); code review found 57% train-test overlap in the accepted paper's experiments."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: the system produced 'the first entirely AI-generated peer-review-accepted workshop paper' — this is documented with specific scores (6.33 average, individual 6/7/6). The claim of eliminating template dependency, using tree search, and VLM feedback are all described in detail in Section 3. The abstract appropriately hedges with 'workshop-level' rather than claiming conference-level quality."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims that the v2 improvements (tree search, VLM feedback, template removal) led to the ability to produce workshop-quality papers. However, no controlled experiment isolates the contribution of each component. The comparison with v1 is confounded by many simultaneous changes. The paper states v2 'eliminates reliance on human-authored code templates' and 'leverages a novel progressive agentic tree-search methodology' as causes of improvement, but these are not experimentally validated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is generally careful about bounding its claims. Section 5 explicitly states: 'the acceptance occurred at a workshop level rather than at the main conference track, and only one of the three AI-generated submissions was accepted.' It notes workshop acceptance rates (60-80%) vs conference rates (20-30%) and states the system 'does not yet consistently reach the rigorous standard required for top-tier conference publications.' The title itself bounds to 'Workshop-Level.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses several alternative explanations and confounds: that workshop acceptance rates are much higher than conference rates (Section 5), that the accepted paper's 100% attention model accuracy was due to task simplicity not model quality (Code Review C.1.2), that the best paper was selected from multiple seeds (selection bias), and that the acceptance may reflect the ICBINB workshop's specific focus on negative results rather than general quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 in Appendix A specifies the models used: 'Claude 3.5 Sonnet (v2)' for code generation, 'GPT-4o' for LLM/VLM feedback agents and summary report agent. While 'GPT-4o' lacks a specific snapshot date, 'Claude 3.5 Sonnet (v2)' is a specific version identifier. The writeup mentions 'o1' for the reflection stage. This is borderline but provides more specificity than just saying 'Claude' or 'GPT-4.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix B, covering idea generation, experiment prompts, plot aggregation, writeup prompts, writeup reflection, VLM reflection, and VLM image review prompts. Some contain template placeholders (e.g., {tool_descriptions}, {idea_text}), but the actual prompt structure and instructions are provided in full. The fill values are contextual (e.g., experiment summaries) rather than tuned parameters."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports LLM/VLM hyperparameters including max tokens and temperature for each component. Table 3 reports tree search hyperparameters including debug probability, maximum debug depth, maximum experiment runtime per node, and node allocation per stage. Temperature values are explicitly stated (0.5 for code generation, 0.5 for feedback, 1.0 for summary)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in substantial detail: the experiment progress manager's four stages (Section 3.2.1), the parallelized agentic tree search with node types (Section 3.2.2), the VLM reviewer pipeline (Section 3.4), the idea generation workflow (Section 3.1), and the writeup/reflection pipeline. Figure 1 provides a workflow diagram and Figure 2 illustrates the tree search stages."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the process: idea generation produced ~40 ideas, from which three were selected based on alignment with workshop theme. For each selected idea, multiple complete manuscripts were generated with different random seeds, and the best was selected for submission (Section 4.2). The pest detection dataset was manually reduced to 1/10th size (noted in Appendix C.3.1). The data pipeline for the system itself is well-documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations & Ethical Considerations' and provides substantive discussion of the system's limitations, including workshop vs. conference quality gaps, challenges in generating novel hypotheses, and the inability to consistently produce even workshop-level papers."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: workshop acceptance rates (60-80%) are much higher than conferences (20-30%), making the achievement less impressive than it might seem; only one of three submissions was accepted; the system 'does not yet consistently reach the rigorous standard required for top-tier conference publications'; specific weaknesses like inability to formulate 'genuinely novel, high-impact hypotheses' or 'truly innovative experimental methodologies.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what it does not show: 'the current version of The AI Scientist-v2 does not yet consistently reach the rigorous standard required for top-tier conference publications, nor does it even reach workshop-level consistently' (Section 5). It also notes the study only addresses whether the system 'can produce at least one paper that survives peer review, and not what fraction of the time it can do so' (Section 4.2)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full AI-generated manuscripts are included in Appendix C with annotations. The workshop reviewer comments are reproduced (two of three reviewers granted permission). The experiment code repositories are linked on GitHub. The internal reviews and code reviews are provided in full."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how the three manuscripts were generated: broad topical prompts aligned with the ICBINB workshop scope, idea generation producing ~40 ideas, selection of three ideas, running each through the full pipeline with multiple seeds, and selection of the best manuscript per idea for submission (Section 4.1-4.2)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the arrangement with ICLR leadership and workshop organizers: reviewers were informed some submissions might be AI-generated, offered opt-out, and the AI-generated papers were included among 43 total submissions. Post-review, reviewers were informed of the experiment. IRB approval was obtained (H24-02652). Section 4.1 describes the controlled evaluation steps."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from idea generation to submission is documented: idea generation phase produced ~40 ideas (Section 4.2), three selected for full runs, multiple seeds per idea, best manuscript selected per idea, submitted to blind peer review among 43 total submissions. Each stage of the AI Scientist-v2 pipeline is described (Sections 3.1-3.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure or acknowledgments section is present in the paper. Author affiliations include Sakana AI (a company), University of British Columbia, Vector Institute, and FLAIR/Oxford. Jeff Clune is listed as a Canada CIFAR AI Chair. No specific grants or funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sakana AI (four authors), University of British Columbia, Vector Institute, FLAIR/University of Oxford, and Canada CIFAR AI Chair. The detailed author contributions section (pages 19) specifies each person's role."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Sakana AI is the primary affiliation for four of eight authors, and the AI Scientist is Sakana AI's product. The company has a direct commercial interest in demonstrating that their AI Scientist system can produce peer-reviewed research. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is provided. Multiple authors are affiliated with Sakana AI, the company behind the AI Scientist system. No declaration of patents, equity, or other financial interests is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It evaluates an agentic system's ability to produce scientific manuscripts. The relevant evaluation is by human peer review, not by benchmarking model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for the same reason: the system is not being evaluated on a benchmark where training data contamination would be relevant. The peer review process is the evaluation mechanism."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable: the evaluation is via peer review, not benchmark performance. The AI-generated papers internally use benchmarks, but the paper's own evaluation of the AI Scientist-v2 system is not a benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The study involves human participants (workshop reviewers evaluating AI-generated papers). No pre-registration is mentioned. The study protocol was arranged with ICLR and workshop organizers, but no pre-registration link (OSF, AsPredicted, etc.) is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'we obtained IRB approval from the University of British Columbia (H24-02652)' (Sections 4.1 and 5). This was obtained before the workshop submission."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographics of the workshop reviewers are reported. The paper does not describe reviewer expertise, experience level, or any characterization beyond their role as ICLR workshop reviewers."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that reviewers 'were informed in advance that some submissions might be AI-generated, but were not told which submissions were produced by The AI Scientist-v2. Reviewers could also opt out of reviewing potentially AI-generated manuscripts' (Section 4.1). This constitutes explicit inclusion/exclusion criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control assignment. The AI-generated papers were included among the regular 43 submissions and reviewed through the standard workshop review process. Randomization in the traditional sense does not apply."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Blinding is clearly described: 'Reviewers were informed in advance that some submissions might be AI-generated, but were not told which submissions were produced by The AI Scientist-v2' (Section 4.1). 'Reviewers were informed of the experiment only after peer review' (Section 4.1). This constitutes single-blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report how many reviewers opted out of reviewing potentially AI-generated manuscripts, nor whether any reviewers dropped out during the process. It only reports the final reviewer scores and comments."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: 'The total time required for The AI Scientist-v2 to generate a single paper depends on the complexity of the problems. Based on our experience, this process usually takes anywhere from several hours to a maximum of 15 hours, which is the runtime limit we have set' (Appendix A). However, no API costs in dollars or tokens consumed are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated in terms of API costs, total tokens consumed, or total compute hours across all experiments (multiple seeds per idea, three ideas, idea generation runs). Only the per-paper wall-clock time range (several hours to 15 hours) is mentioned. The total cost of the study is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "The AI Scientist-v2 produced the first entirely AI-generated paper to successfully navigate peer review at a recognized workshop.",
       "evidence": "One of three AI-generated manuscripts received reviewer scores of 6, 7, and 6 (average 6.33) at the ICLR 2025 ICBINB workshop, surpassing the workshop's acceptance threshold and placing in roughly the top 45% of submissions (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "The AI Scientist-v2 eliminates reliance on human-authored code templates, generalizing effectively across diverse ML domains.",
       "evidence": "The paper describes the template-free approach (Section 3.2) and demonstrates the system generating papers across three different domains (compositional generalization, label noise, pest detection). However, the only quantitative evaluation is the workshop submission outcomes (Section 4).",
       "supported": "moderate"
     },
     {
       "claim": "The agentic tree search methodology enables deeper and more systematic exploration of scientific hypotheses compared to v1's linear approach.",
       "evidence": "The tree search algorithm is described in detail (Section 3.2.2) with four stages and specialized node types. However, no direct quantitative comparison with v1's linear approach is provided. The claim rests on the architectural description and the workshop acceptance outcome.",
       "supported": "weak"
     },
     {
       "claim": "The system does not yet consistently reach the rigorous standard required for top-tier conference publications.",
       "evidence": "Two of three submissions were rejected. The accepted paper had identified issues: 57% train-test overlap, incorrect figure captions, hallucinated claims, and missing references (Sections 4.2, 5, Appendix C). Internal review rated it borderline accept for workshop only.",
       "supported": "strong"
     },
     {
       "claim": "VLM integration improves the visual quality and clarity of manuscripts.",
       "evidence": "The VLM feedback mechanism is described in Section 3.4, but no ablation study compares manuscripts generated with and without VLM feedback. The accepted paper still had figure caption inaccuracies noted by both internal and external reviewers.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "case-study",
     "benchmark-eval"
   ],
   "key_findings": "The AI Scientist-v2 is an end-to-end agentic system for automated scientific discovery that uses tree-search-based experimentation, an experiment manager, and VLM feedback. Three fully AI-generated papers were submitted to the ICLR 2025 ICBINB workshop, and one was accepted with an average reviewer score of 6.33/10. The accepted paper contained notable flaws including 57% train-test overlap, incorrect figure captions, and hallucinated citations, placing the achievement in context of workshop-level (60-80% acceptance rate) rather than conference-level quality. The authors identify that the system does not yet consistently produce even workshop-quality papers.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Four of eight authors are affiliated with Sakana AI, the company that builds the AI Scientist system. No competing interests statement is provided. The paper evaluates and promotes Sakana AI's product, creating a clear conflict of interest."
     },
     {
       "flag": "Selection bias in evaluation",
       "detail": "Multiple manuscripts were generated per idea across different seeds, and the best was selected for submission. This selection process inflates the apparent success rate. The paper acknowledges this but does not quantify how many total manuscripts were generated or what the distribution of quality looked like."
     },
     {
       "flag": "Very small sample size for headline claim",
       "detail": "The claim of producing 'the first AI-generated peer-reviewed workshop paper' is based on submitting only 3 papers to a single workshop with a 60-80% acceptance rate. The 1/3 success rate is not statistically distinguishable from the baseline acceptance rate, yet the paper treats this as a significant milestone."
     },
     {
       "flag": "Known data integrity issues in accepted paper",
       "detail": "The authors' own code review found approximately 57% overlap between training and test sets in the accepted paper's experiments (Appendix C.1.2), yet this paper was still submitted and accepted. This undermines the validity of the accepted paper's experimental results and raises questions about the system's scientific rigor."
     },
     {
       "flag": "No ablation study for system components",
       "detail": "The paper introduces multiple novel components (tree search, experiment manager, VLM reviewer, template-free generation) simultaneously but provides no ablation to determine which components matter. The improvement over v1 cannot be attributed to any specific innovation."
     }
   ],
   "cited_papers": [
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Predecessor system (v1) that this paper builds upon; directly relevant to understanding the evolution of AI-driven scientific discovery systems."
     },
     {
       "title": "AIDE: AI-Driven Exploration in the Space of Code",
       "authors": ["Zhengyao Jiang", "Dominik Schmidt", "Dhruv Srikanth"],
       "year": 2025,
       "arxiv_id": "2502.13138",
       "relevance": "Tree search approach for LLM-driven code generation that inspired the AI Scientist-v2's experimentation methodology; relevant as an agentic coding system."
     },
     {
       "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2025,
       "relevance": "Benchmark for evaluating ML engineering agents, directly relevant to evaluating agentic AI systems for research tasks."
     },
     {
       "title": "Can LLMs Generate Novel Research Ideas? A Large-Scale Human Study with 100+ NLP Researchers",
       "authors": ["Chenglei Si", "Diyi Yang", "Tatsunori Hashimoto"],
       "year": 2025,
       "relevance": "Large-scale study evaluating LLM idea generation capabilities through human evaluation, directly relevant to AI-driven research quality assessment."
     },
     {
       "title": "RE-Bench: Evaluating Frontier AI R&D Capabilities of Language Model Agents Against Human Experts",
       "authors": ["Hjalmar Wijk", "Tao R. Lin", "Joel Becker"],
       "year": 2024,
       "arxiv_id": "2411.15114",
       "relevance": "Benchmark comparing AI agent R&D capabilities to human experts, relevant to understanding the frontier of agentic AI research capabilities."
     },
     {
       "title": "An Evaluation of Sakana's AI Scientist for Autonomous Research",
       "authors": ["Joeran Beel", "Min-Yen Kan", "Moritz Baumgart"],
       "year": 2025,
       "arxiv_id": "2502.14297",
       "relevance": "Independent evaluation of The AI Scientist-v1, directly relevant as external assessment of automated research systems."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "relevance": "Foundational agentic scaffolding framework for iterative self-improvement of LLM agents, relevant to agentic AI methodology."
     },
     {
       "title": "Agent Laboratory: Using LLM Agents as Research Assistants",
       "authors": ["Samuel Schmidgall"],
       "year": 2025,
       "arxiv_id": "2501.04227",
       "relevance": "Concurrent work on using LLM agents as research assistants, relevant to the survey scope of agentic AI for research."
     },
     {
       "title": "CycleResearcher: Improving Automated Research via Automated Review",
       "authors": ["Yixuan Weng", "Minjun Zhu", "Guangsheng Bao"],
       "year": 2025,
       "relevance": "System for automated research from idea generation to manuscript drafting, directly relevant as a concurrent automated scientific discovery system."
     },
     {
       "title": "Superintelligent Agents Pose Catastrophic Risks: Can Scientist AI Offer a Safer Path?",
       "authors": ["Yoshua Bengio"],
       "year": 2025,
       "arxiv_id": "2502.15657",
       "relevance": "Discusses safety implications of agentic vs. scientist AI systems, relevant to AI safety considerations in the agentic AI space."
     },
     {
       "title": "Towards an AI Co-Scientist",
       "authors": ["Juraj Gottweis", "Wei-Hung Weng", "Alexander Daryin"],
       "year": 2025,
       "arxiv_id": "2502.18864",
       "relevance": "Google's AI Research Copilot effort, a major industry contribution to AI-assisted scientific discovery."
     },
     {
       "title": "SciCode: A Research Coding Benchmark Curated by Scientists",
       "authors": ["Minyang Tian", "Luyu Gao"],
       "year": 2024,
       "relevance": "Research-oriented coding benchmark spanning physics, chemistry, and biology, relevant to evaluating AI research capabilities."
     }
   ]
 }