ai-research-survey

scan.json (28262B)

---

 {
   "paper": {
     "title": "ASTROVISBENCH: A Code Benchmark for Scientific Computing and Visualization in Astronomy",
     "authors": [
       "Sebastian Joseph",
       "Syed Murtaza Husain",
       "Stella S. R. Offner",
       "Stéphanie Juneau",
       "Paul Torrey",
       "Adam S. Bolton",
       "Juan P. Farias",
       "Niall Gaffney",
       "Greg Durrett",
       "Junyi Jessy Li"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2505.20538"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We release the code and data for ASTROVISBENCH at astrovisbench.github.io' in the abstract, and provides the URL https://astrovisbench.github.io/ at the top of the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (864 tasks from 110 Jupyter notebooks) is released at the same URL. The paper says 'We release the code and data for ASTROVISBENCH.' The source notebooks are from publicly available collections (Astro Data Lab, STScI)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix F mentions the hardware used (two 56-Core Intel Xeon MAX 9480 CPUs, 128GB RAM, ~100GB storage), but there is no requirements.txt, Dockerfile, conda environment, or detailed listing of library versions needed to reproduce the benchmark execution environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released at the project website, the paper itself does not contain step-by-step reproduction instructions, a README with commands, or a 'Reproducing Results' section. It is unclear how to set up and run the full benchmark evaluation pipeline from the paper alone."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports only point estimates for all metrics (Crash %, VIscore, NoE %, MiE %, MaE %) across all eight models. No confidence intervals, error bars, or uncertainty measures are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Gemini 2.5 Pro dominated in its ability to produce code that executes') but no statistical significance tests are used to support these comparisons. Rankings are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. The paper presents raw percentages but does not contextualize differences with effect size measures. Some raw differences are given (e.g., crash rates range from 30.8% to 64.1%), but these lack formal effect size framing."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 864 tasks (432 processing + 432 visualization) from 110 notebooks, but there is no justification for why this size is sufficient, no power analysis, and no discussion of whether the sample enables reliable model comparisons."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs of each model. No variance, standard deviation, or multi-run spread is reported. Each model was evaluated once with temperature=1, but no repeated evaluations were conducted to measure result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eight state-of-the-art LLMs are compared against each other as baselines (Gemini 2.5 Pro, Claude 3.7 Sonnet, Claude 4.0 Opus, o3-mini, GPT-4o, QwQ, Qwen-2.5, Llama-4 Maverick), providing mutual comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models are all contemporary and represent the state of the art at the time of writing: Gemini 2.5 Pro, Claude 4.0 Opus, Llama-4 Maverick, o3-mini, etc. These are recent models from major AI labs."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has multiple design choices (query generation, underspecification clarification, VLM judge selection, processing/visualization split) but no ablation study measures the impact of these individual components on benchmark results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Crash % (code execution success), VIscore (variable inspection), VisFail % (visualization failure), and a three-category error classification (NoE, MiE, MaE) for visualization evaluation. Additionally, Fleiss' kappa and Spearman's rho are used for inter-annotator agreement."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Five professional astronomers (post-PhD researchers and faculty) evaluated 135x2 LLM-generated visualizations, providing gold-standard judgments. The expert annotations were used to validate the automatic VLM-as-judge evaluator (Section 3.2.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The entire benchmark of 864 tasks is used for evaluation. There is no separation into dev/test splits. The 30-task agreement subset is for validating the VLM judge, not for tuning model performance, but no held-out test set concept applies to the LLM evaluations themselves."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down into processing tasks and visualization tasks (Table 5), with separate metrics for each. Error type distributions are provided (Figure 4), and visualization errors are categorized into no error, minor error, and major error."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides a detailed error analysis, discussing execution errors (Figure 4 with top 10 error types), visualization errors (domain-specific convention failures, suboptimal scaling, readability issues), and Appendix E shows examples of no error, minor error, and major error judgments with expert rationales."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that all models perform poorly on the benchmark, with even the best model (Gemini 2.5 Pro) failing at least 58% of the time on visualizations. They also note that Gemini 2.5 Pro's high execution rate does not translate to high scientific correctness (VIscore was not among the highest)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'a significant gap in their ability to engage in astronomy research as useful assistants,' which is supported by Table 5 showing high crash rates (30-64%), low VIscores (0.47-0.69), and high major error rates (14-31%). The abstract also claims the VLM-as-judge is 'validated against annotation by five professional astronomers,' which is supported by Table 2 showing high correlations."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper largely avoids strong causal claims. It describes what models fail at (e.g., 'many of these models lack the knowledge necessary to use niche, domain-specific libraries') based on empirical observation from error analysis. The causal language used is modest and supported by the specific error patterns observed in Section 6."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames results within the astronomy domain and does not overclaim generalization to all scientific fields. The title and abstract specify 'astronomy domain.' The conclusion says this 'paves the way for the future development of models that can aid researchers across a wide range of domains' but frames this as future work rather than a demonstrated claim."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for why models fail. For example, failures could be due to prompt design, context length limitations, or the specific choice of notebooks rather than fundamental model limitations. The Limitations section discusses noise from LLM-based construction and VLM judge limitations but does not consider alternative explanations for the observed performance gaps."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists model names (GPT-4o, Claude 3.7 Sonnet, Claude 4.0 Opus, Gemini 2.5 Pro, o3-mini, Qwen-2.5 72B, QwQ 32B, Llama-4 Maverick 17Bx128E) but does not provide specific API version identifiers or snapshot dates for the proprietary models. No version strings like 'gpt-4o-2024-xx-xx' are given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides the full prompts used for benchmark construction (Appendix B.2), underspecification clarification (B.3), code generation (B.4), and VLM evaluation (B.1). These are the actual prompt texts with template placeholders whose fill values are defined."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D reports hyperparameter settings: temperature=1 for all models, top_p=1 for most (0.95 for Gemini), max_tokens=1024 for Claude 3.7 Sonnet (with 10 responses hitting the limit noted), and max_tokens=8000 with 3000 thinking tokens for Claude 4.0 Opus. Open-source models used Together.AI API defaults."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is a straightforward single-turn code generation task: given a prompt, each model generates code, which is then executed and evaluated."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the full pipeline from notebook collection (Astro Data Lab and STScI sources) through dependency extraction, stage splitting (setup/processing/visualization), query generation with GPT-4o, and underspecification clarification. The construction methodology is detailed with filtering criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations' subsection appears at the end of Section 8 (Conclusion), discussing three specific limitations: LLM-aided benchmark construction may introduce noise, VLM-based evaluation may not align with expert judgments, and variable inspection has limitations with certain runtime objects."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) GPT-4o-generated queries may introduce hallucinations (though experts verified a subset), (2) VLM judge may diverge from expert assessments (mitigated by correlation validation), (3) pickling limitations prevent evaluation of certain runtime objects. These are not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper focuses on astronomy, it does not explicitly state what the results do NOT show. For example, it does not state that results may not generalize to other scientific domains, that the notebook sources (NOIRLab, STScI tutorials) may not represent all astronomy research workflows, or that single-turn evaluation does not reflect interactive coding assistance."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that code and data are released at astrovisbench.github.io, which would include the benchmark tasks, source notebooks, and evaluation data, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes notebook collection from Astro Data Lab and STScI, the types of notebooks (tutorials and scientific use cases), the target audience (astronomy graduate students and professional researchers), and the range of tasks covered."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "For the expert evaluation, the paper describes the annotators as 'five professional astronomers, all of whom have a doctoral degree in astronomy, astrophysics, or physics, and are working as researchers or faculty members' (Section 3.2.1). They engaged in 6 hours of group discussions plus asynchronous annotation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the pipeline: notebook collection from two sources, dependency extraction from visualization cells, splitting into three stages (setup/processing/visualization), query generation with GPT-4o, underspecification clarification with GPT-4o, and code matching verification. The final benchmark sizes are stated (864 tasks from 110 notebooks)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding from the National Science Foundation under Cooperative Agreement 2421782, the Simons Foundation grant MPS-AI-00010515 (NSF-Simons AI Institute for Cosmic Origins - CosmicAI), and computational resources from TACC."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Texas at Austin, NSF NOIRLab, University of Virginia, SLAC National Accelerator Laboratory, and Texas Advanced Computing Center. None of the authors appear to be affiliated with the AI companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (NSF, Simons Foundation) are independent scientific funding agencies with no financial stake in the performance of any particular LLM evaluated in the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper. While this appears to be a purely academic study, the absence of a formal declaration means this criterion is not met."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates eight LLMs on benchmark tasks but does not state the training data cutoff dates for any of the models. This is relevant because the source notebooks from Astro Data Lab and STScI are publicly available and could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The source notebooks are publicly available GitHub repositories (github.com/astro-datalab/notebooks-latest, spacetelescope.github.io/notebook-infrastructure/), which are likely in the training data of the evaluated models. This potential overlap is not discussed anywhere in the paper."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from publicly available Jupyter notebooks that have been online for years. Since most evaluated models were trained on web data including GitHub repositories, contamination is a serious concern. The paper does not address this at all, despite it being a critical threat to validity for a benchmark paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The expert annotation task is a benchmark validation exercise with professional astronomers, not a human subjects study requiring pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The expert annotation involves professional researchers providing domain judgments, not a human subjects experiment requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The expert annotators are professional astronomers serving as domain validators, not study participants. Their qualifications (doctoral degrees, researcher/faculty status) are reported as relevant background."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Expert annotators were domain specialists recruited for benchmark validation, not participants in a human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment with randomization to conditions was conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment requiring blinding was conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study with potential attrition was conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report any API costs, tokens consumed per model, or wall-clock time for running the benchmark evaluations across eight models. This is relevant since the evaluation involves 864 tasks per model plus VLM-based evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F states the hardware used: 'two 56-Core Intel Xeon MAX 9480 CPUs with 128GB of RAM' and notes '~100GB of storage' per model evaluation, with '~50GB for the execution environment' and '~50GB for pickled objects.' However, total API spend and GPU hours are not mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "ASTROVISBENCH is the first benchmark for both scientific computing and visualization in the astronomy domain.",
       "evidence": "The paper constructs 864 tasks (432 processing + 432 visualization) from 110 Jupyter notebooks sourced from Astro Data Lab and STScI collections (Section 2). Table 6 in Appendix C compares with related benchmarks and shows no prior astronomy-specific workflow benchmark exists.",
       "supported": "strong"
     },
     {
       "claim": "The VLM-as-judge (Claude 3.5 Sonnet) achieves high correlation with expert astronomer judgments for visualization evaluation.",
       "evidence": "Table 2 shows Claude 3.5 Sonnet achieves Spearman correlation of 0.822 (avg) and 0.828 (majority) with expert judgments, with p < 1e-29. This was validated on 30 tasks annotated by all 5 experts (Section 3.2.2).",
       "supported": "moderate"
     },
     {
       "claim": "State-of-the-art LLMs show a significant gap in their ability to engage in astronomy research workflows as useful assistants.",
       "evidence": "Table 5 shows crash rates of 30.8-64.1% for processing tasks, VIscores of 0.47-0.69, and combined visualization failure (crash + VisFail + MaE) rates exceeding 58% for all models. Even the best model (Gemini 2.5 Pro) produces correct visualizations only 15.7% of the time (Section 5).",
       "supported": "strong"
     },
     {
       "claim": "Better code execution success does not necessarily mean better scientific computing accuracy.",
       "evidence": "Gemini 2.5 Pro had the lowest crash rate (30.8%) but its VIscore (0.600) was not the highest — o3-mini had a higher VIscore (0.694) despite a higher crash rate (51.4%). This is discussed in Section 5.1.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs lack knowledge necessary to use niche, domain-specific libraries and APIs in astronomy.",
       "evidence": "Section 6 identifies FileNotFoundError, queryClientError (ADQL queries), and AttributeError as top errors. The paper attributes these to LLMs hallucinating file paths, function calls, and arguments for domain-specific tools. Table 4 lists 38 libraries with 26 astronomy-specific ones.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ASTROVISBENCH introduces 864 benchmark tasks for evaluating LLMs on astronomy-specific scientific computing and visualization workflows. Evaluation of eight state-of-the-art LLMs reveals substantial performance gaps: even the best model (Gemini 2.5 Pro) crashes on 30.8% of processing tasks and produces correct visualizations only 15.7% of the time. The paper develops a VLM-as-judge approach validated against five professional astronomers (Spearman rho = 0.828 with majority labels). Key failure modes include inability to use domain-specific APIs, hallucinating file paths and function calls, and violations of astronomy-specific plotting conventions.",
   "red_flags": [
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The source notebooks are from publicly available GitHub repositories (astro-datalab/notebooks-latest, STScI collections) that have been online for years. Since all evaluated models train on web data including GitHub, the test data may already be in training sets. This fundamental contamination concern is never discussed, which could inflate or distort model performance patterns."
     },
     {
       "flag": "No variance or repeated runs",
       "detail": "All eight models were evaluated with temperature=1 (stochastic sampling) but results appear to be from single runs only. With 864 tasks and stochastic generation, different runs could yield meaningfully different results. No standard deviations, confidence intervals, or repeated measurements are reported."
     },
     {
       "flag": "Small expert validation sample",
       "detail": "The VLM-as-judge is validated against only 30 tasks annotated by all 5 experts. While the correlation is high (0.828), this is a small sample to validate an automatic evaluator that is then used to assess all 432 visualization tasks across 8 models. Moderate inter-annotator agreement (Fleiss' kappa = 0.53) on this small sample adds further uncertainty."
     },
     {
       "flag": "GPT-4o used in benchmark construction",
       "detail": "GPT-4o was used to generate queries and handle underspecification clarification during benchmark construction, but GPT-4o is also one of the evaluated models. This creates a potential advantage or disadvantage for GPT-4o depending on whether the generated queries are biased toward or against its capabilities."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM coding ability on real-world software engineering tasks, directly comparable to ASTROVISBENCH's domain-specific code evaluation."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2025,
       "relevance": "Benchmark for LLM code generation with diverse function calls, relevant to evaluating how models handle complex API usage."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational HumanEval benchmark for LLM code generation evaluation, establishing the evaluation paradigm used in this work."
     },
     {
       "title": "The AI scientist: Towards fully automated open-ended scientific discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "relevance": "Explores fully automated AI-driven scientific research, directly relevant to the goal of LLMs as scientific research assistants."
     },
     {
       "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
       "authors": ["Giulio Starace", "Oliver Jaffe", "Dane Sherburn"],
       "year": 2025,
       "relevance": "Benchmark evaluating AI's ability to replicate ML research workflows, closely related to ASTROVISBENCH's focus on scientific research workflows."
     },
     {
       "title": "SciCode: A research coding benchmark curated by scientists",
       "authors": ["Minyang Tian", "Luyu Gao"],
       "year": 2024,
       "relevance": "Benchmark for scientific coding tasks curated by domain experts, directly comparable to ASTROVISBENCH's approach of expert-curated scientific benchmarks."
     },
     {
       "title": "MLE-bench: Evaluating machine learning agents on machine learning engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury"],
       "year": 2025,
       "relevance": "Benchmark for evaluating ML engineering capabilities of LLM agents, relevant as a scientific coding benchmark comparison."
     },
     {
       "title": "MatPlotAgent: Method and evaluation for LLM-based agentic scientific data visualization",
       "authors": ["Zhiyu Yang", "Zihan Zhou"],
       "year": 2024,
       "relevance": "Agentic framework for LLM-based visualization code generation, directly relevant as a visualization-focused LLM evaluation approach."
     },
     {
       "title": "VisEval: A Benchmark for Data Visualization in the Era of Large Language Models",
       "authors": ["Nan Chen", "Yuge Zhang", "Jiahang Xu", "Kan Ren", "Yuqing Yang"],
       "year": 2025,
       "relevance": "Benchmark for evaluating LLM-generated visualizations using execution-based and VLM evaluation, directly comparable evaluation methodology."
     },
     {
       "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",
       "authors": ["Qian Huang", "Jian Vora", "Percy Liang", "Jure Leskovec"],
       "year": 2023,
       "relevance": "Benchmark for LLM agents performing ML experimentation tasks, relevant to the broader theme of AI-assisted scientific research."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Rigorous evaluation methodology for LLM code generation, relevant to understanding evaluation challenges in code benchmarks."
     }
   ]
 }