ai-research-survey

scan.json (27662B)

---

 {
   "paper": {
     "title": "Plot2Code: A Comprehensive Benchmark for Evaluating Multi-modal Large Language Models in Code Generation from Scientific Plots",
     "authors": [
       "Chengyue Wu",
       "Zhixuan Liang",
       "Yixiao Ge",
       "Qiushan Guo",
       "Zeyu Lu",
       "Jiahao Wang",
       "Ying Shan",
       "Ping Luo"
     ],
     "year": 2025,
     "venue": "Findings of the Association for Computational Linguistics: NAACL 2025",
     "doi": "10.18653/v1/2025.findings-naacl.164"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Plot2Code introduces a 368-plot benchmark (matplotlib + plotly) for evaluating MLLMs' ability to generate code from scientific plots. GPT-4V achieves the highest rating of 7.68/10, with open-source models significantly lagging behind closed-source ones. Traditional low-level metrics (MSE, SSIM) fail to distinguish image quality, while the proposed GPT-4V rating and text-match ratio show significant discriminative power validated against human evaluation (Pearson r=0.479). Prompt strategies like CoT and Plan-and-Solve do not clearly improve performance on this visual coding task.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository is provided: https://github.com/TencentARC/Plot2Code (linked in the abstract/header area)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace under Apache-2.0 license: https://huggingface.co/datasets/TencentARC/Plot2Code (Appendix E.6)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is mentioned in the paper. Only general library names (matplotlib, plotly) are referenced."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While prompts are given in the appendix, there are no instructions for running the full evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 3 report point estimates only (e.g., pass rates, ratings) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 performs hypothesis tests (two-sample t-tests) to validate evaluation metrics, reporting t-statistics and p-values in Table 4. Correlation analysis with Kendall's Tau, Pearson, and Spearman coefficients in Table 5."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw scores and differences between models are reported, but no formal effect sizes (Cohen's d, odds ratios, etc.) are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('Robustness Analysis upon Sample Size') shows that scores stabilize above 60 samples (Figure 8), empirically justifying the 132-sample matplotlib benchmark size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main evaluation results in Table 3 are single-run point estimates with no variance, standard deviation, or spread measures across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "14 models are compared, including both closed-source (GPT-4V, Claude-3-Opus, Gemini-Pro) and open-source (DeepSeek-VL, LLaVA, Mini-Gemini) models (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include contemporary systems: GPT-4V, Claude-3-Opus, Gemini-Pro, DeepSeek-VL, LLaVA-1.6, all from 2023-2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 ablates prompt strategies (CoT, PS+), resolution settings (low vs. high), and OCR token inclusion (Figure 5). Section 4.4 provides pairwise comparisons."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: code pass rate, text-match ratio, and GPT-4V rating judgement (Section 3.4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is conducted: 100 people rate GPT-4V generated samples (absolute rating), and 46 people complete pairwise comparison questionnaires (Appendix C)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is a dedicated test set. Pre-trained models are evaluated zero-shot without any fine-tuning on the benchmark data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides per-plot-type breakdowns across 6 categories for three models. Table 1 shows statistics by type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B shows case studies including a direct asking failure (Figure 10, rated 3/10). The paper discusses challenges with text-dense plots and cases where models struggle."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CoT and PS+ prompt strategies show no clear advantage (Section 4.3, Figure 4). Higher resolution does not always help (Mini-Gemini-34B-HD underperforms Mini-Gemini-8x7B-HD)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about GPT-4V achieving 7.68/10, text-dense plot challenges, GPT-4V rating correlating with human evaluation, and cross-validation consistency are all supported by results in Sections 4-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'inclusion of image input is beneficial,' 'adding OCR tokens improved performance') are supported by controlled ablation comparisons where a single variable is changed (Figure 4, Figure 5)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Comprehensive Benchmark' for code generation from 'Scientific Plots,' but the benchmark only covers matplotlib and plotly libraries. The Limitations section acknowledges it 'may not fully capture the complexity and variety of other types of programming tasks' but the title and abstract overclaim comprehensiveness."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed performance differences. For example, performance gaps between open-source and closed-source models could be due to training data differences, model size, or instruction tuning quality, but these are not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly validates the GPT-4V rating proxy against human evaluation, showing moderate correlation (Pearson 0.479). They also demonstrate that traditional metrics (MSE, SSIM) fail as proxies (Table 4), distinguishing between proxy and actual visual similarity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'GPT-4V,' 'Claude-3-Opus,' 'Gemini-Pro,' 'DeepSeek-VL-7B.' No API snapshot dates or specific version identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts for code generation (direct and conditional asking), instruction generation, evaluation rating, and pair evaluation are provided in Appendix A.1-A.4."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or other API sampling settings for any of the 14 evaluated models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive direct prompts and produce single-turn code outputs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the three-stage filtering pipeline: generation filtering (extracting single code blocks from HTML), type filtering (excluding animations/widgets), and manual curation (3 criteria specified)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' paragraph follows Section 6 (Conclusion), discussing potential contamination and scope constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies two specific threats: (1) contamination from pre-training data potentially including benchmark plots, and (2) the dataset not capturing other programming task types beyond chart generation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section states the benchmark 'may not fully capture the complexity and variety of other types of programming tasks beyond chart generation,' explicitly bounding the scope to plotting tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset with plots, code, and instructions is publicly available on HuggingFace (https://huggingface.co/datasets/TencentARC/Plot2Code) under Apache-2.0 license."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the collection: crawling matplotlib/plotly gallery website links, extracting code blocks, then three-stage filtering."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the 46-person pairwise evaluation, participants are described as 'colleagues from the lab, who hold at least a bachelor's degree.' For the 100-person absolute rating study, no recruitment details are provided at all."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the full pipeline: web crawling → generation filtering (single code blocks) → type filtering (excluding animations, widgets, event handling) → manual curation (3 criteria), resulting in 368 examples."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Hong Kong, ARC Lab at Tencent PCG, and Shanghai Jiao Tong University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors from Tencent (which develops Mini-Gemini) evaluate their own models alongside competitors. No funding source is disclosed, making independence unassessable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 14 models evaluated, despite the benchmark data coming from publicly available matplotlib/plotly galleries."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions 'potential contamination, as pre-trained models might use the data as part of its training data' but provides no analysis of actual overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The matplotlib/plotly gallery examples are publicly available online and were almost certainly included in training data for models like GPT-4V. The paper acknowledges this risk but takes no steps to address it (no canary strings, no temporal analysis, no decontamination)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the 100-person absolute rating or the 46-person pairwise evaluation studies."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation studies."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "For the 46-person study, only 'colleagues from the lab, who hold at least a bachelor's degree' is stated. For the 100-person study, no demographics are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "Minimal criteria stated for 46-person study ('bachelor's degree'). No criteria stated for the 100-person study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a rating/evaluation task, not an experimental study comparing conditions among human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study on human participants. The pairwise evaluation does swap image positions to mitigate order bias (Section 4.4), but this is about evaluator bias, not participant blinding in an experimental sense."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on how many people were invited versus how many completed the evaluation tasks."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time reported for evaluating 14 models across 368 plots."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget or hardware specifications are mentioned for running the evaluation pipeline."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 3 appear to be single-run evaluations. No multi-seed analysis is performed for model outputs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per model is not stated. Results appear to be from a single evaluation pass."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Default settings appear to be used but are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations are reported (default, CoT, PS+) in Table 3 and Figure 4, not just the best one. No cherry-picking of the best configuration."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across 14 models and multiple metrics are reported without any correction for multiple testing (e.g., Bonferroni)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors from Tencent evaluate Mini-Gemini (their own model) alongside competitors without acknowledging potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes (2B to unknown GPT-4V size) are compared without discussing compute requirements or matching compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 validates the proposed metrics through hypothesis testing (Table 4), demonstrating that GPT-4V rating and text-match ratio significantly distinguish image quality groups, while traditional metrics (MSE, SSIM) do not."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are prompted directly with single-turn inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses matplotlib/plotly gallery examples that have been publicly available for years. Models trained after these galleries were published may have seen the exact code solutions. This temporal leakage is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "In the conditional asking setting, GPT-4 generated instructions may encode implementation-specific details from the ground truth code. This potential feature leakage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All plots come from official gallery examples. Models trained on web data likely encountered these exact examples. The non-independence between training data and test data is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is used (no canary strings, no membership inference, no n-gram overlap analysis)."
       }
     }
   },
   "claims": [
     {
       "claim": "Plot2Code poses considerable challenges, with GPT-4V achieving an overall score of 7.68/10 in conditional asking",
       "evidence": "Table 3 shows GPT-4V (PS+) achieves 7.83 maximum rating; baseline GPT-4V achieves 7.68. Even the best models score well below 10.",
       "supported": "strong"
     },
     {
       "claim": "Open-source models significantly lag behind closed-source models",
       "evidence": "Table 3: best open-source (Mini-Gemini-8x7B-HD) scores 6.08 GPT-4V rating and 58.4% pass rate, vs closed-source GPT-4V at 7.68 rating and 84.1% pass rate.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4V rating judgement correlates well with human evaluations",
       "evidence": "Table 5: Pearson r=0.479 (p=6.89×10⁻⁵⁴), Kendall's Tau=0.437 (p=8.68×10⁻⁴⁹), Spearman r=0.469 (p=1.57×10⁻⁵¹). Correlation stabilizes above 60 samples (Figure 8).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4V rating and text-match ratio significantly distinguish image quality groups, unlike MSE and SSIM",
       "evidence": "Table 4: GPT-4V rating p=1.22×10⁻¹¹, text-match ratio p=9.62×10⁻⁷, rejecting H0. MSE (p=0.22) and SSIM (p=0.21) fail to reject H0.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought and Plan-and-Solve prompt strategies do not yield clear advantages",
       "evidence": "Figure 4 and Table 3: GPT-4V (CoT) achieves 7.75 vs GPT-4V baseline 7.68 rating; pairwise evaluation shows CoT wins 40%, ties 49%, loses 11% vs baseline.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-validation among MLLMs shows high consistency in ratings with minimal bias",
       "evidence": "Figure 9: Cronbach's Alpha values of 0.77, 0.84, and 0.82 for GPT-4V, Claude, and Gemini generated samples respectively.",
       "supported": "moderate"
     },
     {
       "claim": "Higher resolution vision encoders improve MLLM performance on Plot2Code",
       "evidence": "Table 3 comparison of low-res vs HD variants: Mini-Gemini-8x7B goes from 3.76 to 3.87 (direct asking rating), and 5.74 to 6.08 (conditional asking rating).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Severe contamination risk",
       "detail": "Benchmark plots are scraped from matplotlib/plotly official galleries — publicly available web content almost certainly included in training data of all evaluated models. The paper acknowledges this but takes no steps to measure or mitigate it, undermining the validity of all benchmark scores."
     },
     {
       "flag": "Self-evaluation conflict",
       "detail": "Authors from Tencent ARC Lab evaluate Mini-Gemini (their own model family) alongside competitors. While results do not favor Mini-Gemini, the conflict is not disclosed or discussed."
     },
     {
       "flag": "No error bars on main results",
       "detail": "All model evaluations in Table 3 appear to be single-run results with no uncertainty quantification. LLM outputs are stochastic, so single-run results may not be representative."
     },
     {
       "flag": "Moderate human-AI correlation claimed as strong",
       "detail": "Pearson correlation of 0.479 between GPT-4V and human ratings is described as showing the metric 'correlates well' and is 'in general agreement,' but 0.479 is a moderate correlation explaining only ~23% of variance."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational LLM code generation benchmark that Plot2Code extends to multi-modal settings."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation evaluation, a key baseline for comparing code benchmarks."
     },
     {
       "title": "Design2code: How far are we from automating front-end engineering?",
       "authors": ["Chenglei Si", "Yanzhe Zhang", "Zhengyuan Yang", "Ruibo Liu", "Diyi Yang"],
       "year": 2024,
       "arxiv_id": "2403.03163",
       "relevance": "Evaluates MLLMs on UI-to-code generation, directly related to visual code generation benchmarking."
     },
     {
       "title": "MMCode: Evaluating multi-modal code large language models with visually rich programming problems",
       "authors": ["Kaixin Li", "Yuchen Tian", "Qisheng Hu", "Ziyang Luo", "Jing Ma"],
       "year": 2024,
       "arxiv_id": "2404.09486",
       "relevance": "Multi-modal code benchmark integrating images into code tasks, a direct comparator to Plot2Code."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM that represents the state of code generation capabilities."
     },
     {
       "title": "DeepSeek-VL: Towards real-world vision-language understanding",
       "authors": ["Haoyu Lu", "Wen Liu", "Bo Zhang"],
       "year": 2024,
       "arxiv_id": "2403.05525",
       "relevance": "Open-source multi-modal LLM evaluated in the benchmark, relevant to understanding MLLM capabilities."
     },
     {
       "title": "ChartQA: A benchmark for question answering about charts with visual and logical reasoning",
       "authors": ["Ahmed Masry", "Do Xuan Long", "Jia Qing Tan", "Shafiq Joty", "Enamul Hoque"],
       "year": 2022,
       "arxiv_id": "2203.10244",
       "relevance": "Chart understanding benchmark that complements Plot2Code's chart generation evaluation."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Code-specialized LLM relevant to understanding code generation capabilities in the agentic AI space."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2024,
       "relevance": "Establishes LLM-as-judge evaluation methodology that Plot2Code adapts for visual similarity rating."
     },
     {
       "title": "StarCoder: May the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Major open-source code LLM relevant to understanding code generation model development."
     },
     {
       "title": "ChartInstruct: Instruction tuning for chart comprehension and reasoning",
       "authors": ["Ahmed Masry", "Mehrad Shahmohammadi", "Md Rizwan Parvez"],
       "year": 2024,
       "arxiv_id": "2403.09028",
       "relevance": "Chart understanding instruction tuning benchmark, directly related to evaluating MLLM chart capabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "A benchmark dataset useful for researchers evaluating MLLMs, but not directly usable as a practitioner tool."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm expected patterns: GPT-4V leads, open-source lags, prompt strategies don't help much."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised by the work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial findings or challenges to established claims."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code on GitHub and dataset on HuggingFace allow researchers to reproduce evaluations."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates well-known models (GPT-4V, Claude-3, Gemini) and comes from Tencent/HKU."
     }
   }
 }