ai-research-survey

scan.json (29370B)

---

 {
   "paper": {
     "title": "Drawing Pandas: A Benchmark for LLMs in Generating Plotting Code",
     "authors": [
       "Timur Galimzyanov",
       "Sergey Titov",
       "Yaroslav Golubev",
       "Egor Bogomolov"
     ],
     "year": 2024,
     "venue": "IEEE Working Conference on Mining Software Repositories",
     "arxiv_id": "2412.02764",
     "doi": "10.1109/MSR66628.2025.00083"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PandasPlotBench is a 175-task human-curated benchmark for evaluating LLMs on generating plotting code from Pandas DataFrames. GPT-4o and Claude 3.5 Sonnet lead with ~89 task-based mean scores, while significant task compression (down to single sentences) minimally affects performance. Plotly generation is notably harder with 22% incorrect code vs 1.8% for Matplotlib. Large open-source models (Llama 3.1 70B, 405B) approach proprietary model performance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/JetBrains-Research/PandasPlotBench. The paper states 'we also open-source our implementation of the baseline methods and the evaluation code, available on our GitHub page.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace: https://huggingface.co/datasets/JetBrains-Research/PandasPlotBench. Explicitly stated in abstract and Section II-A."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification in the paper. The paper mentions using a Jupyter Notebook environment but provides no library versions or dependency details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides no step-by-step reproduction instructions. It links to GitHub and supplementary materials but does not include a 'Reproducing Results' section or specific commands to run the benchmark."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Despite running each experiment 5 times, the paper reports only mean scores and ratios in Tables I-III. No confidence intervals, error bars, or ± notation are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4o and Claude 3.5 Sonnet 'share the highest scores' and that task compression has 'minimal effect' without any statistical significance tests. Comparative claims are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as raw scores (0-100 scale) in tables without explicit effect size calculations, percentage improvements, or baseline-relative measures. The reader can compare numbers but no effect sizes are computed or discussed."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 175 data points reduced from 501 through filtering, but no justification is given for why 175 is sufficient for the claims being made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we ran the benchmark five times for each factor to ensure the robustness of our results' but reports only mean scores. No standard deviations, IQR, or any spread measure is reported across the 5 runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models compared: GPT-4o, Claude 3.5 Sonnet, Claude 3 Opus, Claude 3 Haiku, Gemini 1.5 Pro, and Llama 3.1/3.2 variants (Table I). A 'no task' control condition is also included (Table III)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All tested models are contemporary (2024 vintage): GPT-4o, Claude 3.5 Sonnet, Gemini 1.5 Pro, Llama 3.1/3.2 family. These represent state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies prompt components: DataFrame description format (Section III), task length with 5 levels from full to none (Table III), and target plotting library (Table II). These serve as ablations of the prompt design."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics used: visual scoring (0-100 comparison to ground truth), task-based scoring (0-100 adherence to task description), and incorrect code ratio. Ratio of 'good' scores (≥75) also reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section II-C: 'The first author of the paper, with 5 years of experience in Python and data visualization, went through all the data points and manually scored them judging the adherence of the resulting plot to the task description.' Human-LLM score correlation analyzed (Pearson 0.85 for task-based)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses the same 175 tasks for both selecting the best DataFrame description method ('we experimented with different DataFrame descriptions and found that the head(5) method... delivered the best performance') and reporting final results. No dev/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by model (Table I), by plotting library (Table II), and by task length (Table III). These provide meaningful category-level insights into where performance varies."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B: 'We manually reviewed these cases and found that most errors were due to the incorrect usage of the Plotly library API by the language model.' Plotly failure analysis and small Llama model failures discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: Plotly's 22% incorrect code rate, small Llama models 'often fail to produce functional code' (Table I), the 'no task' condition producing only 23% good scores, and CodeBERT Score found inadequate and abandoned (Section II-C)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: task compression having 'minimal effect' is shown in Table III (scores drop from 89 to 85 for single sentence), Plotly challenges shown in Table II (22% incorrect code), and benchmark/model comparisons in Table I."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (e.g., 'shortening of tasks has a minimal effect on plotting capabilities') are supported by controlled experimental manipulation — varying one factor at a time while holding others constant. The task length experiment varies only the task prompt while keeping data descriptions and instructions constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded: 'Our benchmark focuses on generating code for visualizing tabular data—such as a Pandas DataFrame.' Limitations explicitly bound the scope to Python, 175 Matplotlib-derived tasks, and concise DataFrames. However, the benchmark name 'PandasPlotBench' appropriately scopes expectations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for key results. For example, Plotly's poor performance could be partly due to the benchmark being derived from Matplotlib gallery tasks (structurally favoring Matplotlib-style plots), but this is mentioned only as a limitation, not as an alternative explanation for the observed result."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims mostly match the granularity of its measurements — it reports scores on its specific benchmark rather than making broad claims about 'coding ability.' The distinction between visual scores and task-based scores (Pearson 0.58 correlation) is explicitly discussed, and both are validated against human judgment."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'GPT-4o', 'Claude 3.5 Sonnet', 'Gemini 1.5 Pro' etc. without snapshot dates or API versions. Per the schema, marketing names without snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'For the full list of instruction prompts used for processing the data, generating the results, and scoring... please refer to the supplementary materials' with a link to GitHub repository containing the actual prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or any other API hyperparameters are reported for any of the tested models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive prompts directly and return code in a single turn."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II-A documents the full pipeline with counts at each stage: 501 gallery scripts → 307 valid after execution → 201 after GPT-4 splitting and manual review → 175 after final validation. Each step's criteria are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Limitations and Future Work' is a dedicated section discussing multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section V discusses specific threats: potential bias toward Matplotlib code in training data, possible bias toward OpenAI models from using GPT-4 for task generation, limited dataset size of 175, concise DataFrames not representative of real-world data, and single-person manual scoring."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V explicitly states scope boundaries: limited to Python ('we limited ourselves to Python code'), only Matplotlib-derived tasks ('Our dataset is currently limited to 175 data points originating from the Matplotlib gallery'), and concise DataFrames ('the DataFrames in the dataset are mostly concise, containing only the data required for plotting')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset including CSV files, tasks, and ground truth plots is available on HuggingFace. Evaluation code and supplementary materials available on GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II-A describes the complete data collection procedure: sourcing from Matplotlib gallery, script execution in Jupyter, GPT-4 splitting, manual verification at each step, and final validation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. Data sourced from the public Matplotlib gallery. The single human evaluator is the first author, not a recruited participant."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented with counts at each stage: 501 → 307 (after execution filtering) → 201 (after GPT-4 splitting + manual review) → 175 (after final validation). Each transformation step is described in Section II-A."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or acknowledgments mentioning grants, sponsors, or funding agencies. Authors are from JetBrains Research (a corporate lab), but funding is not explicitly disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: JetBrains Research and Delft University of Technology. JetBrains makes development tools including AI-powered features, which is relevant context."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "While JetBrains makes development tools, the benchmark does not evaluate any JetBrains products. The results do not favor or disfavor JetBrains. The funder has no direct stake in which LLM scores highest."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure section in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (GPT-4o, Claude, Gemini, Llama)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V discusses potential overlap: 'models may still exhibit bias towards publicly available ground-truth Matplotlib code. However, this code has been modified, and solutions using other libraries remain unseen.' Also: 'our tasks are less prone to models memorizing training data.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly designs for contamination resistance: 'Our benchmark aims to be free from data leakage, as the tasks and data files are synthetically generated.' Section V acknowledges the Matplotlib gallery source code may be in training data but argues the tasks and solutions are novel."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The benchmark evaluation involves only LLM outputs and automated/author scoring."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The single human scorer is the first author evaluating LLM outputs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I reports wall-clock time per item (s/item) for each model, ranging from 0.3s (Llama 3.2 1B) to 14.0s (Claude 3 Opus). However, no API costs or token counts are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. No GPU hours, total API costs, or hardware specifications reported beyond per-item latency."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper ran each experiment 5 times but reports only mean scores. No seed sensitivity analysis or per-run results are provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section III: 'For all experiments, we ran the benchmark five times for each factor to ensure the robustness of our results.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper experimented with different DataFrame description formats but does not report how many configurations were tried or the search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The best DataFrame description method (head(5) + column names/types) was selected by testing on the same 175-task dataset used for final reporting. No validation/test split was used, and not all configurations tried are reported."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 10 models and multiple experimental conditions without any statistical significance tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate all models on it without discussing potential author-evaluation bias or having independent evaluators."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While per-item latency is reported alongside scores in Table I, there is no analysis of performance as a function of compute budget or comparison at matched compute levels."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its benchmark (175 Matplotlib-derived tasks with concise DataFrames) actually measures the claimed capability of 'effectiveness as assistants in visual data exploration.' The gap between controlled benchmark tasks and real-world data visualization is not analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. Models are tested directly via API with the same prompt structure."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal aspects — when the Matplotlib gallery code was published vs. when model training data was collected. The argument focuses on synthetic generation but ignores the timeline."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through context. The DataFrame descriptions and task prompts could contain information that makes generation trivially easy."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and test examples share structural similarities. The benchmark tasks derive from the public Matplotlib gallery, which models may have seen variants of during training."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method applied (no canary strings, membership inference, n-gram overlap analysis). The paper argues contamination resistance by design but does not test for it."
       }
     }
   },
   "claims": [
     {
       "claim": "Task compression (shortening instructions to a single sentence) has minimal effect on LLM plotting capabilities.",
       "evidence": "Table III shows task-based mean score drops from 89 (full task) to 85 (single sentence), and visual score from 75 to 71. Good score ratio drops from 0.91 to 0.85 (task-based).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o and Claude 3.5 Sonnet are the highest-scoring models for plotting code generation.",
       "evidence": "Table I: GPT-4o achieves 89 task-based mean, Claude 3.5 Sonnet achieves 88. Both outperform other models.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs struggle significantly with Plotly compared to Matplotlib and Seaborn.",
       "evidence": "Table II: Plotly has 22% incorrect code vs 1.8% for Matplotlib, and task-based mean score of 68 vs 89 for Matplotlib.",
       "supported": "strong"
     },
     {
       "claim": "Large open-source Llama models approach proprietary model performance.",
       "evidence": "Table I: Llama 3.1 405B achieves 86 task-based mean vs 89 for GPT-4o; Llama 3.1 70B achieves 82.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o judge's task-based scoring correlates strongly with human evaluation (Pearson r=0.85).",
       "evidence": "Section II-C reports Pearson correlation of 0.85 between human scores and task-based scores, validated by first author scoring all data points.",
       "supported": "weak"
     },
     {
       "claim": "The benchmark is free from data leakage by virtue of being synthetically generated.",
       "evidence": "Section I states tasks and data files are synthetically generated. Section V argues there is 'no direct link between the answers to benchmark tasks and the raw repository data.'",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Single human evaluator",
       "detail": "Human validation was performed by only the first author. No inter-rater reliability measured. The paper acknowledges this: 'the manual scoring of the data points was carried out only by the first author, as a general sanity check.' This is insufficient to validate the benchmark's scoring."
     },
     {
       "flag": "GPT-4o as both contestant and judge",
       "detail": "GPT-4o is the highest-scoring model in Table I and is also used as the judge (GPT-4o Judge) for scoring all models. This creates a potential self-preference bias where GPT-4o may rate its own outputs more favorably."
     },
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "Each experiment was run 5 times but only mean scores are reported with no standard deviations, confidence intervals, or any spread measure. The reader cannot assess whether observed differences are meaningful or within noise."
     },
     {
       "flag": "Configuration selection on test data",
       "detail": "The best DataFrame description format was selected by testing on the same 175 tasks used for final reporting. No held-out validation set was used, risking optimistic results."
     }
   ],
   "cited_papers": [
     {
       "title": "MatPlotAgent: Method and evaluation for LLM-based agentic scientific data visualization",
       "authors": ["Z. Yang", "Z. Zhou", "S. Wang"],
       "year": 2024,
       "arxiv_id": "2402.11453",
       "relevance": "Directly related benchmark for LLM-based data visualization with agentic methods; primary comparison point for this work."
     },
     {
       "title": "Plot2Code: A comprehensive benchmark for evaluating multi-modal large language models in code generation from scientific plots",
       "authors": ["C. Wu", "Y. Ge", "Q. Guo"],
       "year": 2024,
       "arxiv_id": "2405.07990",
       "relevance": "Benchmark for multi-modal LLM code generation from plots; closely related evaluation of LLM plotting capabilities."
     },
     {
       "title": "ChartMimic: Evaluating LMM's cross-modal reasoning capability via chart-to-code generation",
       "authors": ["C. Shi", "C. Yang", "Y. Liu"],
       "year": 2024,
       "arxiv_id": "2406.09961",
       "relevance": "1,000 human-curated examples for chart-to-code generation; directly comparable benchmark for LLM visualization capabilities."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Y. Lai", "C. Li", "Y. Wang"],
       "year": 2023,
       "relevance": "Data science code generation benchmark including Matplotlib tasks; establishes baseline for LLM code generation evaluation."
     },
     {
       "title": "nvBench: A large-scale synthesized dataset for cross-domain natural language to visualization task",
       "authors": ["Y. Luo", "J. Tang", "G. Li"],
       "year": 2021,
       "arxiv_id": "2112.12926",
       "relevance": "Large-scale NL-to-visualization benchmark with 7,274 visualizations; related evaluation of automated visualization generation."
     },
     {
       "title": "Leveraging large language models for data analysis automation",
       "authors": ["J. A. Jansen", "A. Manukyan", "N. Al Khoury", "A. Akalin"],
       "year": 2025,
       "relevance": "Studies LLM automation of data analysis pipelines including visualization; directly relevant to LLM code generation capabilities."
     },
     {
       "title": "LLMs for science: Usage for code generation and data analysis",
       "authors": ["M. Nejjar", "L. Zacharias", "F. Stiehle", "I. Weber"],
       "year": 2023,
       "relevance": "Evaluates LLM usage for scientific code generation and data analysis; relevant to understanding LLM coding capabilities."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["A. Dubey", "A. Jauhri", "A. Pandey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Open-source LLM family evaluated in this benchmark; important for understanding open vs proprietary model performance gaps."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["J. Achiam", "S. Adler", "S. Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, used for dataset construction; central model in LLM capability evaluation."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["P. Vaithilingam", "T. Zhang", "E. L. Glassman"],
       "year": 2022,
       "relevance": "User study evaluating LLM code generation tools; provides context on challenges in producing fully executable code."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained code model whose CodeBERT Score metric was evaluated and rejected in this benchmark's development."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners using LLMs for data visualization can use this benchmark to evaluate tools; the task compression finding is directly useful for UI design."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Plotly underperformance is expected given training data distribution; the minimal effect of task compression is mildly interesting but not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict in the findings."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and dataset publicly available on GitHub and HuggingFace; researchers can run the benchmark but it's not pip-installable."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "JetBrains is well-known in developer tooling but not a top-tier AI research lab; evaluates recognizable models (GPT-4o, Claude)."
     }
   }
 }