ai-research-survey

scan.json (26983B)

---

 {
   "paper": {
     "title": "ArtifactsBench: Bridging the Visual-Interactive Gap in LLM Code Generation Evaluation",
     "authors": [
       "Chenchen Zhang",
       "Yuhang Li",
       "Can Xu",
       "Jiaheng Liu",
       "Ao Liu",
       "Changzhi Zhou",
       "Ken Deng",
       "Dengpeng Wu",
       "Guanhua Huang",
       "Kejiao Li",
       "Qi Yi",
       "Ruibin Xiong",
       "Shihui Hu",
       "Yue Zhang",
       "Yuhao Jiang",
       "Zenan Xu",
       "Yuanxing Zhang",
       "Wiggin Zhou",
       "Chayse Zhou",
       "Fengzong Lian"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.04952",
     "doi": "10.48550/arXiv.2507.04952"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We open-source ArtifactsBench, including the benchmark, evaluation harness, and baseline results at https://artifactsbenchmark.github.io' (abstract and Section 1). A concrete URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset of 1,825 tasks is stated to be released at the project website. Section A.4 states 'We release dataset specs, checklist templates, evaluation scripts, and referee settings.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is described in the paper. The execution environment is described at a high level (Playwright, headless Chromium, 1024x768) but no dependency versions or environment recreation details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the evaluation pipeline is described conceptually, no step-by-step reproduction instructions (README with commands, scripts to run) are provided in the paper itself. The project website is referenced but the paper does not contain reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3, 6, and 7 are reported as point estimates without confidence intervals or error bars. No uncertainty quantification is provided for the benchmark scores."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., one model outperforms another, generalists outperform specialists) but provides no statistical significance tests. All comparisons are based on raw score differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (Cohen's d, etc.) are reported. Differences between models are presented as raw score gaps without formal quantification of effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 1,825 tasks and the human validation study uses 280 queries with 6 models, but no justification for why these specific sample sizes were chosen is provided. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any results. It is unclear whether experiments were run multiple times. Scores appear to be single-run evaluations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 30+ LLMs as baselines spanning multiple families (Qwen, DeepSeek, Gemma, GPT, Claude, Gemini, Hunyuan, Seed) as described in Section 4.1 and Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent models: Claude 4.0-Sonnet, GPT-5, Gemini-2.5-Pro, DeepSeek-R1-0528, Qwen3 family — all from 2025. The baselines are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4 and 5 present ablation studies on the evaluation pipeline: with/without images, single vs. multiple screenshots, with/without answer text, image vs. caption-based evaluation. These demonstrate the contribution of individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: overall score, vision-oriented vs. code-oriented scores, per-interactivity-level breakdowns (SV, MMD, HD, II), per-category breakdowns, Pair ACC for human agreement, and normalized Footrule consistency for ranking correlation with WebDev Arena."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 and A.3 describe a double-blind human study with 280 queries and 6 models judged by multiple front-end engineers. Pairwise agreement (Pair ACC) is computed between automated referee and human experts."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe a held-out test set. All 1,825 tasks appear to be used for evaluation without a train/dev/test split. The human validation uses a random subset of 280 queries from the same pool."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides breakdowns by interactivity level (SV, MMD, HD, II) and by task category (GAME, SVG, WEB, SI, MS). Figure 5 shows performance across difficulty tiers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses that 'All models score lowest on Intensive Interactive tasks and complex Management System scenarios.' The paper identifies systematic failure modes on intensive-interactive tasks. Section A.1 acknowledges that discrete screenshot sampling may miss complex interaction failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 shows that removing images or answers degrades performance. The paper reports that specialist models (coder, VL) underperform generalists, which is a somewhat negative finding for the specialist paradigm. Distillation on limited data is shown to yield only modest gains."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 94.4% ranking consistency with WebDev Arena (supported by Figure 7 and Section 4.4), up to 90.95% pairwise agreement with human experts (supported by Table 4), and evaluation of 30+ LLMs (supported by Table 3). All abstract claims have corresponding results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies (Tables 4, 5) make causal claims about the contribution of screenshots and code-aware judging through controlled single-variable manipulations: adding images improves Pair ACC, multiple screenshots further improve it, removing answers degrades quality. These ablations are adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract use broad language ('LLM Code Generation Evaluation') but the benchmark focuses specifically on interactive visual artifacts (web apps, games, SVGs). The claims about model capabilities are stated broadly ('generalist skills outperform specialist expertise') but tested only on this specific benchmark domain. Section A.1 acknowledges some limitations but does not bound the generalization of the main findings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its key findings. For example, it does not consider whether the MLLM referee's own biases could explain the high correlation with WebDev Arena, or whether the finding that generalists outperform specialists could be an artifact of task selection bias. The limitations section (A.1) discusses future work directions but not alternative explanations for observed results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Most models are specified with marketing names only: 'Gemini-2.5-Pro', 'Claude 4.0-Sonnet', 'GPT-4o'. Some have snapshot dates (e.g., 'GPT-4.1-2025-04-14', 'O1-2024-12-17', 'Claude Sonnet 4 (20250514)') but many do not have version identifiers or API snapshot dates. 'Hunyuan-Turbos-Preview' and 'Hunyuan-A13B' lack version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides full prompt texts in the appendix: query quality filtering prompt (Figure 12), category classification prompts (Figures 13, 14), checklist generation prompts (Figures 15, 16), and the final scoring prompt (Figure 17). These are the actual prompts used, not summaries."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states 'temperature and top-p tuned per official recommendations; max tokens sufficient to prevent truncation' but does not report the actual values. Specific temperature, top-p, and max_tokens settings are not provided."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is described in detail: sandbox execution with Playwright (headless Chromium), deterministic seeds, 1024x768 resolution, three-step screenshot capture, dual-referee scoring, 10-dimension checklists. The pipeline architecture is shown in Figure 3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the full data construction pipeline including sourcing and filtering, de-duplication (MinHash + semantic similarity + perceptual hashing), prompt rewriting, difficulty calibration, checklist generation, solvability validation, and execution harness setup. Figure 2 provides a visual overview."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 contains a dedicated 'Limitations and Future Work' section discussing limitations of the screenshot-based evaluation approach and the single-turn evaluation scope."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section A.1 discusses specific limitations: (1) discrete screenshot sampling may miss fluid/complex interactions in long-horizon tasks, (2) the benchmark evaluates single-turn generation only, not iterative refinement or agentic development. These are specific to this study's design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section A.1 explicitly states what the benchmark does NOT evaluate: it 'does not assess an LLM's capability to function as an autonomous agent that can iteratively refine an artifact based on feedback, debug its own code in response to errors, or plan and execute a multi-step development process.' The section also notes that screenshot sampling 'may not fully capture the fluidity, correctness, and robustness of the entire interactive experience.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that the benchmark data, evaluation harness, and baseline results are open-sourced at https://artifactsbenchmark.github.io. Section A.4 confirms 'We release dataset specs, checklist templates, evaluation scripts, and referee settings.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the data collection procedure in detail: aggregating candidates from expert showcases, open SVG/web-snippet datasets, web case studies, and LLM visual-to-query from screenshots, followed by automated filtering, de-duplication, prompt rewriting, and solvability validation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human validation study (280 queries, Section A.3), participants are described only as 'multiple engineers with extensive front-end development experience.' No details about how many annotators participated, how they were recruited, or whether they were internal Tencent employees. This is relevant because internal employees could have conflicts of interest."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 and Figure 2 document the full pipeline from sourcing through filtering, de-duplication, rewriting, calibration, solvability validation, to final benchmark construction. The realized difficulty split (559/611/655) is stated. The checklist calibration process with Cohen's kappa >= 0.8 is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. There is no acknowledgments section listing grants or funding. The work is from Tencent but no explicit funding statement is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed in Section 7 (Contributions and Acknowledgements). Most authors are from Tencent, with some from NJU and PKU. One contributor is listed as 'Independent.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The paper is produced by Tencent's Hunyuan team and evaluates Hunyuan models (Hunyuan-Turbos-Preview, Hunyuan-A13B) within the benchmark. Tencent has a direct financial interest in their models performing well. The funder/employer is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Given that the authors work at Tencent and evaluate Tencent's Hunyuan models, a financial interest declaration would be appropriate but is absent."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No model training data cutoff dates are stated. The paper evaluates 30+ LLMs on the benchmark without discussing when their training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes contamination control: 'Two-stage filtering: (i) MinHash + semantic similarity over prompts, checklists, and normalized DOM/CSS/JS; (ii) screenshot perceptual hashing to catch visually near-identical artifacts.' The dataset is stated to be self-constructed with contamination control measures."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly addresses contamination under 'De-duplication & contamination control' with two-stage filtering including MinHash, semantic similarity, and perceptual hashing. Flagged items are 're-authored or discarded.' The benchmark is newly constructed, reducing contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human validation study (280 queries, 6 models) is not pre-registered. No link to a pre-registration is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation study involving front-end engineers as annotators."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluators are described only as 'multiple engineers with extensive front-end development experience.' No demographics (number of annotators, years of experience, etc.) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for human evaluators are stated. The only qualifier is 'extensive front-end development experience.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human study is a validation/annotation task (rating artifact quality), not an experimental study comparing treatment vs. control conditions. Randomization of condition assignment is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section A.3 states: 'The process follows a double-blind protocol: annotators remain unaware of the MLLM's scores, and the samples appear in randomized order to mitigate bias.'"
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about how many human evaluators started vs. finished the study, or whether any annotations were excluded."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, or tokens consumed are reported for the evaluation pipeline despite it requiring running 30+ LLMs on 1,825 tasks plus dual-referee scoring."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is stated. The paper does not quantify the cost of running the full evaluation pipeline."
       }
     }
   },
   "claims": [
     {
       "claim": "ArtifactsBench achieves 94.4% normalized Footrule ranking consistency with WebDev Arena.",
       "evidence": "Section 4.4 and Figure 7 show the ranking correlation between ArtifactsBench (judged by Gemini-2.5-Pro) and WebDev Arena, reporting 94.4% consistency.",
       "supported": "moderate"
     },
     {
       "claim": "The automated MLLM referee achieves up to 90.95% pairwise agreement with human experts.",
       "evidence": "Table 4 shows Gemini-2.5-Pro with multiple screenshots achieves 90.95% Pair ACC. The human study uses 280 queries and 6 models judged by multiple front-end engineers (Section A.3).",
       "supported": "moderate"
     },
     {
       "claim": "Instruction-tuned generalist models consistently outperform specialist coder/VL models on visual artifact generation.",
       "evidence": "Section 4.2 and Table 3 show Qwen-2.5-Instruct surpasses both Qwen-2.5-coder and Qwen2.5-VL-72B across the evaluated tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Performance scales with model size and deliberation time.",
       "evidence": "Section 4.2 describes scaling trends within the Qwen2.5 and Qwen3 families in Table 3, and notes that 'slow thinkers' score higher.",
       "supported": "moderate"
     },
     {
       "claim": "Multiple screenshots better capture dynamics than a single screenshot for MLLM-as-Judge evaluation.",
       "evidence": "Table 4 shows consistent improvement going from 'w/o img' to 'w/ img' to 'w/ imgs' across all three referee models (e.g., Gemini-2.5-Pro: 79.06% → 87.10% → 90.95%).",
       "supported": "strong"
     },
     {
       "claim": "Proprietary multimodal models show a clear advantage over open-source models.",
       "evidence": "Table 3 shows Gemini-2.5-Pro and Claude 4.0-Sonnet at the top of rankings. Table 6 shows GPT-5 leading at 72.55 average score.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ArtifactsBench introduces a 1,825-task benchmark for evaluating LLM-generated interactive visual artifacts across nine domains (games, web apps, SVG, simulations, etc.), using an automated MLLM-as-Judge evaluation pipeline with dual referees. The benchmark achieves 94.4% ranking consistency with WebDev Arena and up to 90.95% pairwise agreement with human experts. Key findings include that proprietary models (Gemini-2.5-Pro, Claude 4.0-Sonnet) lead the rankings, performance scales with model size, and instruction-tuned generalist models outperform specialist coder or vision-language models on this task.",
   "red_flags": [
     {
       "flag": "Conflict of interest: Tencent evaluating own models",
       "detail": "The benchmark is created by Tencent's Hunyuan team and evaluates Tencent's Hunyuan models (Hunyuan-Turbos-Preview, Hunyuan-A13B). Hunyuan-Turbos-Preview ranks highly among closed-source models (50.97 AVG in Table 3), close to Claude 3.7-Sonnet. The checklist generation itself uses 'Hunyuan-Turbos' (Section 3.1), meaning the evaluation rubrics were generated by the same company's model. No conflict of interest statement is provided."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "Results across 30+ models on 1,825 tasks are reported as point estimates with no confidence intervals, error bars, significance tests, or variance across runs. It is unclear whether results are from single runs or averaged."
     },
     {
       "flag": "Hyperparameters not fully specified",
       "detail": "The paper states 'temperature and top-p tuned per official recommendations' without providing the actual values used for each model. Different temperature settings could meaningfully affect scores, and the vague description prevents reproduction."
     },
     {
       "flag": "Human study details insufficient",
       "detail": "The human validation study (280 queries, 6 models) does not report the number of annotators, their demographics, how they were recruited, or whether they are Tencent employees. If annotators are internal, this compounds the conflict-of-interest concern."
     },
     {
       "flag": "Circular evaluation risk",
       "detail": "Hunyuan-Turbos is used to generate the 10-dimension checklists (Section 3.1), and Hunyuan models are then evaluated using those checklists. While human refinement is mentioned, the initial checklist generation by the same company's model creates a potential circularity where the evaluation criteria may be biased toward what Hunyuan models produce."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational LLM code generation benchmark (HumanEval) that ArtifactsBench aims to extend beyond algorithmic correctness."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Key benchmark for LLM-based software engineering that ArtifactsBench complements by focusing on visual/interactive artifacts."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-bench and chatbot arena",
       "authors": ["Lianmin Zheng"],
       "year": 2023,
       "relevance": "Establishes the LLM-as-Judge paradigm that ArtifactsBench builds upon for automated evaluation."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of the top-performing open-source models evaluated on ArtifactsBench, relevant to understanding LLM code generation capabilities."
     },
     {
       "title": "WebGen-Bench: Evaluating LLMs on generating interactive and functional websites from scratch",
       "authors": ["Zimu Lu"],
       "year": 2025,
       "arxiv_id": "2505.03733",
       "relevance": "Directly related benchmark for web page generation that ArtifactsBench compares against."
     },
     {
       "title": "FullFront: Benchmarking MLLMs across the full front-end engineering workflow",
       "authors": ["Haoyu Sun"],
       "year": 2025,
       "arxiv_id": "2505.17399",
       "relevance": "Related benchmark for front-end web development evaluation, comparing comprehension and generation capabilities."
     },
     {
       "title": "Web-Bench: A LLM code benchmark based on web standards and frameworks",
       "authors": ["Kai Xu"],
       "year": 2025,
       "arxiv_id": "2505.07473",
       "relevance": "Related benchmark focusing on DOM alignment and web task automation that ArtifactsBench compares against (69.4% vs 94.4% WebDev Arena consistency)."
     },
     {
       "title": "MLLM-bench: evaluating multimodal LLMs with per-sample criteria",
       "authors": ["Wentao Ge"],
       "year": 2023,
       "arxiv_id": "2311.13951",
       "relevance": "Related work on using per-sample evaluation criteria for multimodal LLM assessment."
     },
     {
       "title": "WebArena: A realistic web environment for building autonomous agents",
       "authors": ["Shuyan Zhou"],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Influential web agent benchmark; WebDev Arena is used as the gold standard for human preference alignment in ArtifactsBench."
     },
     {
       "title": "Qwen3 technical report",
       "authors": ["An Yang"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Technical report for one of the major model families extensively evaluated on ArtifactsBench."
     },
     {
       "title": "CodeCriticBench: A holistic code critique benchmark for large language models",
       "authors": ["Alexander Zhang"],
       "year": 2025,
       "arxiv_id": "2502.16614",
       "relevance": "Related benchmark for evaluating LLM code critique capabilities, relevant to code quality evaluation methodology."
     }
   ]
 }