ai-research-survey

scan.json (22454B)

---

 {
   "paper": {
     "title": "Modular Layout Synthesis (MLS): Front-end Code via Structure Normalization and Constrained Generation",
     "authors": ["Chong Liu", "Ming Zhang", "Fei Li", "Hao Zhou", "Xiaoshuang Chen", "Ye Yuan"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2512.18996",
     "doi": "10.48550/arXiv.2512.18996"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MLS proposes a three-stage UI-to-Code framework (visual-to-structure, blueprint compression with motif harvesting, constraint-based multi-framework generation) that separates layout parsing, reuse extraction, and framework-specific synthesis. The paper claims significant improvements over baselines in code reusability and structural integrity across React/Vue/Angular. However, the authors explicitly state that ALL experimental results are synthetic (randomly generated), making every empirical claim unverifiable.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on publicly available benchmarks: Design2Code (Si et al. 2025), Vision2UI (Gui et al. 2024), and WebSight (Laurençon et al. 2024). However, the 300-case multi-framework subset is described as 'manually converted' and synthetic, and is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, scripts, or README are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MLS 'significantly outperforms' baselines but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-3 report absolute metric values for all methods, allowing effect size computation (e.g., CLIP 0.844 vs 0.821 for next-best baseline, Reuse@K 0.412 vs 0.275)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are stated (484, 2000, 300) but no justification is given for why these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baselines are compared: Direct MLLM Prompting, DCGen, LayoutCoder, WebSight-Sightseer, and WebVIA-UI2Code (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent 2024-2025 work: DCGen (Wan et al. 2024), LayoutCoder (Wu et al. 2025), WebVIA-UI2Code (Xu et al. 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows ablations removing component mining, type inference, and constrained decoding. However, all results are synthetic."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics are used: CLIP (visual similarity), TED (tree edit distance), Reuse@K, DupRate, and TypeCheck."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated. For a UI code generation system, human judgment of code quality and maintainability would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test/validation splits or held-out evaluation. The paper mentions using existing test sets but does not clarify separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks down results by framework (React, Vue, Angular). Table 3 provides per-ablation breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.7 mentions 'qualitative analysis' and 'typical cases' but only describes successes. No failure cases are shown or discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows MLS improving over baselines. No negative results or failed approaches are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims MLS 'significantly outperforms existing baselines' but the Section 4 disclaimer states 'All numeric results in this section are synthetic (randomly generated but internally consistent).' Synthetic results cannot support empirical claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation study (Table 3) makes causal claims about component contributions (e.g., 'component mining mainly boosts reuse'), but results are synthetic and cannot justify causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion claim broad superiority ('superior code reusability and structural integrity across multiple frameworks') but scope is not bounded. The title and framing suggest general applicability beyond the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for results are discussed. No threats to validity or confounding factors are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures automated metrics (CLIP, TED, Reuse@K, DupRate, TypeCheck) but frames these as demonstrating 'production-ready code' and 'superior code reusability.' No discussion of the gap between these proxy metrics and actual developer productivity or code maintainability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "No specific model versions are mentioned. The paper references using an 'LLM' in Module C and a 'visual-semantic encoder' in Module A without naming or versioning either."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Module C uses an LLM with a 'constraint-based generation protocol' but no actual prompts or system instructions are provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters are mentioned symbolically (λg, λr, ϵ, dmax, amin, η, γ) but no actual values are provided. No LLM temperature or sampling settings are stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-module pipeline (Module A: visual-to-structure, Module B: blueprint compression, Module C: constrained generation) is described in detail in Section 3, including the constrained decoding protocol."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing steps are documented. The paper references datasets but does not describe how they were prepared for the experiments."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section. The paper has only a Broader Impact section (Section 5) and a brief Conclusion (Section 6)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what settings or scenarios are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data or experimental outputs are available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The 300-case multi-framework subset is described only as 'manually converted' with no detail on the conversion process. The synthetic disclaimer undermines all data descriptions."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented from raw inputs to final evaluation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Nanjing University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosures statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses an LLM in Module C but does not state any training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLM's training data includes any of the benchmark examples."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Design2Code and other benchmarks may be in LLM training data. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The system involves an LLM and a vision encoder but no inference cost, latency, or token consumption is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, GPU hours, or training time is mentioned."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. All results are single point estimates."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated anywhere."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Multiple hyperparameters are introduced (λg, λr, η, γ, dmax, amin) but no search budget or tuning methodology is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the reported configuration was selected from alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 5 baselines and 5 metrics with no correction applied (no statistical tests at all)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system against their own implementations/runs of baselines with no acknowledgment of potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MLS adds multiple processing stages (vision encoder, blueprint compression, constrained LLM decoding) over simpler baselines, but compute differences are not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the metrics (CLIP, TED, Reuse@K, DupRate, TypeCheck) actually capture code quality or real-world usability."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MLS uses a complex multi-stage scaffold while baselines may use simpler pipelines. This confound between the scaffold and the method's contributions is not addressed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The benchmarks (Design2Code, Vision2UI) may have been in the LLM's training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used or mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "MLS significantly outperforms existing baselines on Design2Code, achieving CLIP 0.844, TED 0.233, Reuse@K 0.412, DupRate 0.241, TypeCheck 86.9%.",
       "evidence": "Table 1, Section 4.4. However, the paper explicitly states: 'All numeric results in this section are synthetic (randomly generated but internally consistent)' (Section 4 disclaimer).",
       "supported": "unsupported"
     },
     {
       "claim": "MLS achieves superior multi-framework portability (avg 0.76) compared to baselines (0.58-0.63).",
       "evidence": "Table 2, Section 4.5. Same synthetic disclaimer applies.",
       "supported": "unsupported"
     },
     {
       "claim": "Component mining is the key driver of reuse improvement, while constrained decoding and type inference primarily improve compilation success.",
       "evidence": "Table 3 ablation study, Section 4.6. Same synthetic disclaimer applies.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "All results explicitly synthetic",
       "detail": "Section 4 contains a disclaimer: 'All numeric results in this section are synthetic (randomly generated but internally consistent) to provide a complete paper draft.' This means every empirical claim in the paper is unsupported by actual experiments. The abstract and conclusion do not mention this, presenting synthetic results as if they demonstrate real performance."
     },
     {
       "flag": "Misleading abstract and conclusion",
       "detail": "The abstract states 'Evaluations show that MLS significantly outperforms existing baselines' and the conclusion states results 'improve code reusability, portability across React/Vue/Angular, and structural integrity.' Neither mentions that results are synthetic, creating a false impression of validated empirical findings."
     },
     {
       "flag": "No model or implementation details",
       "detail": "Despite describing a concrete system with three modules, the paper never names the LLM used in Module C, the architecture of the visual-semantic encoder in Module A, or any concrete hyperparameter values. It is unclear whether MLS was actually implemented."
     },
     {
       "flag": "No limitations discussion",
       "detail": "The paper has no limitations section, no threats to validity, and no discussion of when or why the approach might fail."
     },
     {
       "flag": "Results appear too clean",
       "detail": "MLS beats every baseline on every metric (Tables 1-3) with no variance reported. This is consistent with the synthetic nature of the results but would be suspicious even without the disclaimer."
     }
   ],
   "cited_papers": [
     {
       "title": "Design2Code: Benchmarking Multimodal Code Generation for Automated Front-end Engineering",
       "authors": ["Chenglei Si", "Yanzhe Zhang", "Ryan Li", "Zhengyuan Yang", "Ruibo Liu", "Diyi Yang"],
       "year": 2025,
       "relevance": "Major benchmark for UI-to-code generation, directly relevant to evaluating LLM code generation capabilities."
     },
     {
       "title": "Automatically Generating UI Code from Screenshot: A Divide-and-Conquer-Based Approach",
       "authors": ["Yuxuan Wan", "Chaozheng Wang", "Yi Dong", "Wenxuan Wang", "Shuqing Li", "Yintong Huo", "Michael R. Lyu"],
       "year": 2024,
       "arxiv_id": "2406.16386",
       "relevance": "Divide-and-conquer prompting pipeline for UI-to-code, a key baseline for LLM-based code generation approaches."
     },
     {
       "title": "Unlocking the Conversion of Web Screenshots into HTML Code with the WebSight Dataset",
       "authors": ["Hugo Laurençon", "Léo Tronchon", "Victor Sanh"],
       "year": 2024,
       "arxiv_id": "2403.09029",
       "relevance": "Large-scale synthetic dataset and finetuned VLM for screenshot-to-HTML, relevant to LLM-based code generation evaluation."
     },
     {
       "title": "MLLM-based UI2Code Automation Guided by UI Layout Information",
       "authors": ["Fan Wu", "Cuiyun Gao", "Shuqing Li", "Xinjie Wen", "Qing Liao"],
       "year": 2025,
       "arxiv_id": "2506.10376",
       "relevance": "Layout-guided approach to LLM-based UI code generation, directly comparable baseline."
     },
     {
       "title": "WebVIA: A Web-based Vision-Language Agentic Framework for Interactive and Verifiable UI-to-Code Generation",
       "authors": ["Mingde Xu", "Zhen Yang", "Wenyi Hong"],
       "year": 2025,
       "arxiv_id": "2511.06251",
       "relevance": "Agentic framework for UI-to-code with interactive verification, relevant to agentic AI code generation."
     },
     {
       "title": "Type-Constrained Code Generation with Language Models",
       "authors": ["Niels Mündler", "Jingxuan He", "Hao Wang", "Koushik Sen", "Dawn Song", "Martin Vechev"],
       "year": 2025,
       "arxiv_id": "2504.09246",
       "relevance": "Type-constrained decoding for LLM code generation, directly relevant to constrained code synthesis methods."
     },
     {
       "title": "Grammar-Constrained Decoding for Structured NLP Tasks Without Finetuning",
       "authors": ["Saibo Geng", "Martin Josifoski", "Maxime Peyrard", "Robert West"],
       "year": 2023,
       "relevance": "Grammar-constrained decoding for LLMs, foundational work for the constrained generation approach used in MLS."
     },
     {
       "title": "Grammar-Aligned Decoding",
       "authors": ["Kyunghyun Park"],
       "year": 2024,
       "relevance": "Studies quality distortions from grammar-constrained decoding in LLMs, relevant to understanding constrained generation tradeoffs."
     },
     {
       "title": "pix2code: Generating Code from a Graphical User Interface Screenshot",
       "authors": ["Tony Beltramelli"],
       "year": 2018,
       "relevance": "Seminal work on end-to-end screenshot-to-code generation."
     },
     {
       "title": "Generating Structured Outputs from Language Models",
       "authors": ["Anonymous"],
       "year": 2025,
       "arxiv_id": "2501.10868",
       "relevance": "Structured generation frameworks for LLMs, relevant to constrained code generation."
     }
   ]
 }