ai-research-survey

scan.json (27879B)

---

 {
   "paper": {
     "title": "Automatically Generating Web Applications from Requirements Via Multi-Agent Test-Driven Development",
     "authors": [
       "Yuxuan Wan",
       "Tingshuo Liang",
       "Jiakai Xu",
       "Jingyu Xiao",
       "Yintong Huo",
       "Michael Lyu"
     ],
     "year": 2025,
     "venue": "arXiv preprint (submitted to ACM conference)",
     "arxiv_id": "2509.25297"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The code of TDDev is available at https://github.com/yxwan123/TDDev' in both the abstract and Section 8 (Data Availability)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the Req-to-App-MM dataset (an extension of WebGen-Bench with Gemini-generated images) but does not provide a download link or release the augmented dataset. WebGen-Bench itself is referenced but the multimodal extension is not explicitly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'iMac with 10 Core Intel Core i9 processor, 32GB RAM' and that 'All LLM models are accessed through the official API services' (Section 4.4), but no requirements.txt, Dockerfile, or detailed dependency/library version listing is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a GitHub link but does not include step-by-step reproduction instructions in the paper itself. There is no 'Reproducing Results' section or specific commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Tables 6-9) are reported as point estimates (e.g., '78.2% accuracy') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TDDev outperforms baselines (e.g., '14.4% improvement') based solely on comparing numbers in tables without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides percentage improvements with baseline context throughout. For example, 'TDDev achieves the highest accuracy (78.2%), a relative improvement of +30% over Cursor (60.2%) and more than triple the performance of Bolt.diy (25.6%)' (Section 5.1.1). This provides enough context for the reader to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses only 10 websites sampled from WebGen-Bench ('we randomly select 10 websites while preserving the category distribution', Section 4.6.1). No justification for this small sample size is provided, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any experimental runs. Results appear to be single-run numbers with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares TDDev against two baselines: Bolt.diy (open-source) and Cursor (proprietary), both described as 'widely used open-sourced and proprietary industry-level code-agent frameworks' (Section 4.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Bolt.diy (17.7k GitHub stars) and Cursor (360k active users by 2024) are contemporary, actively used tools. Both are described with current metrics and are state-of-the-art in this domain."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 5.1.3) ablates the multi-step test case generation agent vs. a straightforward single-step variant. RQ3 (Section 5.2) ablates the testing agent feedback across 0, 1, 2, and 3 rounds. Both ablations isolate specific components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: accuracy (weighted Yes/Partial/No), fail-to-start rate, appearance score (1-5), visual similarity (1-5), and per-category breakdowns for functionality, data display, and design validation (Tables 6-7)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes manual verification of UI agent testing results on 5 generated web applications (28 test cases). Two annotators independently labeled each case with a third resolving disagreements. RQ4 (Section 5.3) includes a user study with 3 developers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the test set of WebGen-Bench (Section 4.6.1), which is a separately defined benchmark. The 10 websites are randomly selected from this test set, not from any data used during development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 provides per-category breakdowns across both instruction categories (Content, User Interaction, Data Management) and test case categories (Functionality, Data Display, Design Validation)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 ('Failure Analysis and Future Works') discusses failures including scaling feedback rounds introducing variance, open-source LLM formatting failures, and coding ability limitations of specific models."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 shows that 1-2 rounds of feedback actually decrease accuracy from 58.3% to 25.3%, which is openly reported and discussed: 'accuracy drops to 25.3%, suggesting that 1-2 rounds may be insufficient' (Section 5.2). Open-source model failures are also reported in Section 6.2.2."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '14.4% improvement on overall accuracy compared to state-of-the-art baselines.' Table 6 shows TDDev with Claude-4-Sonnet at 70.2% vs Cursor at 63.8%, which is approximately a 6.4 percentage point (or ~10% relative) improvement. With GPT-4.1 the improvement is larger (78.2% vs 60.2% = 30% relative). The 14.4% figure is somewhat ambiguous (not clearly explained as which comparison), but the general direction of improvement is supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (RQ2, RQ3) that use controlled single-variable manipulation: removing the multi-step test generation (Table 8) or varying feedback rounds (Table 9). These ablation designs adequately support the causal claims about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and abstract claim effectiveness for 'full-stack web application generation' generally, but all results come from only 10 websites from WebGen-Bench with two backbone LLMs. The paper does not explicitly bound its claims to this narrow test setting. The broad title 'Automatically Generating Web Applications' overstates the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section 7) only discusses two generic threats (agentic system design effectiveness and UI agent testing reliability) without considering alternative explanations for the observed improvements, such as whether the improvements come from additional compute/tokens rather than the TDD approach itself, or whether the choice of templates creates an unfair advantage."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4.1' and 'Claude-4-Sonnet' without specifying snapshot dates or API versions. For open-source models it mentions 'Qwen-2.5-VL-72B' (which includes size) and 'DeepSeek-V3.1'. The main models lack version specificity beyond marketing names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides several actual prompts: the context selection prompt (Section 3.2), the development prompt (Section 3.2), the visual similarity evaluation prompt (Section 4.6.3), and the straightforward test case generation prompt (Section 5.1.3). While some use placeholders (e.g., '[Available Files]', '[Context Buffer]'), the core system instructions and evaluation prompts are provided in full."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 states 'Temperatures are set to 0 for all models. Max tokens are set to the maximum allowable value for each model.' While minimal, this covers the key LLM API hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail across Section 3: the three-agent architecture (test generation, development, testing), workflow orchestration, template fetching, context selection, file editing mechanisms, retry logic (bounded retry mechanism in Section 3.3), and feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6.1 describes the dataset construction: starting from WebGen-Bench test set, augmenting with Gemini-2.5-Flash-Image generated design images, selecting 10 websites while preserving category distribution. Table 5 provides category statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Threat to Validity') provides a dedicated threats-to-validity discussion with two identified threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The two threats discussed in Section 7 are fairly generic: (1) 'performance may be constrained by the effectiveness of individual components' and (2) 'evaluation relies on UI agent testing, which requires high levels of reliability and accuracy.' These are not specific to this study's particular design choices or observed patterns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound the scope to specific types of web applications, specific model families, or specific complexity levels. The discussion of open-source model limitations (Section 6.2.2) is the closest, but this is framed as 'future work' rather than explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw test results, generated applications, BrowserUse logs, and human annotation data are not available for independent verification. Only aggregate metrics in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6.1 describes the dataset construction from WebGen-Bench, the image augmentation process using Gemini-2.5-Flash-Image, the sampling strategy (10 websites preserving category distribution), and the evaluation process using BrowserUse."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the user study (RQ4, Section 5.3), the paper states 'we recruit three developers (two research staff who have previously developed at least two web applications, and one front-end developer from a startup company)' but does not describe how they were recruited, whether they volunteered, or whether there is selection bias in using researchers from the authors' institution."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: dataset construction (Section 4.6.1) → functionality evaluation via BrowserUse (Section 4.6.2) → visual evaluation via Claude-4-Sonnet (Section 4.6.3). The scoring formula is explicitly stated. The test case execution process is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Chinese University of Hong Kong, Columbia University, and Singapore Management University. The paper evaluates third-party tools (GPT-4.1, Claude-4-Sonnet, Cursor, Bolt.diy) and the authors are not affiliated with those companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed, it is impossible to assess whether the funder is independent of the outcome. The absence of a funding disclosure means this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or financial interest declaration in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4.1 and Claude-4-Sonnet to generate web applications and evaluate them, but does not state training data cutoff dates for any model used. This matters because WebGen-Bench or similar web development benchmarks could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WebGen-Bench data or similar web application generation tasks appeared in the training data of GPT-4.1 or Claude-4-Sonnet. Given that the models may have seen web development code and specifications, this is a relevant concern."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WebGen-Bench was published in 2025 (arXiv:2505.03733), so depending on model training cutoffs it may or may not be contaminated. The paper does not address this. The Req-to-App-MM extension uses generated images from Gemini which adds novelty, but the text requirements from WebGen-Bench could still be contaminated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The user study in RQ4 involves 3 human developers but no pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the user study involving 3 developers."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 describes participants: 'two research staff who have previously developed at least two web applications, and one front-end developer from a startup company.' While minimal, this characterizes their experience level and role."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No explicit inclusion/exclusion criteria are stated for participant selection. The paper states they were 'recruited' following the methodology of Chen et al. [11] but does not specify screening criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The user study (RQ4) is not a comparative experimental study with randomized assignment. All participants use both tools sequentially (TDDev first, then Bolt.diy), making this a within-subjects design without randomization."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Participants know which tool they are using (TDDev vs. Bolt.diy), and the order is fixed (TDDev first, then Bolt.diy), which could introduce order effects."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "With only 3 participants and results reported for all 3 (Table 10 shows averages), there is implicitly no attrition. All participants completed both conditions."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports per-round costs: TDDev costs ~0.36 USD/round (split between ~10K tokens for development and ~10K for testing, ~4 min each). Bolt.diy costs ~0.18 USD/round. Cursor costs 20 USD/month subscription."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 states hardware used ('iMac with 10 Core Intel Core i9 processor, 32GB RAM'). Table 4 provides time per round (~4 min development + ~4 min testing). Section 4.5 discusses cost structures. Total wall-clock time for the user study is 18.7 minutes for TDDev."
       }
     }
   },
   "claims": [
     {
       "claim": "TDDev achieves a 14.4% improvement on overall accuracy compared to state-of-the-art baselines.",
       "evidence": "Table 6 shows TDDev with Claude-4-Sonnet at 70.2% accuracy vs. Bolt.diy at 44.5% and Cursor at 63.8%. With GPT-4.1, TDDev achieves 78.2% vs. Bolt.diy at 25.6% and Cursor at 60.2%. The 14.4% figure is not clearly derived from any single comparison in the paper.",
       "supported": "moderate"
     },
     {
       "claim": "TDDev has zero fail-to-start rate, while baselines have 20-80% fail-to-start rates.",
       "evidence": "Table 6 shows TDDev has 0.0% fail-to-start in both GPT-4.1 and Claude-4-Sonnet configurations, while Bolt.diy has 80.0%/20.0% and Cursor has 40.0%/20.0% fail-to-start rates.",
       "supported": "strong"
     },
     {
       "claim": "The multi-step test case generation agent improves functionality accuracy from 33.3% to 61.1% over single-step generation.",
       "evidence": "Table 8 (Section 5.1.3, RQ2) shows straightforward test generation achieves 33.3% functionality accuracy vs. 61.1% for multi-step, with overall accuracy improving from 59.1% to 70.2%.",
       "supported": "moderate"
     },
     {
       "claim": "Three rounds of testing feedback improve accuracy from 58.3% (no feedback) to 70.2%.",
       "evidence": "Table 9 (Section 5.2, RQ3) shows accuracy progression: no feedback 58.3%, 1 round 25.3%, 2 rounds 25.2%, 3 rounds 70.2%. Notably, intermediate rounds decrease performance.",
       "supported": "moderate"
     },
     {
       "claim": "TDDev eliminates the need for manual intervention, reducing manual effort to zero.",
       "evidence": "Table 10 (Section 5.3, RQ4) shows TDDev requires 0.0 minutes of manual intervention vs. 4.7 minutes for Bolt.diy, with 0 intervention frequency and 0 additional prompt words. Based on user study with 3 developers.",
       "supported": "weak"
     },
     {
       "claim": "BrowserUse achieves 82.8% alignment rate with manual testing, and 100% alignment on Yes/No cases.",
       "evidence": "Table 11 (Section 6.1) reports 82.8% overall alignment and 100% Yes/No alignment for BrowserUse vs. 46.4% and 66.7% for WebVoyager, based on manual verification of 28 test cases across 5 applications.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "TDDev, a multi-agent TDD framework for full-stack web application generation, outperforms Bolt.diy and Cursor on WebGen-Bench with 10 test websites, achieving 70-78% accuracy depending on backbone LLM compared to 25-64% for baselines. Ablation studies show that multi-step test case generation and 3 rounds of iterative feedback both contribute meaningfully to performance, though 1-2 rounds of feedback actually decrease performance. A small user study with 3 developers found that TDDev eliminated all manual intervention compared to Bolt.diy. The framework achieves zero fail-to-start rate while baselines fail to launch 20-80% of the time.",
   "red_flags": [
     {
       "flag": "Very small evaluation sample",
       "detail": "Only 10 websites are used for evaluation from WebGen-Bench. This is acknowledged in the paper but not justified. With N=10, individual website results dominate aggregate metrics, and category-level breakdowns (Table 7) are based on as few as 3-4 websites per category."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (14.4% improvement, triple performance) are based on raw percentage comparisons without any significance tests. With N=10, observed differences could easily be due to chance."
     },
     {
       "flag": "Tiny user study (N=3)",
       "detail": "RQ4 claims practical advantages based on only 3 developers. No statistical inference is possible with N=3. The order is not counterbalanced (all use TDDev first, then Bolt.diy), introducing potential order effects."
     },
     {
       "flag": "Non-monotonic feedback results",
       "detail": "Table 9 shows that 1-2 rounds of feedback dramatically decrease accuracy (from 58.3% to 25.3%), while 3 rounds recover to 70.2%. This non-monotonic pattern on N=10 could indicate high variance and cherry-picked iteration count rather than a robust finding."
     },
     {
       "flag": "Self-evaluation using own UI agent",
       "detail": "TDDev uses BrowserUse for both internal testing (as part of the framework) and external evaluation. The same tool is used to drive the system and to evaluate it, creating a potential alignment bias where the evaluation favors the system's own testing approach."
     },
     {
       "flag": "Cost comparison favors TDDev unfairly",
       "detail": "TDDev's total development time is longer (18.7 min vs. 15.2 min for Bolt.diy per Table 10), and 3 rounds of TDD adds significant token cost. The paper emphasizes 'zero manual intervention' but the higher total time and compute cost are downplayed."
     },
     {
       "flag": "14.4% improvement claim unclear",
       "detail": "The abstract's headline claim of '14.4% improvement' does not cleanly match any single comparison in the results tables. The derivation of this specific figure is not explained in the paper."
     }
   ],
   "cited_papers": [
     {
       "title": "WebGen-Bench: Evaluating LLMs on Generating Interactive and Functional Websites from Scratch",
       "authors": ["Zimu Lu", "Yunqiao Yang", "Houxing Ren", "Haotian Hou", "Han Xiao", "Ke Wang", "Weikang Shi", "Aojun Zhou", "Mingjie Zhan", "Hongsheng Li"],
       "year": 2025,
       "arxiv_id": "2505.03733",
       "relevance": "Primary benchmark used for evaluation; empirical study on LLM agents' ability to generate full-stack websites."
     },
     {
       "title": "Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2025,
       "relevance": "Study on LLM-based SE agents, relevant to understanding agentic AI capabilities in software engineering."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Applies test cases for fault localization in autonomous program repair, related agentic approach to code improvement."
     },
     {
       "title": "Test-Driven Development and LLM-based Code Generation",
       "authors": ["Noble Saji Mathews", "Meiyappan Nagappan"],
       "year": 2024,
       "doi": "10.1145/3691620.3695527",
       "relevance": "Empirically demonstrates TDD principles benefit LLM-based code generation using human-written tests."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper highlighting limits of single-sample code generation, motivating iterative testing approaches."
     },
     {
       "title": "Design2Code: How Far Are We From Automating Front-End Engineering?",
       "authors": ["Chenglei Si", "Yanzhe Zhang", "Zhengyuan Yang", "Ruibo Liu", "Diyi Yang"],
       "year": 2024,
       "arxiv_id": "2403.03163",
       "relevance": "Benchmark and evaluation of MLLM capabilities for front-end code generation from screenshots."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jiaru Qian", "Tian Wang", "Kechi Zhang", "Zhi Jin", "Ge Li"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey covering LLM-based agent approaches to code generation, directly relevant to the survey scope."
     },
     {
       "title": "Repairagent: An autonomous, llm-based agent for program repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Autonomous LLM-based agent for program repair, related agentic software engineering approach."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Beth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Measures AI impact on developer productivity, relevant to understanding practical benefits of AI coding tools."
     },
     {
       "title": "Towards autonomous testing agents via conversational large language models",
       "authors": ["Robert Feldt", "Sungmin Kang", "Juyeon Yoon", "Shin Yoo"],
       "year": 2023,
       "relevance": "Early work on LLM-based autonomous testing agents, foundational to the testing agent design in this paper."
     },
     {
       "title": "WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models",
       "authors": ["Hongliang He", "Wenlin Yao", "Kaixin Ma", "Wenhao Yu"],
       "year": 2024,
       "relevance": "Web navigation agent using multimodal LLMs, used as comparison for UI agent evaluation accuracy."
     }
   ]
 }