ai-research-survey

scan.json (21871B)

---

 {
   "paper": {
     "title": "The Hidden Risks of LLM-Generated Web Application Code: A Security-Centric Evaluation of Code Generation Capabilities in Large Language Models",
     "authors": [
       "Swaroop Dora",
       "Deven Lunkad",
       "Naziya Aslam",
       "S. Venkatesan",
       "Sandeep Kumar Shukla"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.20612"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Evaluates security of web application code generated by 5 LLMs (ChatGPT, DeepSeek, Claude, Gemini, Grok) across 48 security parameters in 6 categories. All models fail entirely on HTTP security headers (0/12). Only SQL injection protection is universally implemented. No model implements MFA, CAPTCHA, or CORS policies. The authors conclude human expertise remains essential for secure deployment of LLM-generated web code.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or data release mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The generated code samples and evaluation data are not released or made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions. The prompts are listed in Table II but no instructions for replicating the evaluation procedure."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars. Results are single yes/no evaluations per model with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about which models are more secure but uses no statistical tests — only raw checklist counts."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table V shows compliance counts (e.g., 3/11, 8/10) but no effect sizes, no baseline context for magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only 4 prompts are used to generate code from each model. No justification for why this sample size is sufficient, and no acknowledgment it may be too small."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each model is tested once with each prompt. No repeated runs, no variance across runs reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 5 LLMs against each other and against security best practices (NIST, OWASP), which serve as the baseline standard."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o, DeepSeek v3, Claude 3.5 Sonnet, Gemini 2.0 Flash, and Grok 3 — all contemporary at time of writing."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate — this is an evaluation of existing LLMs, not a proposed method."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluates across 6 security categories with 48 individual parameters and provides both compliance counts and risk assessments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The security evaluation itself is a human expert assessment — the authors manually inspected generated code against their security checklist."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No train/test split concept applies — this is a manual evaluation of generated code against security criteria."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table IV provides detailed per-parameter results and Table V provides per-category compliance summaries for all 6 categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV extensively discusses specific failures per model — e.g., Claude lacks secure cookie flags, DeepSeek/Gemini vulnerable to XSS, all fail on HTTP headers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main finding IS negative — no LLM generates fully secure code. Specific failures documented across all categories."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about critical vulnerabilities in authentication, session management, input validation, and HTTP headers are all supported by the detailed analysis in Table IV and Section IV."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes observational claims about what LLMs produce, not causal claims about why. No causal language like 'X causes Y' — findings are descriptive."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title claims evaluation of 'LLM-Generated Web Application Code' broadly, but tests only 4 PHP/HTML/MySQL prompts for a single e-commerce authentication scenario. Results may not generalize to other web frameworks, languages, or application types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, models may perform differently with different prompting strategies, temperature settings, or explicit security instructions — Section V briefly mentions this but does not substantively discuss it as an alternative explanation for the results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures compliance with a self-designed security checklist and frames it as evaluating 'security' of LLM-generated code. No discussion of whether checklist compliance is a valid proxy for actual security in deployed applications."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table I lists GPT-4o, DeepSeek v3, Claude 3.5 Sonnet, Gemini 2.0 Flash Experimental, Grok 3. These are marketing names without API versions or snapshot dates (e.g., no 'gpt-4o-2024-05-13')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides the actual prompts used, with descriptions of each prompt's purpose and content. Four structured prompts for the e-commerce authentication system are given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other generation parameters reported for any model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used — models are queried directly via their conversational interfaces."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how generated code was collected, stored, or processed for evaluation. The paper jumps from prompts to results with no intermediate methodology."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The Discussion (Section V) makes recommendations but does not discuss study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. No mention of single-run limitation, prompt sensitivity, evaluator subjectivity, or narrow scope."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The scope is implicitly narrow (PHP/MySQL e-commerce auth) but this is never acknowledged as a boundary."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated code samples are not available for independent verification. Only the summarized yes/no results in Table IV are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe when code was generated, which interfaces were used (API vs web chat), or whether generation was repeated. Only the prompts and final results are shown."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — the study evaluates LLM outputs, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the pipeline from code generation to security evaluation. How was each security parameter assessed? Who evaluated? Were there multiple evaluators?"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section. No mention of grants, sponsors, or funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: IIIT Allahabad and IIT Kanpur. None of the authors appear affiliated with the evaluated LLM companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the 5 models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models may have seen similar security evaluation scenarios or e-commerce authentication patterns in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompts are custom-designed but no discussion of whether similar prompts or security evaluation scenarios exist in training data. The prompts reference common web security patterns that are widely documented online."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of API costs, tokens consumed, or any cost metrics for the evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget or hardware information provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each model was tested once with each prompt. No repeated runs across seeds or temperature settings."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each prompt was run. Appears to be a single run but this is never explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning — models are used as-is via their default interfaces."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection — models are used with default settings."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests performed, so no multiple comparison issue."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing LLMs, not the authors' own system. No self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All models are given the same prompts with no compute budget differences to compare."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper designs its own security checklist but does not discuss whether checklist compliance actually measures real-world security. No validation of the evaluation instrument itself."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved — models are queried directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models may have been trained on security evaluation patterns or similar e-commerce authentication examples."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompts (which include security nudges like 'following industry-standard security practices') leak evaluation criteria to the models."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Custom prompts are used rather than a benchmark dataset, so train/test independence in the traditional sense does not apply."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied."
       }
     }
   },
   "claims": [
     {
       "claim": "None of the tested LLMs fully align with industry security best practices for web application code generation.",
       "evidence": "Table IV and V show all models fail in multiple security categories. All score 0/12 on HTTP security headers. Best authentication score is 3/11 (Grok). Section IV-A details specific failures.",
       "supported": "strong"
     },
     {
       "claim": "All LLMs implement SQL injection protection through parameterized queries.",
       "evidence": "Table IV shows all 5 models use parameterized queries and properly escape special characters (Section IV-A4).",
       "supported": "moderate"
     },
     {
       "claim": "No LLM implements HTTP security headers (CSP, X-Frame-Options, HSTS, etc.).",
       "evidence": "Table IV shows all 12 HTTP security header parameters are 'No' for all 5 models. Table V confirms 0/12 across all models.",
       "supported": "moderate"
     },
     {
       "claim": "Claude and DeepSeek generated code with extreme risk due to missing secure cookie flags.",
       "evidence": "Figure 1a and Table IV show Claude and DeepSeek lack Secure, HttpOnly, and SameSite cookie flags, rated as extreme risk in Table III.",
       "supported": "moderate"
     },
     {
       "claim": "Human expertise is crucial for secure deployment of LLM-generated code.",
       "evidence": "Section V discusses this qualitatively based on the findings, but no empirical evidence comparing human-reviewed vs unreviewed code is presented.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-run evaluation with no repetition",
       "detail": "Each model appears to be tested once per prompt with no repeated runs. LLM outputs are stochastic — a single run cannot establish reliable security profiles. The same prompt could yield different code with different security properties."
     },
     {
       "flag": "Extremely narrow test scope presented as general evaluation",
       "detail": "Only 4 prompts for one scenario (PHP/MySQL e-commerce authentication) are used, but the title and conclusions imply general findings about LLM-generated web application code security. Results may not generalize to other languages, frameworks, or application types."
     },
     {
       "flag": "No inter-rater reliability for manual evaluation",
       "detail": "The security evaluation appears to be conducted by the authors manually, but there is no mention of how many evaluators participated, whether there was independent evaluation, or how disagreements were resolved."
     },
     {
       "flag": "Evaluation instrument not validated",
       "detail": "The 48-parameter security checklist is created by the authors without reference to validation. No comparison with established evaluation frameworks beyond citing OWASP and NIST at a high level."
     },
     {
       "flag": "Prompts include security nudges that bias results upward",
       "detail": "The prompts explicitly mention 'secure', 'industry-standard security practices', and 'robust validation' — yet models still fail. This means real-world usage without such nudges would likely produce even worse results, but this is not discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity discussion despite significant methodological constraints (single run, narrow scope, unvalidated instrument, no inter-rater reliability)."
     }
   ],
   "cited_papers": [
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["N. Perry", "M. Srivastava", "D. Kumar", "D. Boneh"],
       "year": 2023,
       "relevance": "Key prior work on security implications of AI code assistants, finding AI-assisted developers produce more vulnerabilities while being more confident."
     },
     {
       "title": "Security weaknesses of Copilot generated code in GitHub",
       "authors": ["Y. Fu", "P. Liang", "A. Tahir", "Z. Li", "M. Shahin", "J. Yu", "J. Chen"],
       "year": 2023,
       "arxiv_id": "2310.02059",
       "relevance": "Quantifies security vulnerabilities in GitHub Copilot-generated code (32.8% Python, 24.5% JavaScript)."
     },
     {
       "title": "How secure is code generated by ChatGPT?",
       "authors": ["R. Khoury", "A. R. Avila", "J. Brunelle", "B. M. Camara"],
       "year": 2023,
       "relevance": "Examines ChatGPT's ability to generate secure code across 5 languages, finding only 5/21 programs initially secure."
     },
     {
       "title": "LLMs in web development: Evaluating LLM-generated PHP code unveiling vulnerabilities and limitations",
       "authors": ["R. Tóth", "T. Bisztray", "L. Erdődi"],
       "year": 2024,
       "relevance": "Prior work evaluating security of GPT-4 generated PHP code, finding 11.56% of sites could be compromised."
     },
     {
       "title": "Large language model assisted software engineering: prospects, challenges, and a case study",
       "authors": ["L. Belzner", "T. Gabor", "M. Wirsing"],
       "year": 2023,
       "relevance": "Discusses LLMs as tools for software engineering operations including code generation."
     }
   ]
 }