ai-research-survey

scan.json (23844B)

---

 {
   "paper": {
     "title": "Security Degradation in Iterative AI Code Generation: A Systematic Analysis of the Paradox",
     "authors": ["Shivani Shukla", "Himanshu Joshi", "Romilla Syed"],
     "year": 2025,
     "venue": "IEEE-ISTAS 2025",
     "arxiv_id": "2506.11022"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Iterative LLM code refinement without human intervention is associated with a 37.6% increase in critical vulnerabilities after 5 iterations, with vulnerability counts accelerating in later iterations (avg 2.1 early vs 6.2 late). Different prompting strategies produce distinct vulnerability profiles: efficiency-focused prompts cause memory safety issues (42.7%), feature-focused cause concurrency issues (30.4%), and even security-focused prompts introduce cryptographic errors (21.1%). Code complexity correlates with vulnerability count (r=0.64), and the repeated-measures ANOVA shows significant iteration effects (F(9,90)=14.32, p<0.001, η²=0.42).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or data release mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 400 generated code samples and vulnerability analysis data are not released. No dataset link provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification beyond stating GPT-4o with temperature=0.7 and top_p=1.0. No static analysis tool versions, no OS, no dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions provided. The appendix includes example prompts but no scripts or step-by-step guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% CIs reported for regression coefficients (Table V, e.g., complexity β=0.64, 95% CI [0.50, 0.78]) and for the complexity-vulnerability relationship (95% CI: 10.7%-17.9%). Standard deviations reported for iteration-level vulnerability counts."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Repeated measures ANOVA (F(9,90)=14.32, p<0.001), chi-square test (χ²(33)=172.4, p<0.001), multiple regression (F(5,394)=160.2, p<0.001), and post-hoc Tukey HSD tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes reported: η²=0.42 for ANOVA, Cramer's V=0.38 for chi-square, R²=0.67 for regression, β coefficients with CIs. Also reports '14.3% increase per 10% complexity increase.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 10 baseline code samples were chosen. No power analysis. 10 samples is quite small for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported for vulnerability counts per iteration (e.g., SD=0.9, SD=1.2, SD=1.8). Standard errors reported in regression table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The 10 verified-secure baseline code samples serve as the starting point, and the four prompting strategies are compared against each other. The design inherently has baselines (iteration 0 = secure code)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Uses GPT-4o, which was current at the time of study. The comparison is across prompting strategies and iterations rather than across models, so contemporariness applies to the model choice."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study. The four prompting strategies are compared but no components are systematically removed or varied (e.g., temperature settings, different prompt structures)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: vulnerability count, severity distribution (Critical/High/Medium/Low), vulnerability type distribution, cyclomatic complexity, lines of code, functional correctness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual security code review was performed alongside static analysis tools: 'After each iteration, we performed: Static analysis using multiple tools... Manual security code review... Categorization and severity assessment' (Section III-A)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is not a train/test split evaluation. All 400 generated samples are analyzed."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by prompting strategy (Table II), by vulnerability type per strategy (Table III), and by iteration group (early/middle/late)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Three detailed case studies (Section IV-E) trace specific vulnerability evolution patterns. Section IV-F discusses where security improvements did occur (27% of security-focused iterations)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reports that 27% of security-focused iterations resulted in net security improvements (Section IV-F), acknowledging partial success. Also notes security-focused prompts introduced the fewest vulnerabilities overall."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims ('37.6% increase in critical vulnerabilities after five iterations', 'distinct vulnerability patterns') are supported by Tables II-III and the ANOVA results in Section IV."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper frequently uses causal-adjacent language ('iterative LLM refinement improves code security' is 'challenged', effects are 'isolated'). The controlled experiment manipulates prompting strategy, but with only 10 baseline samples the design is underpowered for causal claims. The paper acknowledges the scenario is artificial but still frames conclusions causally."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title says 'A Systematic Analysis of the Paradox' suggesting generality, but results are from one model (GPT-4o), two languages (C, Java), and 10 baseline samples. Limitations mention single model and language constraints but the title and abstract overreach."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for why vulnerabilities increase. Could be temperature effects, code length growth as a confound (they show correlation but don't control for it), or prompt drift. The complexity-vulnerability correlation is noted but not explored as a confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures static analysis tool findings and manual review results, then frames this as 'security degradation.' No discussion of whether static analysis findings map to actual exploitable vulnerabilities. The gap between tool-detected issues and real-world security risk is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "States 'OpenAI's GPT-4o' without a snapshot date or API version. GPT-4o has multiple versions (e.g., gpt-4o-2024-05-13, gpt-4o-2024-08-06). No version specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides 5 example prompts for each of the 4 strategies (20 total). These appear to be the actual prompts used, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=0.7 and top_p=1.0 explicitly stated in Section III-C."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used. The experiment is simple iterative prompting — submit code, receive output, submit output as next input."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the full pipeline: baseline selection with security verification, prompt definition, iterative feedback process, and security analysis procedure. Section III-B describes the 10 baseline categories."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Limitations and Future Work' provides substantive discussion of study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: single model (GPT-4o), two languages only (C, Java), artificial scenario without human intervention, model evolution over time. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI explicitly states what was not tested: other LLMs (Claude, Llama), other languages (Rust, Go), human-AI collaborative workflows. Also acknowledges the 'pure LLM interactions without human intervention' limitation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 400 code samples, static analysis outputs, and manual review data are not released. No supplementary materials available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III details the experimental procedure: 10 baseline samples, 4 strategies, 10 iterations each, using GPT-4o with specified parameters. Static analysis tools named (Clang Static Analyzer, CodeQL, SpotBugs)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 'samples' are code snippets, and the data source is LLM-generated code from a controlled experiment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-F documents the pipeline: generated code → static analysis (3 tools) → manual review → categorization and severity assessment → statistical analysis. Section III-D lists the 12 vulnerability categories and CVSS-based severity levels."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section present. No mention of grants or sponsorship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly stated: University of San Francisco, Vector Institute for AI, University of Massachusetts Boston. None affiliated with OpenAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed at all, so independence cannot be assessed. Absence of disclosure is not absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The study evaluates GPT-4o's code generation on 10 baseline code samples. No mention of GPT-4o's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the baseline code samples or similar patterns exist in GPT-4o's training data. The model may have seen similar security patterns."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 10 baseline samples are described categorically but their provenance is unclear. No discussion of whether similar code exists in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The experiment generates and analyzes code samples."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "400 GPT-4o API calls made but no cost, token count, or latency information reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total API cost, compute resources for static analysis, or time to complete the experiment."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature=0.7 introduces stochasticity but no multi-seed or multi-run analysis is reported. Each iteration appears to be a single run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of how many times each iteration was run. The 400 samples appear to be from single runs (10 samples × 4 strategies × 10 iterations)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature=0.7 chosen without justification. No search over temperature, top_p, or other settings."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection — a single configuration was used throughout."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple statistical tests performed (ANOVA, chi-square, regression, Tukey HSD) but no family-wise error correction discussed across the set of analyses."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not propose a new system to compare against baselines. It evaluates GPT-4o's behavior."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not compare methods at different compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 10 hand-selected baseline code samples are representative of real-world security-critical code. No analysis of whether static analysis tool findings map to exploitable vulnerabilities."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding used. Simple iterative prompting with no agent framework."
       }
     }
   },
   "claims": [
     {
       "claim": "37.6% increase in critical vulnerabilities after just five iterations of LLM refinement",
       "evidence": "Stated in abstract and supported by Table II showing 87 total critical vulnerabilities across strategies. Iteration-level data shows early avg 2.1 (SD=0.9) vs late avg 6.2 (SD=1.8) vulnerabilities per sample.",
       "supported": "moderate"
     },
     {
       "claim": "Different prompting strategies produce distinct vulnerability patterns",
       "evidence": "Table III shows vulnerability type distributions by strategy, with chi-square test confirming significant differences (χ²(33)=172.4, p<0.001, Cramer's V=0.38). Efficiency-focused: 42.7% memory safety; Feature-focused: 30.4% concurrency.",
       "supported": "strong"
     },
     {
       "claim": "Security vulnerabilities accumulate non-linearly across iterations, accelerating in later iterations",
       "evidence": "Repeated measures ANOVA (F(9,90)=14.32, p<0.001, η²=0.42). Post-hoc Tukey HSD shows significant differences between iterations 1-3 and 8-10 (p<0.001).",
       "supported": "moderate"
     },
     {
       "claim": "Even security-focused prompts introduce new vulnerabilities, particularly cryptographic implementation errors (21.1%)",
       "evidence": "Table II shows 38 total vulnerabilities for security-focused prompts (fewest but non-zero). Table III shows 21.1% cryptographic errors. Section IV-C provides qualitative analysis of three patterns.",
       "supported": "moderate"
     },
     {
       "claim": "Positive correlation (r=0.64) between code complexity increases and security vulnerability counts",
       "evidence": "Section IV-D reports r=0.64 (p<0.001), β=0.64 in multiple regression (R²=0.67). 95% CI for complexity effect: 10.7%-17.9% vulnerability increase per 10% complexity increase.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small baseline sample",
       "detail": "Only 10 baseline code samples used. This is extremely small for generalizing about 'security degradation in iterative AI code generation.' The statistical analyses (ANOVA with N=10) have limited power and the samples may not be representative."
     },
     {
       "flag": "Single model tested, broad claims",
       "detail": "Only GPT-4o tested, but conclusions are framed as being about LLMs generally. Title says 'Iterative AI Code Generation' not 'GPT-4o Code Generation.'"
     },
     {
       "flag": "No reproducibility artifacts",
       "detail": "No code, data, or baseline samples released. The 10 baseline code samples, 400 generated outputs, and analysis scripts are all unavailable. Results cannot be independently verified."
     },
     {
       "flag": "Temperature stochasticity not addressed",
       "detail": "Temperature=0.7 introduces randomness but each iteration appears to be a single run. Different random seeds could produce very different vulnerability trajectories. No multi-run analysis."
     },
     {
       "flag": "Proxy-outcome gap unacknowledged",
       "detail": "Static analysis tool findings are treated as equivalent to security vulnerabilities. No discussion of false positive rates of static analysis tools or whether findings represent exploitable vulnerabilities."
     },
     {
       "flag": "Manual review methodology unclear",
       "detail": "Manual security code review is listed as part of the analysis but inter-rater reliability is not discussed. How many reviewers? What was the agreement rate? Were they blinded to the prompting strategy?"
     },
     {
       "flag": "Broken section reference",
       "detail": "Section II-D references 'Section ??' — a broken LaTeX cross-reference, suggesting incomplete preparation."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2022,
       "relevance": "Foundational empirical study of Copilot security finding 40% of generated programs contained vulnerabilities."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["N. Perry", "M. Srivastava", "D. Kumar", "D. Boneh"],
       "year": 2023,
       "relevance": "User study showing AI-assisted developers write less secure code with a false sense of security."
     },
     {
       "title": "Artificial-Intelligence Generated Code Considered Harmful: A Road Map for Secure and High-Quality Code Generation",
       "authors": ["C. J. Chong", "Z. Yao", "I. Neamtiu"],
       "year": 2024,
       "arxiv_id": "2409.19182",
       "relevance": "Found LLM-generated code lacks defensive programming constructs; noted that prompting can introduce issues in previously clean files."
     },
     {
       "title": "A systematic literature review on the impact of AI models on the security of code generation",
       "authors": ["C. Negri-Ribalta", "R. Geraud-Stewart", "A. Sergeeva", "G. Lenzini"],
       "year": 2024,
       "relevance": "SLR synthesizing 19 studies confirming AI models do not produce safe code despite mitigations."
     },
     {
       "title": "Cybersecurity Risks of AI-Generated Code",
       "authors": ["J. Ji", "J. Jun", "M. Wu", "R. Gelles"],
       "year": 2024,
       "relevance": "CSET study finding ~50% of AI-generated code contained exploitable bugs."
     },
     {
       "title": "Refining ChatGPT-generated code: Characterizing and mitigating code quality issues",
       "authors": ["Y. Liu", "T. Le-Cong", "R. Widyasari", "D. Lo", "M. Tao", "S. Han"],
       "year": 2024,
       "relevance": "Analyzed ChatGPT code refinement process finding it can introduce new quality issues."
     },
     {
       "title": "LLM critics help catch LLM bugs",
       "authors": ["N. McAleese"],
       "year": 2024,
       "arxiv_id": "2407.00215",
       "relevance": "Critic-based model for automated feedback on generated code, showing structured prompting improves code quality."
     },
     {
       "title": "LLM4CVE: Enabling Iterative Automated Vulnerability Repair with Large Language Models",
       "authors": ["M. Fakih"],
       "year": 2025,
       "arxiv_id": "2501.03446",
       "relevance": "Explores iterative LLM-based vulnerability repair — complementary perspective to this paper's degradation findings."
     }
   ]
 }