ai-research-survey

scan.json (25838B)

---

 {
   "paper": {
     "title": "The Role of Generative AI in Strengthening Secure Software Coding Practices: A Systematic Perspective",
     "authors": ["Hathal S. Alwageed", "Rafiq Ahmad Khan"],
     "year": 2025,
     "venue": "EASE Companion",
     "arxiv_id": "2504.19461",
     "doi": "10.1145/3727967.3756840"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This SLR identifies 109 GenAI practices for addressing cybersecurity risks in software coding, organized across 11 risk categories (insecure coding, vulnerable dependencies, poor error handling, weak authentication, misconfigured security controls, inadequate encryption, XSS, insufficient logging, race conditions, inadequate testing, supply chain attacks). The paper reports that automated vulnerability detection accounts for 27% of GenAI's role, followed by secure code generation at 23%. However, many listed 'GenAI practices' are standard software engineering practices (e.g., thread synchronization, output encoding) rather than genuinely AI-driven techniques, and no empirical validation of the taxonomy is provided.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No analysis code, scripts, or repository links are provided anywhere in the paper. A survey can release its extraction scripts and analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The list of 31 final papers is not provided as a downloadable dataset. Table 4 lists extracted practices but the underlying data extraction spreadsheet or corpus is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No computational environment is specified. While this is a survey, any analysis tooling or scripts used for data extraction could have been documented."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the search string and databases are listed, a reader would need to guess many steps to reproduce the review."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not run experiments or produce statistical results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No experimental comparisons are made. The paper is a survey that categorizes practices from literature."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted, so effect sizes are not applicable."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experimental sample sizes apply. The number of reviewed papers (31) is an output of the search process, not an experimental design choice requiring power analysis."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted. This is a literature review."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare its findings against prior surveys or systematic reviews on the same topic. No reference to existing surveys on GenAI for secure coding is made as a baseline for comparison."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experimental baselines are applicable to this survey paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. This is a systematic literature review."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experimental evaluation with metrics is conducted."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. This is a literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set is applicable to this survey paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 breaks down 109 GenAI practices across 11 cybersecurity risk categories (insecure coding, vulnerable dependencies, poor error handling, etc.). Figure 2 provides a percentage breakdown of GenAI roles across 6 application areas."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of where GenAI practices fail, cases where they are ineffective, or limitations of specific practices. All 109 practices are presented positively without failure analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every finding is positive about GenAI's role in secure coding. No practices that were found to be ineffective or counterproductive are mentioned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims analysis via 'empirical studies on how these tools help to mitigate security risks' but no original empirical studies are conducted — the paper is purely an SLR. The abstract also claims findings will 'benefit researchers, software engineers and cybersecurity professionals alike in integrating GenAI into a secure development workflow' but the taxonomy lacks empirical validation."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 opens with 'Generative AI (GenAI) significantly contributes to detecting and preventing cybersecurity threats' and Section 5 states 'GenAI has proven effective in identifying vulnerabilities early.' These are causal claims presented without original empirical evidence. The claims are derived from reviewed literature but without quality assessment of the source studies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'GenAI has proven effective in identifying vulnerabilities early in the software development lifecycle' (Section 5) are unbounded. The paper doesn't specify which GenAI tools, which types of vulnerabilities, or which development contexts. The title itself — 'Strengthening Secure Software Coding Practices' — implies general applicability without bounds."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 discusses validity types (external, internal, construct) but does not consider alternative explanations for the findings. For example, no discussion of whether the observed GenAI practices might be repackaged traditional practices, or whether publication bias in the source studies inflates GenAI's perceived effectiveness."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not make direct measurements. No proxy-outcome gap exists to discuss."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in the methodology. This is a manual systematic literature review."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a manual literature review."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are run, so no hyperparameters apply."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides paper counts at each stage per database (2694 initial → 220 initial selection → 31 final). Table 2 provides inclusion and exclusion criteria (IC1-IC5, EC1-EC5). Figure 1 shows the SLR stage-steps. The search string is provided verbatim."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 is titled 'Limitations of the Study' and discusses external validity, internal validity, and construct validity across several paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 discusses three validity types but in generic terms. External validity says 'The general application scope for particular industries together with emerging technologies demands additional specific studies.' Internal validity claims 'a systematic method of paper selection that uses transparent criteria' without identifying specific threats. These are boilerplate discussions of validity categories, not specific threats to this particular study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Section 4 makes general claims about generalizability but doesn't bound specific scope limitations (e.g., which programming languages, which types of GenAI tools, which development contexts are excluded)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The extracted data from the 31 reviewed papers is not available for independent verification. Table 4 summarizes practices but the underlying extraction is not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the search strategy: databases used (IEEE Xplore, ScienceDirect, ACM, SpringerLink, Google Scholar), the search string, inclusion/exclusion criteria (Table 2), and quality assessment criteria (Table 3). Table 1 shows results per database."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is literature from academic databases."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents the pipeline from initial search results (2694) to initial selection (220) to final selection (31) across five databases. Table 2 provides the filtering criteria. Figure 1 diagrams the SLR stages."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Hathal S. Alwageed at Jouf University (Saudi Arabia) and Rafiq Ahmad Khan at University of Malakand (Pakistan). Neither institution has a product being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, making it impossible to assess funder independence. Absence of disclosure is treated as NO."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is conducted. This is a literature review."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. This is a literature review."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. (Paper inclusion/exclusion criteria are covered under data_integrity.)"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows Kitchenham's SLR guidelines [17], uses a structured search string, defines inclusion/exclusion criteria (Table 2), provides a stage diagram (Figure 1), and documents paper counts at each stage (Table 1). While not a PRISMA flow diagram, it follows an established systematic review protocol."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 lists 6 quality assessment criteria (e.g., 'Relevance of the research problem', 'Methodological Rigor') but they are just labels with no scoring rubric, no operationalization, and no reported scores. The paper explicitly cites Petersen et al. [21] to justify 'performing a quality assessment without setting strict criteria.' The 109 practices in Table 4 are presented without any evaluation of the methodological quality of their source studies."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No mention of publication bias anywhere in the paper. No funnel plots, no discussion of whether reviewed papers skew toward positive results about GenAI, no acknowledgment that the literature may overrepresent successful GenAI applications."
       }
     }
   },
   "claims": [
     {
       "claim": "GenAI significantly contributes to detecting and preventing cybersecurity threats in secure software coding.",
       "evidence": "Section 3 opening statement, supported by Table 4 listing 109 practices extracted from 31 reviewed papers. No original empirical validation provided.",
       "supported": "weak"
     },
     {
       "claim": "Automated Vulnerability Detection and Mitigation accounts for the largest proportion (27%) of GenAI's role in addressing cybersecurity risks.",
       "evidence": "Figure 2 shows a percentage breakdown across 6 application areas. The methodology for computing these percentages is not explained.",
       "supported": "weak"
     },
     {
       "claim": "Secure Code Generation and Assistance represents 23% of GenAI's contributions to cybersecurity.",
       "evidence": "Figure 2. No methodology for computing the percentage is described.",
       "supported": "weak"
     },
     {
       "claim": "GenAI has proven effective in identifying vulnerabilities early in the software development lifecycle, preventing threats before they reach the final product.",
       "evidence": "Section 5 conclusion. Claim is derived from the literature review but no original empirical evidence is presented, and reviewed studies are not quality-assessed.",
       "supported": "unsupported"
     },
     {
       "claim": "Organizations following the combination of AI frameworks and maturity models achieve 30% higher efficiency in adoption of AI projects.",
       "evidence": "Section 1, attributed to McKinsey [16]. This is a second-hand citation from a consultancy report, not validated by this study.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Many 'GenAI practices' are standard engineering practices",
       "detail": "Table 4 lists items like 'Thread Synchronization', 'Avoid Shared State', 'Output Encoding', 'Use of Strong Encryption Algorithms', 'Atomic Operations', and 'Formal Verification' as 'GenAI Practices.' These are well-established software engineering practices that predate GenAI by decades. Labeling them as GenAI practices inflates the apparent contribution of GenAI."
     },
     {
       "flag": "No quality assessment of source studies despite claiming one",
       "detail": "Table 3 lists quality assessment criteria but the paper explicitly cites Petersen et al. to avoid applying strict criteria. No quality scores are reported for any of the 31 included papers. This means the 109 practices are extracted without evaluating whether they come from rigorous or weak studies, effectively laundering the signal-to-noise ratio."
     },
     {
       "flag": "Contradictory inclusion/exclusion criteria",
       "detail": "IC4 includes papers 'discussing some AI approaches, models, algorithms... for cybersecurity in software coding' while EC4 excludes papers 'proposing AI approach, model, algorithm... for cybersecurity in software coding.' The distinction between 'discussing' and 'proposing' is unclear and could lead to arbitrary inclusion decisions."
     },
     {
       "flag": "Unexplained percentage methodology",
       "detail": "Figure 2 presents a precise breakdown (27%, 23%, 17%, 14%, 12%, 7%) of GenAI's roles but provides no explanation of how these percentages were calculated — whether from paper counts, practice counts, citation frequency, or author judgment."
     },
     {
       "flag": "Abstract overpromises empirical content",
       "detail": "The abstract states the paper analyzes via 'empirical studies on how these tools help to mitigate security risks' but no original empirical studies are conducted. The paper is entirely a literature review."
     },
     {
       "flag": "Uncritical positive framing",
       "detail": "All 109 practices and all findings are presented positively. No discussion of GenAI failures in security contexts, risks of GenAI-generated insecure code, or limitations of specific practices. The conclusion states 'GenAI has proven effective' without qualification."
     },
     {
       "flag": "Reference numbering inconsistencies",
       "detail": "Several reference numbers in the introduction text appear to mismatch their cited content (e.g., [9] is cited for trustworthiness models by Medeiros et al. in the introduction but reference [9] lists Ulfsnes et al. on Transforming Software Development). This suggests citation management issues."
     }
   ],
   "cited_papers": [
     {
       "title": "Generative Artificial Intelligence for Software Security Analysis: Fundamentals, Applications, and Challenges",
       "authors": ["A. Ding", "G. Li", "X. Yi", "X. Lin", "J. Li", "C. Zhang"],
       "year": 2024,
       "relevance": "Directly addresses GenAI for software security analysis including fundamentals, applications, and challenges."
     },
     {
       "title": "Generative AI in Cybersecurity: A Comprehensive Review of LLM Applications and Vulnerabilities",
       "authors": ["M. A. Ferrag", "F. Alwahedi", "A. A. Battah", "B. Cherif", "A. Mechri", "N. Tihanyi"],
       "year": 2024,
       "relevance": "Comprehensive review of LLM applications and vulnerabilities in cybersecurity, directly relevant to AI security evaluation."
     },
     {
       "title": "Optimizing Secure AI Lifecycle Model Management With Innovative Generative AI Strategies",
       "authors": ["A. O. Almagrabi", "R. A. Khan"],
       "year": 2025,
       "relevance": "Addresses secure AI lifecycle management with GenAI strategies, relevant to AI-assisted software security."
     },
     {
       "title": "Generative AI for software practitioners",
       "authors": ["C. Ebert", "P. Louridas"],
       "year": 2023,
       "relevance": "Examines GenAI from a software practitioner perspective, relevant to practical AI-assisted development."
     },
     {
       "title": "Generative AI for Cyber Security: Analyzing the Potential of ChatGPT, DALL-E and Other Models for Enhancing the Security Space",
       "authors": ["S. Sai", "U. Yashvardhan", "V. Chamola", "B. Sikdar"],
       "year": 2024,
       "relevance": "Analyzes potential of specific GenAI models (ChatGPT, DALL-E) for cybersecurity applications."
     },
     {
       "title": "Harnessing GPT-4 for generation of cybersecurity GRC policies: A focus on ransomware attack mitigation",
       "authors": ["T. McIntosh", "T. Liu", "T. Susnjak", "H. Alavizadeh", "A. Ng", "R. Nowrozy"],
       "year": 2023,
       "relevance": "Empirical study using GPT-4 for cybersecurity policy generation, directly testing LLM capability in security domain."
     },
     {
       "title": "Machine learning techniques for IoT security: Current research and future vision with generative AI and large language models",
       "authors": ["F. Alwahedi", "A. Aldhaheri", "M. A. Ferrag", "A. Battah", "N. Tihanyi"],
       "year": 2024,
       "relevance": "Reviews ML and GenAI techniques for IoT security, relevant to AI-assisted security across software domains."
     },
     {
       "title": "Accelerating Software Quality: Unleashing the Power of Generative AI for Automated Test-Case Generation and Bug Identification",
       "authors": ["Y. Bajaj", "M. Samal"],
       "year": 2023,
       "relevance": "Examines GenAI for automated test generation and bug identification, relevant to AI-assisted code quality."
     },
     {
       "title": "Transforming Software Development with Generative AI: Empirical Insights on Collaboration and Workflow",
       "authors": ["R. Ulfsnes", "N. Moe", "V. Stray", "M. Skarpen"],
       "year": 2024,
       "relevance": "Empirical study on GenAI's impact on software development collaboration and workflow."
     },
     {
       "title": "Systematic Literature Review on Security Risks and its Practices in Secure Software Development",
       "authors": ["R. A. Khan", "S. U. Khan", "H. U. Khan", "M. Ilyas"],
       "year": 2022,
       "relevance": "Prior SLR on security risks in software development, serves as methodological precedent for this work."
     },
     {
       "title": "Generative AI-Enhanced Cybersecurity Framework for Enterprise Data Privacy Management",
       "authors": ["G. S. Nadella", "S. R. Addula", "A. R. Yadulla", "G. S. Sajja", "M. Meesala", "M. H. Maturi"],
       "year": 2025,
       "relevance": "Proposes a GenAI-enhanced cybersecurity framework relevant to enterprise software security practices."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The taxonomy of 109 practices could serve as a reference checklist, but many items are generic and lack actionable detail for practitioners."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Findings confirm the expected narrative that GenAI can help with software security; no surprising or contrarian conclusions."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Discusses cybersecurity risks in software development but does not reveal novel attack vectors or raise new safety concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to existing claims or institutions."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is provided. The output is a literature-derived taxonomy."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors from Jouf University and University of Malakand; no major AI lab or well-known brand association."
     }
   }
 }