ai-research-survey

scan.json (24197B)

---

 {
   "paper": {
     "title": "LLMs in Software Security: A Survey of Vulnerability Detection Techniques and Insights",
     "authors": [
       "Ze Sheng",
       "Zhicheng Chen",
       "Shuning Gu",
       "Heqing Huang",
       "Guofei Gu",
       "Jeff Huang"
     ],
     "year": 2025,
     "venue": "ACM Computing Surveys",
     "arxiv_id": "2502.07049",
     "doi": "10.1145/3769082"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey of 58 papers on LLM-based vulnerability detection finds that decoder-only models (especially GPT-4 and GPT-3.5) account for 67.1% of model usage, C/C++ dominates as the target language at 50% of studies, and 41.3% of studies use code preprocessing techniques (AST, RAG, program slicing). Key gaps identified include narrow focus on function-level binary classification, lack of repository-level datasets, dataset quality/leakage issues, and limited cross-language and cross-file detection capabilities. Fine-tuned large models can achieve near 0.9 F1-scores, but complex real-world vulnerability detection remains challenging.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL: https://github.com/OwenSanzas/LLM-For-Vulnerability-Detection, stated in the abstract as maintaining 'the latest findings.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "While the GitHub repo is mentioned, the paper does not describe releasing a structured dataset of its survey extraction results (e.g., the coded features of all 58 papers, screening decisions, or the full list of ~500 screened papers)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a survey, analysis scripts could have been released with environment details, but none are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the survey analysis are provided. The paper describes the search process at a high level but not with enough detail to fully replicate."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments; only descriptive statistics (counts, percentages) are reported."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no comparative experiments requiring significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments; reports descriptive summaries of others' results."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments or power analysis context."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental runs to report variance over."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 compares this survey against prior related reviews: Yao et al. [129], Xu et al. [124], Zhou et al. [140], and several others, explicitly noting how this survey differs."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared prior surveys (Zhou et al. 2024, Xu et al. 2024, Yao et al. 2024) are contemporary, published in 2024."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system or components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no evaluation metrics applied to the authors' own analysis."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to a literature survey's claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive breakdowns: by programming language (Figure 5, Finding I), by model architecture (Tables 1-2, RQ1), by technique type (RQ3), by dataset scope (Table 4), and by challenge category (RQ4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 (RQ4) extensively discusses challenges and failure modes: limited repository-level coverage, complexity of vulnerability semantics, intrinsic LLM limitations (inconsistent explanations, lack of robustness), and dataset quality issues."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: Ding et al. achieving only 0.21 F1-score, Guo et al. achieving 0.099 F1 on PrimeVul, LLMs lacking robustness against data perturbations (Yin et al.), and inconsistent vulnerability explanations (Haurogne et al., Du et al.)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) systematic analysis, (2) unified framework, and (3) identification of challenges are all substantiated by the paper's four research questions and corresponding sections."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that synthesizes findings from other papers. It does not make independent causal claims from its own analysis; it reports what other papers found."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 clearly bounds the scope: 'This survey focuses exclusively on the application of LLMs in vulnerability detection' with explicit exclusion criteria (no traditional ML, no malware/intrusion detection, 2019-2024 timeframe)."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey/taxonomy paper; the paper synthesizes existing literature rather than presenting empirical results that require alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no measurements of its own; it reports proxies and outcomes from other papers."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; the authors do not use any models themselves."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no prompting is used by the authors."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring hyperparameter reporting."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.1 describes a search starting from top-tier conferences, keyword extraction, and iterative refinement from ~500 papers to 58. However, intermediate filtering stages with counts at each step are not provided, and the specific criteria for advancing papers between stages are vaguely described ('ensuring recency and relevance')."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 is a dedicated 'LIMITATIONS' section discussing factors that may affect the survey's comprehensiveness."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: ~60% of research appears as preprints on arXiv (not peer-reviewed), and terminology variations ('LLM', 'vulnerability detection') may cause search oversights. These are specific to this study's methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly states what is excluded: 'traditional machine learning approaches (conventional CNN, RNN and LSTM),' 'malware analysis or network intrusion detection,' and papers outside the 2019-2024 window."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub repo is provided for 'latest findings,' the paper does not describe releasing the raw survey data: the full list of ~500 screened papers, screening decisions, or the complete coded extraction data for each paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the data collection: starting from top-tier security conferences (S&P, USENIX Security, CCS), extracting key terms, conducting iterative searches every three weeks over two months, screening ~500-600 papers to select 58."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes how papers were found: starting from top-tier security conferences and journals, keyword-based iterative search, with explicit inclusion/exclusion criteria (LLMs in vulnerability detection, 2019-2024, excluding traditional ML methods)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes going from ~500-600 papers to 58 'highly relevant studies' but does not document intermediate filtering stages with counts at each step. The jump from 500 to 58 is not explained with per-stage filtering decisions."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no disclosure of funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Texas A&M University (5 authors) and City University of Hong Kong (1 author), with full email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of funders cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; the authors do not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no model evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no model evaluation is performed. The paper does discuss contamination as a challenge facing the field (Challenge 4, dataset leakage), but this is about surveyed papers, not the survey's own methodology."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no method or system of its own to report costs for."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no computational experiments performed."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, no registered protocol, and no reproducible search queries (exact database queries, date ranges per database). The search is described narratively in Section 2.1 with keywords and venue names, but falls short of a structured protocol. Figure 2 shows the survey structure but is not a PRISMA diagram."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of the 58 included papers. All papers are treated equally regardless of venue, study design rigor, or potential biases. Performance numbers from fine-tuning papers (Table 5) are reported without noting differences in evaluation rigor across studies."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section notes that ~60% of included research appears as preprints on arXiv, but this is framed as a comprehensiveness issue rather than a discussion of publication bias (i.e., whether the literature skews toward positive results, whether negative findings are underrepresented)."
       }
     }
   },
   "claims": [
     {
       "claim": "Decoder-only models account for 67.1% of total LLM usage in vulnerability detection, with GPT-4 being the most frequently used model (29 instances across 58 studies).",
       "evidence": "Section 3.2 (RQ1) reports statistics from analyzing 58 studies: 33 distinct LLMs identified, GPT-4 in 29 instances, GPT-3.5 in 25, encoder-only at 24.2%, encoder-decoder at 8.7%, decoder-only at 67.1%.",
       "supported": "moderate"
     },
     {
       "claim": "C/C++ dominates as the target language in LLM-based vulnerability detection research at 50% of studies, followed by Java at 21.1% and Solidity at 11.8%.",
       "evidence": "Finding I, Section 3.3, Figure 5 present the distribution across 56 selected papers analyzing target programming languages.",
       "supported": "moderate"
     },
     {
       "claim": "41.3% of studies employ code processing techniques (graph representations, RAG, code slicing) to enhance LLM context utilization.",
       "evidence": "Finding III in Section 3.4.1 states this percentage, though the calculation method is not detailed.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuned large models (>10B parameters with PEFT) achieve F1 scores near 0.9 for vulnerability detection.",
       "evidence": "Finding V and Table 5 report various F1 scores: Alam et al. GPT-4o-mini at 0.99, Guo et al. CodeLlama-7B at 0.97, Ma et al. CodeLlama-13B at 0.91. However, these are reported from individual papers with heterogeneous datasets and evaluation setups.",
       "supported": "weak"
     },
     {
       "claim": "Approximately 83% of current studies focus on isolated code snippets rather than repository-level analysis.",
       "evidence": "Challenge 1 in Section 3.5 states 'approximately 40 studies (83%) concentrate on the analysis of isolated code snippets.'",
       "supported": "moderate"
     },
     {
       "claim": "Current datasets face critical limitations: C/C++ language imbalance (~60% coverage) and significant shortage of repository-level datasets.",
       "evidence": "Finding II in Section 3.3 and Table 4 showing dataset distribution. Table 4 lists 42 datasets, with the vast majority at function or file level, and only 2-3 repository-level datasets.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought prompting is the dominant approach for large models (>10B parameters), with 100% of recent studies adopting CoT.",
       "evidence": "Finding IV in Section 3.4.2 makes this claim, but the denominator and time frame for '100% of recent studies' is not specified.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of surveyed papers",
       "detail": "The survey treats all 58 included papers equally without assessing their methodological quality. F1 scores from different papers using different datasets and evaluation protocols are presented side-by-side (Table 5) without noting that these numbers are not directly comparable. This risks laundering weak results alongside strong ones."
     },
     {
       "flag": "Vague and unreproducible selection process",
       "detail": "The paper selection goes from ~500-600 to 58 papers without documenting intermediate screening stages, exact search queries, or specific databases searched. The process is described as 'iterative searches every three weeks' with 'key terms' but exact queries and per-database results are not provided."
     },
     {
       "flag": "Uncritical aggregation of incomparable results",
       "detail": "Performance numbers from different studies using different datasets, different versions of models, different evaluation splits, and different preprocessing are aggregated into findings (e.g., Finding V claims near-0.9 F1). Alam et al.'s 0.99 F1 on their own VulSmart dataset is presented alongside Ding et al.'s 0.21 F1 on PrimeVul without discussing that dataset difficulty and evaluation rigor drive these differences."
     },
     {
       "flag": "Imprecise quantitative claims",
       "detail": "Several findings use precise-sounding percentages (41.3%, 67.1%, 24.2%) derived from a small sample of 58 papers, but the denominator and calculation methodology are often unstated. Finding IV claims '100% of recent studies adopting CoT' without defining 'recent' or the sample size."
     }
   ],
   "cited_papers": [
     {
       "title": "Vulnerability Detection with Code Language Models: How Far Are We?",
       "authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim"],
       "year": 2025,
       "relevance": "ICSE paper systematically evaluating code LLMs for vulnerability detection, finding low F1-scores even with fine-tuning on PrimeVul."
     },
     {
       "title": "LLMs Cannot Reliably Identify and Reason About Security Vulnerabilities (Yet?): A Comprehensive Evaluation, Framework, and Benchmarks",
       "authors": ["Saad Ullah", "Mingji Han", "Saurabh Pujar"],
       "year": 2024,
       "relevance": "IEEE S&P paper providing a comprehensive evaluation framework demonstrating LLM limitations in security vulnerability identification."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Seminal Codex paper evaluating LLMs trained on code, establishing foundational evaluation methodology for code generation."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2024,
       "relevance": "Comprehensive systematic review of LLMs in software engineering, covering a broader scope than this vulnerability-focused survey."
     },
     {
       "title": "Large Language Model for Vulnerability Detection and Repair: Literature Review and Roadmap",
       "authors": ["Xin Zhou", "Sicong Cao", "Xiaobing Sun", "David Lo"],
       "year": 2024,
       "arxiv_id": "2404.02525",
       "relevance": "Closely related survey on LLMs for vulnerability detection and repair; this paper positions itself as an update and extension."
     },
     {
       "title": "LLM agents can autonomously exploit one-day vulnerabilities",
       "authors": ["Richard Fang", "Rohan Bindu", "Akul Gupta", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2404.08144",
       "relevance": "Demonstrates LLM agents' capability to autonomously exploit real-world vulnerabilities, relevant to agentic AI safety assessment."
     },
     {
       "title": "A survey on large language model (LLM) security and privacy: The Good, The Bad, and The Ugly",
       "authors": ["Yifan Yao", "Jinhao Duan", "Kaidi Xu"],
       "year": 2024,
       "relevance": "Broad survey of LLM security and privacy impacts, providing context for this survey's narrower vulnerability detection focus."
     },
     {
       "title": "Large Language Models for Code: Security Hardening and Adversarial Testing",
       "authors": ["Jingxuan He", "Martin Vechev"],
       "year": 2023,
       "relevance": "CCS paper on LLM code security hardening and adversarial testing, directly related to LLM safety in code generation."
     },
     {
       "title": "LLM4Vuln: A unified evaluation framework for decoupling and enhancing LLMs' vulnerability reasoning",
       "authors": ["Yuqiang Sun", "Daoyuan Wu", "Yue Xue"],
       "year": 2024,
       "arxiv_id": "2401.16185",
       "relevance": "Proposes a unified evaluation framework for LLM vulnerability reasoning, including RAG-based enhancement approaches."
     },
     {
       "title": "How far have we gone in vulnerability detection using large language models",
       "authors": ["Zeyu Gao", "Hao Wang", "Yuchen Zhou"],
       "year": 2023,
       "arxiv_id": "2311.12420",
       "relevance": "Evaluates LLM capability boundaries in vulnerability detection, finding significant performance gaps in complex scenarios."
     },
     {
       "title": "Multitask-Based Evaluation of Open-Source LLM on Software Vulnerability",
       "authors": ["Xin Yin", "Chao Ni", "Shaohua Wang"],
       "year": 2024,
       "relevance": "IEEE TSE paper evaluating open-source LLMs on multiple vulnerability-related tasks, finding lack of robustness."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper cited to support claims about larger models improving vulnerability detection capabilities."
     }
   ]
 }