ai-research-survey

scan.json (31245B)

---

 {
   "paper": {
     "title": "LLM-based Vulnerability Detection at Project Scale: An Empirical Study",
     "authors": ["Fengjie Li", "Jiajun Jiang", "Dongchi Chen", "Yingfei Xiong"],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.19239",
     "doi": "10.48550/arXiv.2601.19239"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM-based vulnerability detectors achieve low recall (21.09% for C/C++, 33.82% for Java) on 222 known vulnerabilities but uncover more unique vulnerabilities than traditional static analyzers. Both LLM-based and traditional tools suffer from very high false discovery rates (best-performing tool averages 85.3% SFDR) when applied to 24 real-world open-source projects. Manual analysis of 385 sampled reports reveals shallow interprocedural reasoning (37.5%) and imprecise source/sink identification (19.0%) as the dominant false positive causes. LLM-based detection is computationally expensive, consuming up to hundreds of millions of tokens and requiring multi-day runtimes per project.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'we release all experimental artifacts, including evaluation scripts, prompts, taxonomy labels, and detailed statistics, on our project homepage: https://github.com/Feng-Jay/LLM4Security' (Section I, final contribution bullet)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The in-house benchmark is drawn from three publicly available sources (ReposVul, CWE-Bench-Java, JLeaks), and the authors release taxonomy labels and detailed statistics on their homepage. The real-world project commits are specified in Table III."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV specifies hardware (dual Intel Xeon 6388 CPUs, 512 GB RAM, four NVIDIA A800 GPUs, Ubuntu 20.04.6 LTS) but provides no software dependency specifications such as library versions, requirements.txt, or Docker configuration."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases evaluation scripts on the project homepage and provides detailed experimental setup in Section IV, including how to configure each tool. Specific commit hashes for all evaluated projects are listed in Table III."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables IV-VI report recall percentages and SFDR as point estimates only. No confidence intervals or error bars are provided for any results, despite the LLM-based tools having stochastic outputs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'LLM-based methods achieve an average recall of 21.09%' vs traditional tools achieving near zero) based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "While no formal effect sizes (Cohen's d, etc.) are reported, the paper provides detailed absolute recall and SFDR values with per-CWE breakdowns (Tables IV-VI) and per-tool comparisons that convey the magnitude of differences. E.g., 'RepoAudit 55.00% vs KNighter 0% on CWE-401.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 222 vulnerabilities, 24 projects, or 385 sampled reports constitute adequate sample sizes. No power analysis is discussed. The JLeaks sample of 50 cases is described as 'randomly sampled' but the number is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No variance, standard deviation, or spread measures are reported across experimental runs, despite LLM-based tools using non-deterministic API calls."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 5 LLM-based vulnerability detectors against 2 widely-used traditional static analyzers (CodeQL and Semgrep) as baselines across all experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated tools are recent: RepoAudit (ICML 2025), KNighter (SOSP 2025), IRIS (ICLR 2025), LLMDFA (NeurIPS 2024), INFERROI (ICSE 2025). CodeQL and Semgrep are actively maintained industry-standard tools."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper could have controlled tool parameters (e.g., varying RepoAudit's call-graph exploration depth, or testing tools with/without validation steps) to isolate which design factors most affect performance, but does not do so."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: Recall on the in-house dataset (Section V-A), SFDR on real-world projects (Section V-B), qualitative FP taxonomy (Section V-C), and token/time overhead measurements (Section V-D)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two authors with security and software engineering backgrounds independently examined 385 sampled warnings, classifying each as true positive or false positive, requiring more than 150 human hours. Disagreements were resolved through discussion (Section III-D)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly constructs a real-world dataset separate from the in-house benchmark 'to complement the in-house dataset and avoid data leakage' (Section III-C). The real-world projects use the latest commits not present in the tools' original evaluations."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables IV-VI provide per-CWE-type breakdowns for each tool. Figure 5 shows per-FP-reason distributions. Tables V-VI break down results per project within each CWE type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis in Section V-C with a taxonomy of FP causes (Figure 4), quantified distribution across tools (Figure 5), and detailed illustrative examples (Figures 2, 6-13, Appendix A)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper is primarily about negative findings: low recall, high false discovery rates, and computational infeasibility. All four key findings (Findings 1-4) highlight limitations of current tools."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about low recall, high false discovery rates, shallow interprocedural reasoning as root cause, and substantial computational costs are all supported by detailed results in Sections V-A through V-D and corresponding tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Root cause claims (e.g., 'shallow dataflow reasoning and misidentified source/sink pairs as primary failure causes') are justified through systematic manual analysis of 385 sampled reports using inductive open-coding with double coding by two independent experts (Section V-C)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B (External Validity) explicitly bounds findings to the 5 LLM-based and 2 traditional tools evaluated, C/C++ and Java only, and specific CWE types. The paper notes results 'may not generalize uniformly to systems written in Rust, Go, TypeScript, or large multi-language projects.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B discusses internal validity threats (labeling bias from personal experience, code misinterpretation) and external validity threats (tool selection limitations, language coverage). Section V-C systematically considers multiple root causes for failures rather than attributing to a single factor."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's measurements (recall, SFDR, token usage, time) directly measure what is claimed. Recall measures detection capability; SFDR measures false alarm burden; overhead measures computational cost. The paper does not frame these as proxies for broader claims beyond what was measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table I lists LLM backbones as 'Claude 3.5 Sonnet', 'O3-mini', and 'GPT-4' without API version identifiers or snapshot dates. 'GPT-4' could refer to multiple versions with different capabilities. No API version strings are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that 'prompts' are released as part of experimental artifacts on the project homepage (Section I). Additionally, Figure 9 shows an actual prompt used by IRIS, and the tools' released replication packages include their prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section IV states they 'follow the backbone models and hyperparameters (e.g. function call-chain exploration depth, temperature, and top-p) recommended in the original papers' but does not report the actual values. The reader must consult 5 separate original papers to determine settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B describes each tool's workflow: RepoAudit (multi-agent path exploration with LLM validation), KNighter (LLM-generated CSA checkers), IRIS (LLM-inferred source/sink labels embedded in CodeQL templates), LLMDFA (agent-centric full path exploration), INFERROI (LLM intent inference with CFG exploration). Table I summarizes workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C documents preprocessing: ReposVul filtered to Linux kernel post-2019 for buildability, CWE-Bench-Java filtered to Maven projects, 50 cases randomly sampled from JLeaks, all manually verified. Real-world project selection criteria documented in Section III-C. FP coding procedure described in Section V-C."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B 'Threats to Validity' contains substantive discussion of internal validity (labeling subjectivity, code misinterpretation) and external validity (tool representation, language coverage, evolving models)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B discusses specific threats: classification accuracy influenced by personal experience, selected tools 'may not fully represent all possible architectures, languages, or prompting paradigms,' and 'may not generalize uniformly to systems written in Rust, Go, TypeScript.' Mitigation measures are described (double coding, released artifacts)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states what it does NOT cover: other programming languages (Rust, Go, TypeScript), non-open-source tools, CWE types not in the study, and future models. External validity section explicitly lists what may not generalize."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases 'all experimental artifacts, including evaluation scripts, prompts, taxonomy labels, and detailed statistics' on their project homepage. The in-house dataset sources (ReposVul, CWE-Bench-Java, JLeaks) are publicly available benchmarks."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C details data collection: three source benchmarks with specific filtering criteria (post-2019 Linux kernel for C/C++, Maven projects for CWE-Bench-Java, random sample of 50 from JLeaks), manual verification of correctness, validity, and compilation success."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; the annotators are the authors themselves. Data sources are published benchmarks and publicly available open-source projects with documented selection criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from source benchmarks through filtering criteria to final 222 vulnerabilities is documented (Section III-C). For FP analysis: tool outputs → sampling up to 10 per tool per project → independent double labeling → disagreement resolution → final labels (Section III-D, V-C)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding, grants, or acknowledgments section is present in the paper. Chinese academic papers typically have NSFC or other funding sources, making the absence notable."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Tianjin University (College of Intelligence and Computing), International Joint Institute of Tianjin University, and Peking University (Key Laboratory of High Confidence Software Technologies). None are affiliated with the companies whose tools are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, independence cannot be assessed. The authors are academic researchers from Chinese universities evaluating third-party tools, suggesting no obvious financial conflict, but the absence of a funding statement prevents verification."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the LLMs used (GPT-4, Claude 3.5 Sonnet, O3-mini). The in-house benchmark contains known CVEs from published databases that the LLMs may have encountered during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions constructing a real-world dataset 'to avoid data leakage' (Section III-C) but does not discuss whether the LLMs used by the tools may have been trained on the specific CVEs in the in-house benchmark (sourced from public databases like ReposVul)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The in-house benchmark uses CVEs from published databases (ReposVul, CWE-Bench-Java, JLeaks) that were publicly available before the training cutoffs of GPT-4, Claude 3.5 Sonnet, and O3-mini. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The two annotators labeling false positives are the paper's own authors, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates software tools on open-source codebases."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The annotators are described only as 'two authors with a background in security and software engineering.'"
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Inclusion/exclusion criteria for projects and vulnerabilities are documented separately under data_integrity."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The annotators necessarily knew which tool produced each warning during FP labeling."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table VII reports min/max/avg input tokens, output tokens, and runtime (minutes) per tool. Tables X-XI provide per-project breakdowns. E.g., RepoAudit consumes up to 225M input tokens and requires up to 2,450 minutes per project."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV specifies hardware (dual Intel Xeon 6388, 512GB RAM, four NVIDIA A800 GPUs). Section V-D and Tables VII/X/XI quantify total token consumption and wall-clock time for all experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple-seed results are reported. LLM-based tools use stochastic APIs (temperature/top-p settings) but results are reported from what appears to be a single run per tool per project."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each experiment was run. Given the stochastic nature of LLM outputs, this omission is significant."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper uses default settings from original tools ('follow the backbone models and hyperparameters recommended in the original papers') but does not quantify this as a search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section IV states they use released replication packages with recommended hyperparameters from the original papers, providing a clear and principled rationale for configuration selection rather than cherry-picking the best result."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims across 7 tools × 8 CWE types × multiple metrics without any statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential bias from how they configured or ran the evaluated tools. While they use released packages (mitigating re-implementation bias), they do not acknowledge or discuss this as a consideration per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D explicitly analyzes the tradeoff between computational cost and detection effectiveness. Table VII and the discussion note that tools with highest token consumption (RepoAudit, LLMDFA) do not necessarily achieve the best detection rates."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its in-house benchmark of 222 vulnerabilities across 8 CWE types adequately represents 'real-world vulnerability detection capability.' The representativeness and coverage of the selected CWE types relative to the broader vulnerability landscape is not analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Each tool is evaluated as a complete bundled product with its own scaffold and LLM backbone. The paper compares tools as integrated systems (Table I), not isolated models, so the scaffold IS the thing being tested."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The real-world dataset uses 'latest versions' to reduce leakage, but the in-house benchmark contains published CVEs from databases like ReposVul that were available before the LLMs' training cutoffs. No temporal analysis of when benchmark data was published relative to model training is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, tools are provided with source-sink pairs for the in-house dataset (Section IV), which could leak information about vulnerability locations."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap or non-independence between the three in-house benchmark sources (ReposVul, CWE-Bench-Java, JLeaks) or between the in-house and real-world datasets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. The paper mentions constructing a real-world dataset 'to avoid data leakage' but does not employ any systematic leakage detection (canary strings, membership inference, etc.)."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-based methods exhibit low average recall on known vulnerabilities: 21.09% for C/C++ and 33.82% for Java, but still uncover more unique vulnerabilities than traditional tools.",
       "evidence": "Table IV shows per-CWE recall for all tools. Figure 3 shows unique vulnerability detections per tool, with LLM-based tools contributing substantially more unique detections (e.g., IRIS found 23 unique CWE-722 cases). Section V-A.",
       "supported": "strong"
     },
     {
       "claim": "Both LLM-based and traditional tools exhibit very high false discovery rates in real-world projects, with even the best-performing tool averaging 85.3% SFDR.",
       "evidence": "Tables V-VI report SFDR per tool per project. RepoAudit averages 97.0% SFDR on C/C++, IRIS averages 94.4% on Java. Based on manual examination of 385 sampled warnings requiring >150 human hours. Section V-B.",
       "supported": "strong"
     },
     {
       "claim": "Shallow interprocedural reasoning (37.5%) and imprecise source/sink identification (19.0%) are the dominant root causes of false positives across all tools.",
       "evidence": "Figure 5 quantifies FP reason distribution: A1 (shallow interprocedural) accounts for 136/363 FPs, B1 (incorrect source/sink) accounts for 69/363. Detailed examples in Figures 6-13 and Appendix A. Section V-C.",
       "supported": "strong"
     },
     {
       "claim": "Project-scale LLM-based detection requires up to hundreds of millions of tokens and multi-day runtimes, making scalability a critical bottleneck.",
       "evidence": "Table VII shows RepoAudit consuming up to 225M input tokens and 2,450 minutes; LLMDFA up to 38M input tokens and 4,638 minutes. Per-project details in Tables X-XI. Section V-D.",
       "supported": "strong"
     },
     {
       "claim": "Most missed detections in traditional tools and KNighter stem from source-sink specification mismatches with project-specific APIs.",
       "evidence": "Figure 1 shows 62/64, 64/64, and 64/64 missed C/C++ vulnerabilities for CodeQL, Semgrep, and KNighter respectively are due to source-sink mismatch. Figure 2 provides a detailed illustrative example. Section V-A.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance on stochastic results",
       "detail": "LLM-based tools use non-deterministic API calls (temperature, sampling), yet all results are reported from apparently single runs without any uncertainty quantification, variance, or multi-run averaging."
     },
     {
       "flag": "Small in-house benchmark without size justification",
       "detail": "The in-house benchmark contains only 222 vulnerabilities across 8 CWE types (some CWEs have only 10-14 cases), with no power analysis or justification for why this sample size is sufficient for the comparative claims made."
     },
     {
       "flag": "Author-annotated FP labels with potential bias",
       "detail": "False positive labeling was performed by two of the paper's own authors. While they used double coding with disagreement resolution, independent external annotators would strengthen the validity of the FP taxonomy, especially given >150 hours of effort."
     },
     {
       "flag": "Contamination risk for in-house benchmark",
       "detail": "The in-house dataset uses known CVEs from published databases (ReposVul, CWE-Bench-Java). The LLMs powering the evaluated tools (GPT-4, Claude 3.5 Sonnet) may have been trained on information about these CVEs, potentially inflating or deflating recall in unpredictable ways."
     },
     {
       "flag": "No funding disclosure",
       "detail": "The paper contains no funding acknowledgments or competing interests statement, which is unusual for an academic paper from major Chinese universities."
     }
   ],
   "cited_papers": [
     {
       "title": "GPTScan: Detecting Logic Vulnerabilities in Smart Contracts by Combining GPT with Program Analysis",
       "authors": ["Y. Sun", "D. Wu", "Y. Xue"],
       "year": 2024,
       "relevance": "Combines LLM reasoning with program analysis for vulnerability detection, a key approach in the LLM-augmented code analysis space."
     },
     {
       "title": "LLM-Assisted Static Analysis for Detecting Security Vulnerabilities",
       "authors": ["Z. Li", "S. Dutta", "M. Naik"],
       "year": 2025,
       "relevance": "Proposes IRIS, one of the five evaluated LLM-based vulnerability detectors, demonstrating LLM-CodeQL integration for project-scale analysis."
     },
     {
       "title": "RepoAudit: An Autonomous LLM-Agent for Repository-Level Code Auditing",
       "authors": ["J. Guo", "C. Wang", "X. Xu"],
       "year": 2025,
       "relevance": "Multi-agent LLM framework for repository-level vulnerability detection, one of the primary tools evaluated in this study."
     },
     {
       "title": "KNighter: Transforming Static Analysis with LLM-Synthesized Checkers",
       "authors": ["C. Yang", "Z. Zhao", "Z. Xie"],
       "year": 2025,
       "relevance": "Uses LLMs to automatically generate static analysis checkers, demonstrating LLM-augmented tool synthesis for vulnerability detection."
     },
     {
       "title": "LLMDFA: Analyzing Dataflow in Code with Large Language Models",
       "authors": ["C. Wang", "W. Zhang", "Z. Su"],
       "year": 2024,
       "relevance": "Agent-centric LLM approach for dataflow analysis in vulnerability detection, evaluated in this study."
     },
     {
       "title": "Exploring ChatGPT's Capabilities on Vulnerability Management",
       "authors": ["P. Liu", "J. Liu", "L. Fu"],
       "year": 2024,
       "relevance": "Evaluates general-purpose LLM capabilities for vulnerability detection, providing context on LLM hallucination and reasoning limitations."
     },
     {
       "title": "LLMs Cannot Reliably Identify and Reason About Security Vulnerabilities (Yet?)",
       "authors": ["S. Ullah", "M. Han", "S. Pujar"],
       "year": 2024,
       "relevance": "Comprehensive evaluation of LLM vulnerability detection capabilities showing fundamental reasoning limitations."
     },
     {
       "title": "VulEval: Towards Repository-Level Evaluation of Software Vulnerability Detection",
       "authors": ["X.-C. Wen", "X. Wang", "Y. Chen"],
       "year": 2024,
       "relevance": "Repository-level evaluation framework for LLM-based vulnerability detection, directly relevant to project-scale assessment."
     },
     {
       "title": "Top Score on the Wrong Exam: On Benchmarking in Machine Learning for Vulnerability Detection",
       "authors": ["N. Risse", "J. Liu", "M. Böhme"],
       "year": 2025,
       "relevance": "Critiques benchmarking practices in ML vulnerability detection, addressing construct validity concerns relevant to this survey's methodology assessment."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Q. Wu", "G. Bansal", "J. Zhang"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework foundational to agentic vulnerability detection workflows like RepoAudit."
     },
     {
       "title": "An Empirical Study of Static Analysis Tools for Secure Code Review",
       "authors": ["W. Charoenwet", "P. Thongtanunam", "V.-T. Pham"],
       "year": 2024,
       "relevance": "Prior empirical study of static analysis tools for security, providing methodological precedent for this study's approach."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, the backbone LLM used by three of the five evaluated vulnerability detectors."
     }
   ]
 }