ai-research-survey

scan-v5.json (28161B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "LLM-based Vulnerability Detection at Project Scale: An Empirical Study",
     "authors": [
       "Fengjie Li",
       "Jiajun Jiang",
       "Dongchi Chen",
       "Yingfei Xiong"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.19239",
     "doi": "10.48550/arXiv.2601.19239"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims—low recall (21.09%/33.82%), high false discovery rates (best tool at 85.3% SFDR), and high computational costs (up to hundreds of millions of tokens, multi-day runtimes)—are directly supported by Tables IV–VII and quantitative results.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims specific root causes (shallow interprocedural reasoning, imprecise source/sink identification) account for 37.47% and 19.00% of FPs, but this taxonomy was built by 2 authors with no reported inter-rater reliability statistics (no Cohen's kappa or agreement rate), undermining the causal attribution.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Threats to validity section explicitly acknowledges evaluation covers only 5 tools, 2 languages (Java/C++), 8 CWE types, and notes results may not generalize to Rust, Go, or TypeScript systems.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes failures primarily to tool design choices without considering alternative explanations such as potential tool misconfiguration (using default rather than tuned settings), benchmark difficulty relative to original evaluation settings, or API-level LLM behavior differences.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly explains that SFDR is a sampled estimate (up to 10 warnings per tool per project), that recall is measured only on labeled vulnerable points, and that real-world projects have unknown ground truth requiring manual sampling rather than full measurement.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.B 'Threats to Validity' provides a dedicated discussion of internal validity (expert judgment bias, labeling accuracy) and external validity (tool selection, language/CWE coverage, evolving field).",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats named include: personal experience influencing FP taxonomy coding, selection of only open-source/runnable tools, evaluation limited to Java and C/C++, and lack of generalizability to Rust/Go/TypeScript multi-language projects.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "External validity section explicitly states results may not represent 'all possible architectures, languages, or prompting paradigms' and that custom enterprise-scale deployments may behave differently.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper; no acknowledgments section is present.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Tianjin University (Li, Jiang), International Joint Institute of Tianjin University/Fuzhou (Chen), and Peking University Key Lab (Xiong), with email addresses provided.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding disclosed; cannot assess funder independence.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or financial interest declaration appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined precisely: source/sink, taint analysis, SFDR (sampled false discovery rate), recall formula provided, CWE types defined via MITRE classification, and each tool's workflow is explained.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Five explicit contributions are bulleted at the end of the introduction: first project-scale empirical comparison, systematic detection evaluation, FP taxonomy with 385 labeled reports, overhead measurement, and artifact release.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section II and III.C systematically compare prior work and explain how this study differs—prior work evaluates at function/hunk level with incomplete project context, whereas this study evaluates at full project scale with real-world settings.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Paper explicitly states: 'we release all experimental artifacts, including evaluation scripts, prompts, taxonomy labels, and detailed statistics, on our project homepage: https://github.com/Feng-Jay/LLM4Security.'",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "In-house benchmark built from three public datasets (ReposVul, CWE-Bench-Java, JLeaks); real-world projects at specific commits are identified in Table III; manual FP taxonomy labels are released on GitHub.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Server hardware is described (Intel Xeon 6388, 512GB RAM, NVIDIA A800, Ubuntu 20.04.6 LTS) but no formal dependency specifications (requirements.txt, Dockerfile) are provided by the authors.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Paper provides only a URL pointer to GitHub; no step-by-step reproduction instructions appear in the paper itself, and completeness of the GitHub documentation cannot be verified from the paper.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported for any recall percentages or SFDR values; results are presented as point estimates only.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are used for comparative claims between tools (e.g., that RepoAudit outperforms CodeQL), despite multiple pairwise comparisons being made throughout the paper.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Percentage differences in recall and SFDR across tools are explicitly reported with absolute baselines (e.g., RepoAudit 55.00% recall vs. 0.00% for CodeQL/Semgrep/KNighter on CWE-401).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification is provided for the choice of 222 vulnerabilities, 24 projects, or 'up to 10 warnings per tool per project'; no power analysis or discussion of statistical adequacy.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Overhead table (Table VII) reports min/max/avg but not standard deviation; recall results are single point estimates per CWE type with no measure of spread.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Two traditional static analysis tools (CodeQL and Semgrep) serve as explicit baselines against which all LLM-based methods are compared throughout Tables IV–VI.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "CodeQL and Semgrep are industry-standard, actively maintained tools widely used in practice; they represent the current state of traditional static analysis.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": false,
           "answer": false,
           "justification": "Paper evaluates existing tools rather than proposing new components; no ablation is applicable.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics used: recall on in-house benchmark, SFDR on real-world projects, number of reports, files affected, input/output token consumption, and end-to-end runtime.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Two authors independently labeled 385 sampled warnings as true/false positives requiring 150+ human hours, with disagreements resolved through discussion.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Two distinct evaluation settings: an in-house benchmark with known ground truth and 24 separate real-world projects with no pre-labeled vulnerabilities, providing an out-of-distribution evaluation.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results broken down by CWE type (Tables IV–VI) and by individual project, with per-tool per-project SFDR values reported.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "RQ3 constructs a taxonomy of 8 FP cause categories with quantified distribution across tools (Figure 5), and Appendix A provides illustrated code-level examples for all 9 FP reason types.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The core findings are negative: low recall (21–34%), near-universal high FDR (85–100% SFDR), and severe scalability failures are the primary reported results.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Table I lists 'Claude 3.5 Sonnet,' 'O3-mini,' and 'GPT-4' without specifying snapshot dates or version hashes for any of these models, which can vary significantly in capability.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Example prompts shown inline in Figures 8–9 illustrate actual LLM instructions; paper states all prompts are released on GitHub homepage.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Paper states it follows 'hyperparameters recommended in the original papers' but does not report actual values for temperature, top-p, call-chain exploration depth, or any other parameter.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Section III.B describes each tool's workflow architecture in detail: multi-agent path exploration (RepoAudit), LLM+CSA checker generation (KNighter), LLM+CodeQL source/sink inference (IRIS), iterative dataflow tracing (LLMDFA), and CFG-guided intent inference (INFERROI).",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "In-house dataset construction described in Section III.C including source filtering, compilation verification, and manual checking; real-world project selection criteria (active maintenance, historical vulnerability types, buildability) are documented.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "GitHub release explicitly includes taxonomy labels, detailed statistics, and evaluation scripts; raw detection results and manual annotations are stated to be publicly available.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section III.C describes the construction of the in-house dataset from three source benchmarks with filtering criteria, and real-world project selection process with 24 projects at specific commits.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants recruited; study uses code repositories and existing public benchmark datasets.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Full pipeline documented: source benchmark collection → filtering/verification → tool execution → sampling (up to 10 per tool per project) → independent labeling → disagreement resolution through discussion.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training cutoffs for GPT-4, Claude 3.5 Sonnet, and O3-mini are not stated anywhere in the paper despite these models being central to the evaluated tools.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "While authors mention constructing the real-world dataset 'to avoid data leakage,' they do not discuss whether LLMs may have been trained on the public benchmark vulnerabilities (ReposVul, CWE-Bench-Java, JLeaks) or open-source projects used in evaluation.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Public benchmark datasets (ReposVul published ICSE 2024, CWE-Bench-Java, JLeaks) predate some LLM training cutoffs and are publicly available; no analysis of whether benchmark CVEs appeared in LLM training data is provided.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study; the 2-author manual labeling is researcher coding, not a participant study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table VII provides detailed per-tool min/max/avg input tokens, output tokens, and runtime; Appendix C Tables X–XI give per-project breakdowns; RepoAudit shown to use up to 225M input tokens and 38M output tokens for a single project.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Server specifications fully described (dual Intel Xeon 6388, 512GB RAM, 4x NVIDIA A800, Ubuntu 20.04.6 LTS) and total per-project token consumption and runtimes reported throughout Section V-D.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-based methods achieve low recall on known vulnerabilities: 21.09% average for C/C++ and 33.82% for Java on the in-house benchmark of 222 vulnerabilities.",
       "evidence": "Table IV showing per-CWE detection results across 5 LLM tools and 2 traditional tools; most tools achieve 0% on many CWE types.",
       "supported": "strong"
     },
     {
       "claim": "LLM-based tools uncover more unique vulnerabilities than traditional tools and are complementary to each other.",
       "evidence": "Figure 3 showing unique vulnerabilities per tool with up to 23 unique detections by INFERROI on CWE-722 vs. 0 for traditional tools.",
       "supported": "strong"
     },
     {
       "claim": "Both LLM-based and traditional tools exhibit very high false discovery rates in real-world projects; even the best tool averages 85.3% SFDR.",
       "evidence": "Tables V and VI with SFDR measurements; RepoAudit averages 97.0% SFDR and IRIS averages 94.4% SFDR.",
       "supported": "strong"
     },
     {
       "claim": "Shallow interprocedural reasoning (A1) and imprecise source/sink identification (B1) are the dominant FP causes, accounting for 37.47% and 19.00% of false positives respectively.",
       "evidence": "Figure 5 distribution of FP reasons across tools based on manual coding of 385 sampled reports by 2 authors; inductive open-coding procedure used.",
       "supported": "moderate"
     },
     {
       "claim": "Project-scale LLM-based detection is computationally prohibitive: up to 225M input tokens per project and up to 4,638 minutes (77 hours) for a single project.",
       "evidence": "Table VII overhead measurements and Appendix C Tables X–XI per-project details; LLMDFA averages 2,000 minutes per Java project.",
       "supported": "strong"
     },
     {
       "claim": "Source/sink mismatch is the primary reason for missed detections in the in-house benchmark, accounting for 62/64, 64/64, and 64/64 missed vulnerabilities for CodeQL, Semgrep, and KNighter respectively.",
       "evidence": "Figure 1 showing cause distribution for undetected C/C++ vulnerabilities; Figure 2 illustrating how dev_alloc_skb is not recognized as an allocator.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational",
     "case-study"
   ],
   "key_findings": "LLM-based vulnerability detectors achieve only 21–34% recall on known vulnerabilities and suffer from catastrophically high false discovery rates (85–100% SFDR) when applied to real-world projects, making them currently impractical for deployment. Manual analysis of 385 false positives identified shallow interprocedural dataflow reasoning and imprecise source/sink identification as the dominant failure causes (accounting for over 56% of FPs combined), with LLM-specific failures including hallucinated control flow and prompt non-compliance. Project-scale LLM analysis is computationally prohibitive—up to 225 million input tokens and 77 hours per project—representing a fundamental scalability bottleneck. Despite these limitations, LLM-based tools consistently surface more unique vulnerabilities than traditional static analyzers, suggesting potential value in hybrid or targeted deployment settings.",
   "red_flags": [
     {
       "flag": "No inter-rater reliability statistics",
       "detail": "The entire FP taxonomy and its quantified distribution (Figure 5) rests on manual coding by 2 authors using inductive open-coding, but no Cohen's kappa or inter-rater agreement rate is reported. The central claim about dominant FP causes is therefore unverified."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims between tools (recall differences, SFDR differences, complementarity) are stated without statistical tests or confidence intervals despite the small sample sizes involved."
     },
     {
       "flag": "LLM model versions underspecified",
       "detail": "Table I lists 'GPT-4,' 'Claude 3.5 Sonnet,' and 'O3-mini' without snapshot dates or version hashes; GPT-4 alone encompasses dozens of distinct model checkpoints with significant capability differences."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "Public benchmark datasets (ReposVul, CWE-Bench-Java, JLeaks) and the open-source projects evaluated were publicly available before LLM training cutoffs; no analysis of whether models had seen these vulnerable code patterns during training."
     },
     {
       "flag": "SFDR estimation under-analyzed",
       "detail": "Sampling up to 10 warnings per tool per project produces SFDR estimates with unknown confidence intervals; some cells in Tables V–VI have as few as 1 sampled report, making SFDR claims for those configurations unreliable."
     },
     {
       "flag": "Tool configuration not independently optimized",
       "detail": "Using default hyperparameters 'recommended in original papers' may systematically disadvantage tools on new datasets; the study conflates tool capability with default-configuration performance."
     }
   ],
   "cited_papers": [
     {
       "title": "RepoAudit: An Autonomous LLM-Agent for Repository-Level Code Auditing",
       "relevance": "One of the 5 evaluated LLM-based vulnerability detectors; multi-agent framework central to the study's comparative analysis."
     },
     {
       "title": "LLMDFA: Analyzing Dataflow in Code with Large Language Models",
       "relevance": "One of the 5 evaluated tools; agent-centric multi-path exploration approach evaluated at project scale."
     },
     {
       "title": "LLM-Assisted Static Analysis for Detecting Security Vulnerabilities (IRIS)",
       "relevance": "One of the 5 evaluated tools; uses LLM to infer sources/sinks and augment CodeQL; provides CWE-Bench-Java dataset used in evaluation."
     },
     {
       "title": "KNighter: Transforming Static Analysis with LLM-Synthesized Checkers",
       "relevance": "One of the 5 evaluated tools; generates Clang Static Analyzer checkers from vulnerability-fixing commits."
     },
     {
       "title": "Boosting Static Resource Leak Detection via LLM-Based Resource-Oriented Intention Inference (INFERROI)",
       "relevance": "One of the 5 evaluated tools; CFG-guided LLM approach specialized for CWE-772 resource leaks."
     },
     {
       "title": "VulEval: Towards Repository-Level Evaluation of Software Vulnerability Detection",
       "relevance": "Prior study on LLM vulnerability detection at repository level; directly compared and critiqued in related work for incomplete project-context evaluation."
     },
     {
       "title": "LLMs Cannot Reliably Identify and Reason About Security Vulnerabilities (Yet?): A Comprehensive Evaluation",
       "relevance": "Related empirical study on LLM vulnerability detection limitations; confirms and contextualizes this paper's findings at a different granularity."
     },
     {
       "title": "Top Score on the Wrong Exam: On Benchmarking in Machine Learning for Vulnerability Detection",
       "relevance": "Methodological critique of vulnerability detection benchmarks; cited as motivation for moving beyond function-level evaluation."
     },
     {
       "title": "ReposVul: A Repository-Level High-Quality Vulnerability Dataset",
       "relevance": "Source of C/C++ vulnerabilities for the in-house benchmark; provides ground-truth CVE entries used in RQ1."
     },
     {
       "title": "JLeaks: A Featured Resource Leak Repository Collected from Hundreds of Open-Source Java Projects",
       "relevance": "Source of 50 CWE-772 (resource leak) vulnerabilities for the in-house benchmark used in RQ1 evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable for security practitioners evaluating whether to adopt LLM-based scanning tools; provides concrete cost and accuracy benchmarks across 7 tools."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges widespread optimism about LLM-based security tools with hard numbers showing 85–100% false discovery rates and recalls below 34%, contradicting tool papers' own reported results."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that current LLM security tools are unreliable for real-world deployment, raising concern about organizations falsely believing they are protected by these tools."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly contradicts positive claims from the original tool papers (RepoAudit, IRIS, LLMDFA), but no explicit confrontation or strong controversy angle is developed."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Evaluated tools are publicly available but require expensive LLM API access (up to hundreds of millions of tokens) and multi-day runtimes, making casual reproduction impractical."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses Claude 3.5 Sonnet, GPT-4, and O3-mini as backbones (Anthropic/OpenAI products), but authors are from Chinese universities without major brand recognition."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46781038",
         "title": "An ultra-high-resolution map of (dark) matter",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46781038",
         "created_at": "2026-01-27T15:12:25Z"
       }
     ],
     "top_points": 1,
     "total_points": 1,
     "total_comments": 0
   }
 }