ai-research-survey

scan.json (30602B)

---

 {
   "paper": {
     "title": "Everything You Wanted to Know About LLM-based Vulnerability Detection But Were Afraid to Ask",
     "authors": [
       "Yue Li",
       "Xiao Li",
       "Hao Wu",
       "Minghui Xu",
       "Yue Zhang",
       "Xiuzhen Cheng",
       "Fengyuan Xu",
       "Sheng Zhong"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.13474",
     "doi": "10.48550/arXiv.2504.13474"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our dataset and code are available' with an anonymous link (https://anonymous.4open.science/r/CORRECT) provided in §1. This is an anonymized review link but does provide a working URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset of 2,000 vulnerable-patched program pairs is released at the same anonymous URL. The paper also builds on publicly available sources: MoreFixes, PrimeVul, and ReposVul."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, Dockerfile, library versions) are mentioned in the paper. Only tools used are named (cflow, Joern) without version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the framework methodology but does not include a README or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '67% accuracy', 'precision close to 0.8', 'F1-score exceeds 70%'). No confidence intervals or error bars appear in figures or tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'significantly surpassing the random baseline', 'significantly improved performance') but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context: e.g., precision from ~0.5 (w/o context) to ~0.8 (Strict Mode), (1,0) proportion from 25% random baseline to 37%, accuracy from ~50-55% to 67%. These provide sufficient context for understanding effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The CWE-1000 subset uses 50 pairs per top-level CWE, with three categories having only 30, 10, and 10 pairs respectively. No power analysis or justification for these sample sizes is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for deterministic results, yielding single-run numbers. No variance, standard deviation, or spread measures are reported across experimental conditions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares four evaluation settings: w/o context w/o revision (matching prior work), w/ context w/o revision, Lenient Mode, and Strict Mode. Random guessing baselines (50% accuracy, 25% (1,0) proportion, 0.5 precision) are also established."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 13 evaluated models include contemporary SOTA: DeepSeek-R1, DeepSeek-V3, o3-mini, Qwen2.5 series, and Llama-3.3-70B. Prior work comparisons reference recent studies from 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The framework is evaluated with systematic ablation of its components: context (w/o vs w/ context), revision process (w/o revision vs Lenient vs Strict Mode), and scaling dimensions (model size, sequential vs parallel test-time scaling)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: F1-score, accuracy, recall, precision (conventional metrics), plus pair-wise prediction proportions ((1,0), (1,1), (0,0), (0,1)) as introduced in §5.1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual inspection was conducted for error attribution: the authors 'manually inspected cases where the two models produced inconsistent results on patched code' (§5.3, Table 5), classifying errors into categories. Also, 50 pairs were manually audited for 98% label accuracy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 400-pair CWE-1000 subset is used purely for evaluation. No models were fine-tuned or tuned on this data — all models are evaluated in zero-shot mode with temperature 0."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 and Figure 6 provide detailed breakdowns across 10 top-level CWE categories, classified into Common and Rare groups. F1-scores are reported per CWE type across all models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis in §5.3 and Table 5: error types categorized as Patch Ignored, Patch Deemed Insufficient (with subcategories: Minimum Reasoning, Procedural Error, Mis-Corrected Reasoning). Appendix H provides concrete examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings: sequential scaling degrades recall (~10% drop), reasoning models exhibit overthinking/mis-corrected reasoning (Finding #4), test-time scaling has diminishing returns (Finding #6), and LLMs struggle with rare vulnerability types (Finding #2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 67% accuracy (Finding #1), >70% F1 on key CWEs (Table 4, CWE-664 and CWE-682), precision nearing 0.8 (Figure 4(f)), false positives from reasoning errors (Table 5), diminishing returns (Finding #6)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claim — that context improves performance — is supported by controlled comparison between w/o context and w/ context conditions on the same models and dataset. Model scaling claims are supported by systematic size comparisons across model families."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('Everything You Wanted to Know About LLM-based Vulnerability Detection') and abstract claim broad applicability, but the dataset is entirely C/C++ code from specific projects (Linux, FFmpeg, etc.) covering 99 CWEs. Results may not generalize to other languages or vulnerability types. This scope limitation is not explicitly bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Discussion section explains root causes of prior misconceptions but does not consider alternative explanations for CORRECT's own positive results. For instance, the prompts include ground-truth CWE descriptions, which provides the model with information about what vulnerability type to look for — this potential confound is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "A key contribution of the paper is explicitly distinguishing between correct labels (binary prediction) and correct rationales (root-cause identification). The two evaluation modes (Lenient and Strict) address this gap, and the paper argues prior work conflated label accuracy with true detection capability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 lists model names (Qwen2.5-7B-Inst, Llama-3.1-8B-Inst, DeepSeek-R1, etc.) but o3-mini and GPT-4o (used as LLM-as-a-judge) lack API snapshot dates or version identifiers. Per schema rules, marketing names without snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix E (Figures 10, 11, 12): Context-Rich Vulnerability Assessment Prompt, Rationale Assessment Prompt for vulnerable input, and Rationale Assessment Prompt for patched input. The structural template with variable slots (CWE description, code) is complete."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature set to 0 for all models (§5.1), temperature 0.6 for parallel scaling experiments, max_feedback_rounds = 4 for Strict Mode. These are the key inference hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. CORRECT is a prompting-based evaluation framework that sends single prompts to models and evaluates responses."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in §4.1: CVE record crawling → patch diff extraction → function-level commit filtering → Code Property Graph construction → vulnerability-related context extraction → context merging. Tools used include cflow and Joern."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations' subsection exists within the Discussion (§6), discussing LLM-as-a-judge accuracy (92%) and the impossibility of gathering complete context."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: LLM-as-a-judge accuracy validated at 91/99 (92%) on a random sample of 50 cases reviewed on ds-v3 and ds-r1; the practical impossibility of gathering all context for certain vulnerabilities. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of language limitations (all C/C++), project-type limitations (heavily Linux-kernel), or that results may not transfer to other vulnerability detection scenarios (e.g., web applications, smart contracts)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset and code are released at the anonymous repository URL. The 2,000 vulnerable-patched pairs with context are available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§4.1 describes data collection: sourced from MoreFixes, PrimeVul, and ReposVul; CVE records and patch commits used; repositories cloned and function-level diffs extracted. 364 real-world projects, 99 CWEs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is sourced from established vulnerability datasets (MoreFixes, PrimeVul, ReposVul) which are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the pipeline stages are described (§4.1, steps ➀-➄), exact filtering counts are not provided. The paper does not document how many CVEs were initially collected, how many were filtered at each stage, or what proportion was excluded and why before arriving at 2,000 pairs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section appears in the paper. The absence of any funding disclosure is noted."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Nanjing University (National Key Lab for Novel Software Technology) and Shandong University. No conflict with any evaluated product or company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The absence of a funding disclosure does not confirm independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 13 models evaluated. The CVEs used span multiple years and could overlap with model training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the CVE patches, commit messages, or vulnerability descriptions in the dataset were present in the training data of the evaluated models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is built from publicly available sources (MoreFixes, PrimeVul, ReposVul) containing well-known CVEs. Models trained on internet data could have seen these exact patches. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLMs on vulnerability detection."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on code datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Token counts for thinking tokens are discussed in the test-time scaling analysis (Figure 7), but no overall inference cost, API spend, or per-example cost is reported for the main evaluation across 13 models × 400 pairs × multiple modes."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, API costs, wall-clock time) is stated despite the substantial evaluation: 13 models, 400+ pairs, 4 evaluation modes, plus scaling experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for deterministic results, but no seed sensitivity analysis is reported. The paper does not examine how results vary with different temperatures or sampling configurations for the main evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Temperature=0 implies single deterministic runs for main results, but this is not explicitly documented. For parallel scaling, 3/5/8 samples are stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The choice of temperature=0 follows prior work [51], and max_feedback_rounds=4 is justified by Table 6, but no systematic search was conducted or reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The choice of zero-shot CoT is justified: '(a) it avoids example selection biases present in few-shot approaches, and (b) advanced prompting techniques prove ineffective for SOTA LLMs' (§4.2). max_feedback_rounds=4 is justified by Table 6 analysis."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Numerous comparisons are made across 13 models, 10 CWE categories, 4 evaluation modes, and scaling conditions, but no multiple comparison correction (Bonferroni, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own CORRECT framework against prior evaluation methodologies but do not acknowledge or address the bias of evaluating their own system. No independent evaluation is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 explicitly plots accuracy, precision, and recall as functions of thinking tokens for both o3-mini (at three effort levels) and r1-qn-14b (sequential and parallel scaling). The power-law relationship is discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Construct validity is a central focus of the paper. Sections 3 and 6 extensively argue that prior benchmarks failed to measure real vulnerability detection capability due to missing context, and CORRECT is designed to address this validity gap."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly via prompts without agentic scaffolding, so the scaffold confound does not apply."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The CVEs span multiple years and many predate the training data of evaluated models. Models may have seen the exact vulnerabilities, patches, and CVE descriptions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation prompt includes ground-truth CWE descriptions, telling the model what vulnerability type to look for. This provides information not available in real-world detection scenarios. This potential feature leakage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple CVEs come from the same projects (Linux has 400+ samples per Figure 9c). Structural similarities between test examples from the same codebase are not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference tests, temporal splits, or decontamination pipelines are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Prior evaluations significantly underestimated LLM vulnerability detection capabilities due to missing context. DeepSeek-R1 achieves 67% accuracy and a 37% (1,0) proportion under CORRECT.",
       "evidence": "Figure 4(i),(j) show Strict Mode results vs w/o context baseline (~50-55% accuracy). Figure 5 shows (1,0) proportion reaching 37% vs 25% random baseline. (§5.2, Finding #1)",
       "supported": "strong"
     },
     {
       "claim": "SOTA LLMs achieve precision close to 0.8 in distinguishing vulnerable from patched code, with only ~10% (1,1) false positive proportion.",
       "evidence": "Figure 4(f) shows precision approaching 0.8 for 671B models under Strict Mode. Figure 5 shows all models exhibit ~10% (1,1) proportion. (§5.3, Finding #3)",
       "supported": "strong"
     },
     {
       "claim": "Only 9.5% of false positives in reasoning models result from failing to notice patches; most arise from incorrect inference that patches are insufficient.",
       "evidence": "Table 5: 'Patch Ignored' accounts for 2/21 (9.5%) of ds-r1's FP cases, vs 'Patch Deemed Insufficient' at 19/21 (90.5%). Manual inspection of ds-v3 vs ds-r1 inconsistent cases. (§5.3)",
       "supported": "moderate"
     },
     {
       "claim": "Model size scaling laws exist for vulnerability detection when context is provided, but are invisible in context-free evaluations.",
       "evidence": "Figure 4(c): F1 clustered at 0.5-0.6 without context, no scaling trend. Figure 4(i): clear scaling trends in Strict Mode across Qwen and Llama families. (§5.4, Finding #5)",
       "supported": "moderate"
     },
     {
       "claim": "Test-time scaling follows a power-law relationship, making it uneconomical: 5x thinking tokens yields less than 5% accuracy improvement.",
       "evidence": "Figure 7(d): o3-mini aggregate accuracy shows power-law relationship. o3-mini-high with 5x tokens of medium achieves <0.05 accuracy improvement. (§5.4, Finding #6)",
       "supported": "moderate"
     },
     {
       "claim": "Sequential scaling induces conservative bias that reduces recall by approximately 10%, while parallel scaling avoids this issue.",
       "evidence": "Figure 7(i)-(l): r1-qn-14b sequential scaling shows recall declining from ~0.4 to ~0.2 as tokens increase beyond 2k. Parallel scaling maintains or improves recall. (§5.4, Finding #7)",
       "supported": "moderate"
     },
     {
       "claim": "LLMs achieve F1-scores around 0.7 on common fixed-pattern vulnerabilities (CWE-664, CWE-682) but struggle with rare vulnerability types (CWE-435, CWE-697).",
       "evidence": "Table 4: CWE-664 max F1=0.700, CWE-682 max F1=0.713 vs CWE-435 max F1=0.556, CWE-697 max F1=0.400. Figure 6 shows per-CWE breakdowns. (§5.2, Finding #2)",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper demonstrates that prior negative assessments of LLM vulnerability detection were artifacts of context-deprived evaluations. Under the CORRECT framework with rich contextual information, SOTA models (DeepSeek-R1) achieve 67% accuracy with precision near 0.8, significantly exceeding random baselines. Most false positives stem from reasoning errors (judging patches as insufficient) rather than failing to notice patches. Test-time and model scaling improve performance but with diminishing returns, and sequential scaling introduces conservative biases that reduce recall.",
   "red_flags": [
     {
       "flag": "Ground-truth CWE descriptions provided in prompts",
       "detail": "The evaluation prompt includes the ground-truth CWE category description, telling the model what vulnerability type to look for. Real-world vulnerability detection does not have this information, making the evaluation setting more favorable than practical deployment. This is essentially feature leakage that inflates apparent capability."
     },
     {
       "flag": "No contamination analysis despite public CVE dataset",
       "detail": "The dataset is built from well-known CVEs in public repositories (Linux, FFmpeg, etc.) using established datasets (MoreFixes, PrimeVul, ReposVul). Models trained on internet data almost certainly encountered these specific vulnerabilities and patches during training. No contamination analysis is performed."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Despite making numerous comparative claims ('significantly surpassing', 'significantly improved'), no statistical significance tests are reported. All comparisons are based on point estimates without uncertainty quantification."
     },
     {
       "flag": "Very small sample sizes for some CWE categories",
       "detail": "CWE-435 and CWE-697 have only 10 pairs each, CWE-693 has only 30 pairs. Drawing conclusions about model performance on these categories from such small samples is unreliable."
     },
     {
       "flag": "LLM-as-a-judge with 8% error rate",
       "detail": "GPT-4o is used as LLM-as-a-judge for rationale assessment with a validated accuracy of 92% (91/99). An 8% error rate could systematically bias results, especially given that this evaluation step determines the key distinction between Lenient and Strict modes."
     }
   ],
   "cited_papers": [
     {
       "title": "Vulnerability Detection with Code Language Models: How Far Are We?",
       "authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim"],
       "year": 2024,
       "relevance": "Key prior work on LLM vulnerability detection benchmarking (PrimeVul), one of the 'consensus' papers this work challenges."
     },
     {
       "title": "LLMs cannot reliably identify and reason about security vulnerabilities (yet?): A comprehensive evaluation, framework, and benchmarks",
       "authors": ["Saad Ullah", "Mingji Han", "Saurabh Pujar"],
       "year": 2024,
       "relevance": "SecLLMHolmes evaluation framework for LLM security analysis, a primary baseline this paper argues is flawed due to missing context."
     },
     {
       "title": "To Err is Machine: Vulnerability Detection Challenges LLM Reasoning",
       "authors": ["Benjamin Steenhoek", "Md Mahbubur Rahman", "Monoshi Kumar Roy"],
       "year": 2024,
       "arxiv_id": "2403.17218",
       "relevance": "Evaluates LLM reasoning on vulnerability detection across model scales, provides 'Consensus #3' (plateaued performance) that this paper challenges."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "DeepSeek-R1 reasoning LLM evaluated as SOTA model in this paper, achieving the best vulnerability detection performance."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational CoT prompting technique used in this paper's prompt-driven reasoning approach for vulnerability detection."
     },
     {
       "title": "Scaling llm test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Test-time scaling theory that this paper empirically tests in the vulnerability detection domain."
     },
     {
       "title": "Top score on the wrong exam: On benchmarking in machine learning for vulnerability detection",
       "authors": ["Niklas Risse", "Marcel Böhme"],
       "year": 2024,
       "arxiv_id": "2408.12986",
       "relevance": "Critiques ML benchmarking for vulnerability detection, aligns with this paper's argument about flawed evaluation practices."
     },
     {
       "title": "VulEval: Towards repository-level evaluation of software vulnerability detection",
       "authors": ["Xin-Cheng Wen", "Xinchen Wang"],
       "year": 2024,
       "arxiv_id": "2404.15596",
       "relevance": "Repository-level vulnerability evaluation framework; performs some context augmentation with caller-callee relationships."
     },
     {
       "title": "LLM4Vuln: A unified evaluation framework for decoupling and enhancing llms' vulnerability reasoning",
       "authors": ["Yuqiang Sun", "Daoyuan Wu"],
       "year": 2024,
       "arxiv_id": "2401.16185",
       "relevance": "LLM vulnerability reasoning evaluation framework with RAG-based context augmentation, a direct comparison point."
     },
     {
       "title": "Reasoning with LLMs for Zero-Shot Vulnerability Detection",
       "authors": ["Arastoo Zibaeirad", "Marco Vieira"],
       "year": 2025,
       "arxiv_id": "2503.17885",
       "relevance": "VulnSage: zero-shot reasoning approach for vulnerability detection that introduced 'not sure' responses to address context limitations."
     },
     {
       "title": "Benchmarking LLMs and LLM-based Agents in Practical Vulnerability Detection for Code Repositories",
       "authors": ["Alperen Yildiz", "Sin G Teo"],
       "year": 2025,
       "arxiv_id": "2503.03586",
       "relevance": "JitVul: benchmarks LLM-based agents for repository-level vulnerability detection, directly relevant to evaluation methodology."
     },
     {
       "title": "s1: Simple test-time scaling",
       "authors": ["Niklas Muennighoff", "Zitong Yang"],
       "year": 2025,
       "arxiv_id": "2501.19393",
       "relevance": "Test-time scaling methodology ('Wait' appending technique) directly used in this paper's sequential scaling experiments."
     }
   ]
 }