ai-research-survey

scan-v4.json (36596B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Everything You Wanted to Know About LLM-based Vulnerability Detection But Were Afraid to Ask",
     "authors": [
       "Yue Li",
       "Xiao Li",
       "Hao Wu",
       "Minghui Xu",
       "Yue Zhang",
       "Xiuzhen Cheng",
       "Fengyuan Xu",
       "Sheng Zhong"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2504.13474",
     "doi": "10.48550/arXiv.2504.13474"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 67% accuracy (Finding #1), >70% F1 on key CWEs (Table 4, CWE-664 and CWE-682), precision nearing 0.8 (Figure 4(f)), false positives from reasoning errors (Table 5), diminishing returns (Finding #6).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claim — that context improves performance — is supported by controlled comparison between w/o context and w/ context conditions on the same models and dataset. Model scaling claims are supported by systematic size comparisons across model families.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('Everything You Wanted to Know About LLM-based Vulnerability Detection') and abstract claim broad applicability, but the dataset is entirely C/C++ code from specific projects (Linux, FFmpeg, etc.) covering 99 CWEs. Results may not generalize to other languages or vulnerability types. This scope limitation is not explicitly bounded.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Discussion section explains root causes of prior misconceptions but does not consider alternative explanations for CORRECT's own positive results. For instance, the prompts include ground-truth CWE descriptions, which provides the model with information about what vulnerability type to look for — this potential confound is not discussed.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "A key contribution of the paper is explicitly distinguishing between correct labels (binary prediction) and correct rationales (root-cause identification). The two evaluation modes (Lenient and Strict) address this gap, and the paper argues prior work conflated label accuracy with true detection capability.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations' subsection exists within the Discussion (§6), discussing LLM-as-a-judge accuracy (92%) and the impossibility of gathering complete context.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: LLM-as-a-judge accuracy validated at 91/99 (92%) on a random sample of 50 cases reviewed on ds-v3 and ds-r1; the practical impossibility of gathering all context for certain vulnerabilities. These are specific to this study.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of language limitations (all C/C++), project-type limitations (heavily Linux-kernel), or that results may not transfer to other vulnerability detection scenarios (e.g., web applications, smart contracts).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section appears in the paper. The absence of any funding disclosure is noted.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Nanjing University (National Key Lab for Novel Software Technology) and Shandong University. No conflict with any evaluated product or company.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The absence of a funding disclosure does not confirm independence.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'Context' is explicitly defined (callee functions, type declarations, global variables, slicing paths); 'reasoning' is defined via System 1/System 2 framing in §2.1; 'vulnerability detection' evaluation modes are formally defined in §4.3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly listed: the CORRECT framework, a 2,000-pair context-rich dataset, and evidence refuting three community consensus beliefs.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 systematically compares CORRECT against eight prior evaluation frameworks along four dimensions, and §2.2 explains specific limitations of each prior approach that CORRECT addresses.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper states 'Our dataset and code are available' with an anonymous link (https://anonymous.4open.science/r/CORRECT) provided in §1. This is an anonymized review link but does provide a working URL.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The dataset of 2,000 vulnerable-patched program pairs is released at the same anonymous URL. The paper also builds on publicly available sources: MoreFixes, PrimeVul, and ReposVul.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No environment specifications (requirements.txt, Dockerfile, library versions) are mentioned in the paper. Only tools used are named (cflow, Joern) without version details.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided. The paper describes the framework methodology but does not include a README or scripts to replicate experiments.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are reported as point estimates (e.g., '67% accuracy', 'precision close to 0.8', 'F1-score exceeds 70%'). No confidence intervals or error bars appear in figures or tables.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes numerous comparative claims (e.g., 'significantly surpassing the random baseline', 'significantly improved performance') but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports improvements with baseline context: e.g., precision from ~0.5 (w/o context) to ~0.8 (Strict Mode), (1,0) proportion from 25% random baseline to 37%, accuracy from ~50-55% to 67%. These provide sufficient context for understanding effect magnitudes.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The CWE-1000 subset uses 50 pairs per top-level CWE, with three categories having only 30, 10, and 10 pairs respectively. No power analysis or justification for these sample sizes is provided.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Temperature is set to 0 for deterministic results, yielding single-run numbers. No variance, standard deviation, or spread measures are reported across experimental conditions.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "The paper compares four evaluation settings: w/o context w/o revision (matching prior work), w/ context w/o revision, Lenient Mode, and Strict Mode. Random guessing baselines (50% accuracy, 25% (1,0) proportion, 0.5 precision) are also established.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The 13 evaluated models include contemporary SOTA: DeepSeek-R1, DeepSeek-V3, o3-mini, Qwen2.5 series, and Llama-3.3-70B. Prior work comparisons reference recent studies from 2024-2025.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The framework is evaluated with systematic ablation of its components: context (w/o vs w/ context), revision process (w/o revision vs Lenient vs Strict Mode), and scaling dimensions (model size, sequential vs parallel test-time scaling).",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics are used: F1-score, accuracy, recall, precision (conventional metrics), plus pair-wise prediction proportions ((1,0), (1,1), (0,0), (0,1)) as introduced in §5.1.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Manual inspection was conducted for error attribution: the authors 'manually inspected cases where the two models produced inconsistent results on patched code' (§5.3, Table 5), classifying errors into categories. Also, 50 pairs were manually audited for 98% label accuracy.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "The 400-pair CWE-1000 subset is used purely for evaluation. No models were fine-tuned or tuned on this data — all models are evaluated in zero-shot mode with temperature 0.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 and Figure 6 provide detailed breakdowns across 10 top-level CWE categories, classified into Common and Rare groups. F1-scores are reported per CWE type across all models.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Extensive failure analysis in §5.3 and Table 5: error types categorized as Patch Ignored, Patch Deemed Insufficient (with subcategories: Minimum Reasoning, Procedural Error, Mis-Corrected Reasoning). Appendix H provides concrete examples.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Several negative findings: sequential scaling degrades recall (~10% drop), reasoning models exhibit overthinking/mis-corrected reasoning (Finding #4), test-time scaling has diminishing returns (Finding #6), and LLMs struggle with rare vulnerability types (Finding #2).",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Table 3 lists model names (Qwen2.5-7B-Inst, Llama-3.1-8B-Inst, DeepSeek-R1, etc.) but o3-mini and GPT-4o (used as LLM-as-a-judge) lack API snapshot dates or version identifiers. Per schema rules, marketing names without snapshot dates do not count as specified versions.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full prompt templates are provided in Appendix E (Figures 10, 11, 12): Context-Rich Vulnerability Assessment Prompt, Rationale Assessment Prompt for vulnerable input, and Rationale Assessment Prompt for patched input. The structural template with variable slots (CWE description, code) is complete.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Temperature set to 0 for all models (§5.1), temperature 0.6 for parallel scaling experiments, max_feedback_rounds = 4 for Strict Mode. These are the key inference hyperparameters.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. CORRECT is a prompting-based evaluation framework that sends single prompts to models and evaluates responses.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "The data pipeline is documented in §4.1: CVE record crawling → patch diff extraction → function-level commit filtering → Code Property Graph construction → vulnerability-related context extraction → context merging. Tools used include cflow and Joern.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Dataset and code are released at the anonymous repository URL. The 2,000 vulnerable-patched pairs with context are available for verification.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "§4.1 describes data collection: sourced from MoreFixes, PrimeVul, and ReposVul; CVE records and patch commits used; repositories cloned and function-level diffs extracted. 364 real-world projects, 99 CWEs.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data is sourced from established vulnerability datasets (MoreFixes, PrimeVul, ReposVul) which are standard benchmarks.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "While the pipeline stages are described (§4.1, steps ➀-➄), exact filtering counts are not provided. The paper does not document how many CVEs were initially collected, how many were filtered at each stage, or what proportion was excluded and why before arriving at 2,000 pairs.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates are stated for any of the 13 models evaluated. The CVEs used span multiple years and could overlap with model training data.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the CVE patches, commit messages, or vulnerability descriptions in the dataset were present in the training data of the evaluated models.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The dataset is built from publicly available sources (MoreFixes, PrimeVul, ReposVul) containing well-known CVEs. Models trained on internet data could have seen these exact patches. This contamination risk is not addressed.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study. It is a benchmark evaluation of LLMs on vulnerability detection.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The study evaluates LLMs on code datasets.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Token counts for thinking tokens are discussed in the test-time scaling analysis (Figure 7), but no overall inference cost, API spend, or per-example cost is reported for the main evaluation across 13 models × 400 pairs × multiple modes.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget (GPU hours, API costs, wall-clock time) is stated despite the substantial evaluation: 13 models, 400+ pairs, 4 evaluation modes, plus scaling experiments.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "Temperature is set to 0 for deterministic results, but no seed sensitivity analysis is reported. The paper does not examine how results vary with different temperatures or sampling configurations for the main evaluation.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of experimental runs is not explicitly stated. Temperature=0 implies single deterministic runs for main results, but this is not explicitly documented. For parallel scaling, 3/5/8 samples are stated.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameter search budget reported. The choice of temperature=0 follows prior work [51], and max_feedback_rounds=4 is justified by Table 6, but no systematic search was conducted or reported.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "The choice of zero-shot CoT is justified: '(a) it avoids example selection biases present in few-shot approaches, and (b) advanced prompting techniques prove ineffective for SOTA LLMs' (§4.2). max_feedback_rounds=4 is justified by Table 6 analysis.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "Numerous comparisons are made across 13 models, 10 CWE categories, 4 evaluation modes, and scaling conditions, but no multiple comparison correction (Bonferroni, etc.) is applied.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors evaluate their own CORRECT framework against prior evaluation methodologies but do not acknowledge or address the bias of evaluating their own system. No independent evaluation is conducted.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "Figure 7 explicitly plots accuracy, precision, and recall as functions of thinking tokens for both o3-mini (at three effort levels) and r1-qn-14b (sequential and parallel scaling). The power-law relationship is discussed.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": true,
           "justification": "Construct validity is a central focus of the paper. Sections 3 and 6 extensively argue that prior benchmarks failed to measure real vulnerability detection capability due to missing context, and CORRECT is designed to address this validity gap.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is involved. Models are evaluated directly via prompts without agentic scaffolding, so the scaffold confound does not apply.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of temporal leakage. The CVEs span multiple years and many predate the training data of evaluated models. Models may have seen the exact vulnerabilities, patches, and CVE descriptions during training.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The evaluation prompt includes ground-truth CWE descriptions, telling the model what vulnerability type to look for. This provides information not available in real-world detection scenarios. This potential feature leakage is not discussed.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Multiple CVEs come from the same projects (Linux has 400+ samples per Figure 9c). Structural similarities between test examples from the same codebase are not discussed.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference tests, temporal splits, or decontamination pipelines are applied.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Prior context-free evaluations significantly underestimated LLM vulnerability detection capability; SOTA models achieve 67% accuracy and 37% (1,0) proportion under CORRECT's Strict Mode versus near-random without context.",
       "evidence": "Figure 4(d) shows w/o context accuracy clustered 0.50–0.55; Figure 4(j) shows Strict Mode accuracy reaching 0.67 for DeepSeek-R1; Figure 5 shows (1,0) proportion of 37% vs 25% random baseline.",
       "supported": "strong"
     },
     {
       "claim": "SOTA LLMs (671B) achieve precision approaching 0.8 in distinguishing vulnerable from patched code when provided with sufficient context.",
       "evidence": "Figure 4(f) shows DeepSeek-R1 and DeepSeek-V3 reaching precision ~0.8 in Strict Mode, while Figure 4(a) shows all models clustered near 0.5 without context.",
       "supported": "strong"
     },
     {
       "claim": "Only ~9.5% of false positives from reasoning models reflect genuine failure to distinguish patched code; the majority arise because the model incorrectly deems the patch insufficient.",
       "evidence": "Table 5 manual analysis: 'Patch Ignored' accounts for 2 cases (R→Non-R direction) vs 19 for 'Patch Deemed Insufficient' in reasoning models; 9.5% figure cited in Finding #4.",
       "supported": "moderate"
     },
     {
       "claim": "A clear model size scaling law emerges for vulnerability detection only when context is provided; without context, smaller models perform comparably to 671B SOTA models.",
       "evidence": "Figure 4(c) shows F1 clustered 0.5–0.6 regardless of model size w/o context; Figure 4(g)(i) show clear positive correlation with model size under CORRECT.",
       "supported": "strong"
     },
     {
       "claim": "Test-time scaling follows a power-law relationship where 5x thinking tokens yields less than 5% accuracy improvement, making it economically unviable as the primary scaling strategy.",
       "evidence": "Figure 7(d) shows aggregate accuracy trend for o3-mini-low/medium/high; paper states 'o3-mini-high, with five times the thinking tokens of o3-mini-medium, achieves an accuracy improvement of less than 0.05'.",
       "supported": "strong"
     },
     {
       "claim": "LLMs fail to generalize to rare and out-of-distribution vulnerability types even with context, achieving F1 scores as low as 0.4 (CWE-697).",
       "evidence": "Table 4 shows rare CWEs (CWE-435, CWE-697, CWE-703) max F1 of 0.556, 0.400, 0.479 respectively, versus 0.700–0.713 for common CWEs.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The CORRECT framework demonstrates that three widely-held beliefs about LLM vulnerability detection (unreliable, insensitive to patches, plateaued with scale) are artifacts of context-free evaluations rather than fundamental model limitations. When provided with callee functions, type declarations, and execution context, SOTA models (DeepSeek-R1) achieve 67% accuracy and precision near 0.8 in Strict Mode, vastly outperforming prior evaluations' near-random results. The primary bottleneck is not binary classification but recall (~0.5 for SOTA), and most false positives arise from reasoning errors (deeming patches insufficient) rather than genuine inability to detect patches. Both model scaling and test-time scaling improve performance, but with diminishing returns and trade-offs: sequential scaling causes recall degradation via overthinking, while LLMs systematically fail on rare/out-of-distribution vulnerability types regardless of context.",
   "red_flags": [
     {
       "flag": "Contamination unaddressed",
       "detail": "The dataset consists of historical CVE records from public repositories (Linux, FFmpeg, ImageMagick, etc.) that almost certainly appear in the training corpora of all 13 evaluated models; no training cutoffs are stated and no decontamination analysis is performed."
     },
     {
       "flag": "CWE hints in prompts inflate performance",
       "detail": "Prompt I includes the ground-truth CWE description, effectively telling the model what vulnerability type to look for. This substantially aids detection and confounds the comparison with prior work that used blind evaluation."
     },
     {
       "flag": "LLM-as-a-judge circularity",
       "detail": "GPT-4o is used to evaluate rationale quality for GPT-4o-adjacent models, with judge accuracy validated on only 50 manually reviewed cases (92%). This creates a circular evaluation loop for OpenAI models and insufficient validation sample."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims across 13 models and 4 evaluation modes are based on point estimates with no confidence intervals, error bars, or significance tests applied."
     },
     {
       "flag": "Anonymous code link not permanent",
       "detail": "The code and data link (anonymous.4open.science) is an anonymous review platform; such links are typically temporary and may not survive post-publication, undermining reproducibility claims."
     },
     {
       "flag": "400-pair sample unjustified",
       "detail": "Only 400 of 2,000 pairs are used for primary evaluation; the subsampling rationale (CWE-1000 classification) is described but no power analysis or sufficiency argument is made for this sample size."
     }
   ],
   "cited_papers": [
     {
       "title": "Vulnerability Detection with Code Language Models: How Far Are We? (PrimeVul)",
       "relevance": "Primary dataset source and key prior evaluation work that this paper challenges for context-free methodology"
     },
     {
       "title": "LLMs Cannot Reliably Identify and Reason About Security Vulnerabilities (Yet?): A Comprehensive Evaluation, Framework, and Benchmarks (SecLLMHolmes)",
       "relevance": "Key comparison establishing Consensus #1 (unreliability); CORRECT directly refutes its conclusions"
     },
     {
       "title": "To Err is Machine: Vulnerability Detection Challenges LLM Reasoning (Steenhoek et al.)",
       "relevance": "Establishes Consensus #3 (plateaued performance); CORRECT shows this is an artifact of missing context"
     },
     {
       "title": "LLM4Vuln: A Unified Evaluation Framework for Decoupling and Enhancing LLMs' Vulnerability Reasoning",
       "relevance": "Prior context-augmented framework compared in Table 1"
     },
     {
       "title": "VulEval: Towards Repository-Level Evaluation of Software Vulnerability Detection",
       "relevance": "Prior work on context augmentation via caller-callee relationships"
     },
     {
       "title": "MoreFixes: A Large-Scale Dataset of CVE Fix Commits Mined Through Enhanced Repository Discovery",
       "relevance": "One of three dataset sources for CORRECT's 2,000 program pairs"
     },
     {
       "title": "ReposVul: A Repository-Level High-Quality Vulnerability Dataset",
       "relevance": "One of three dataset sources for CORRECT's 2,000 program pairs"
     },
     {
       "title": "Top Score on the Wrong Exam: On Benchmarking in Machine Learning for Vulnerability Detection (Risse & Böhme)",
       "relevance": "Foundational critique of vulnerability detection benchmarking methodology that motivates this work"
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "relevance": "Best-performing model in evaluations; reasoning LLM whose capabilities the paper reassesses"
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "relevance": "Foundational prompting technique used in prior vulnerability detection evaluations this paper critiques"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses how to properly evaluate and deploy LLMs for security vulnerability detection, with actionable guidance on context provision and scaling strategy trade-offs."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "Explicitly and successfully challenges three established community consensus beliefs, showing LLMs are actually much better at vulnerability detection than reported — a significant reversal."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Security vulnerability detection is directly safety-relevant; the finding that prior evaluations were systematically wrong has implications for trust in deployed LLM-based security tools."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "The framing as refuting community 'consensus beliefs' and the direct critiques of named prior papers create genuine scientific controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and dataset are available at an anonymous link, enabling practitioners to apply CORRECT to evaluate their own LLMs on vulnerability detection."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from Nanjing and Shandong universities without major lab brand recognition, though they evaluate prominent models like DeepSeek-R1 and o3-mini."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "27146649",
         "title": "Geometric Deep Learning: Grids, Groups, Graphs, Geodesics, and Gauges",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=27146649",
         "created_at": "2021-05-13T20:00:26Z"
       },
       {
         "hn_id": "45166677",
         "title": "Geometric Deep Learning Grids, Groups, Graphs, Geodesics, and Gauges [pdf]",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45166677",
         "created_at": "2025-09-08T10:39:40Z"
       },
       {
         "hn_id": "42855137",
         "title": "Why a Race to Artificial Superintelligence Is Self-Defeating [pdf]",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42855137",
         "created_at": "2025-01-28T17:27:43Z"
       },
       {
         "hn_id": "43788230",
         "title": "Show HN: A new way to verify remote AI model execution (no TEEs, no ZK)",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43788230",
         "created_at": "2025-04-24T22:31:33Z"
       },
       {
         "hn_id": "44796040",
         "title": "From Large to Super-Tiny: End-to-End Optimization for Cost-Efficient LLMs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44796040",
         "created_at": "2025-08-05T09:39:59Z"
       },
       {
         "hn_id": "44968425",
         "title": "Consumer Autonomy or Illusion? Rethinking Consumer Agency in Age of Algorithms",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=44968425",
         "created_at": "2025-08-21T02:16:50Z"
       },
       {
         "hn_id": "45483510",
         "title": "A Convex Formulation of Compliant Contact Between Filaments and Rigid Bodies",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45483510",
         "created_at": "2025-10-05T17:33:41Z"
       },
       {
         "hn_id": "42836005",
         "title": "Autonomy-of-Experts Models (ArXiv)",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42836005",
         "created_at": "2025-01-27T00:43:16Z"
       },
       {
         "hn_id": "42008373",
         "title": "Geometric Deep Learning: Grids, Groups, Graphs, Geodesics, and Gauges",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42008373",
         "created_at": "2024-10-31T16:19:04Z"
       },
       {
         "hn_id": "30395596",
         "title": "Geometric Deep Learning: Grids, Groups, Graphs, Geodesics, and Gauges",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30395596",
         "created_at": "2022-02-19T09:10:06Z"
       }
     ],
     "top_points": 3,
     "total_points": 23,
     "total_comments": 2
   }
 }