ai-research-survey

scan.json (32655B)

---

 {
   "paper": {
     "title": "VulScribeR: Exploring RAG-based Vulnerability Augmentation with LLMs",
     "authors": [
       "Seyed Shayan Daneshvar",
       "Yu Nong",
       "Xu Yang",
       "Shaowei Wang",
       "Haipeng Cai"
     ],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2408.04125",
     "doi": "10.1145/3760775"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "VulScribeR proposes three LLM-based strategies (Mutation, Injection, Extension) for augmenting vulnerable code datasets, leveraging RAG for diversity and relevance. Injection and Extension consistently outperform SOTA baselines (VulGen, VGX, ROS) across 4 datasets, 3 DLVD models, and 3 LLMs, with Injection beating baselines by 15-28% in average F1-score at 5K samples. Unlike baselines that degrade beyond 5K samples, VulScribeR scales to 15K samples with continued improvement, at a cost of ~$1.88 per 1K samples.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors state 'We made our source code, experimental results, and the augmented datasets publicly available for future research [1]' with reference [1] pointing to https://github.com/shayandaneshvar/VulScribeR."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package includes augmented datasets (reference [1]). Additionally, all base datasets used (Devign, Reveal, BigVul, PrimeVul) are publicly available, and the paper specifies the cleaned versions from VGX."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware ('four 24GB Nvidia RTX 3090s') but does not list software dependencies, library versions, requirements.txt, Dockerfile, or conda environment specifications in the paper text."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While a replication package is linked, the paper itself does not contain a 'Reproducing Results' section or specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-8 report point estimates (Precision, Recall, F1) without confidence intervals, error bars, or ± notation. Despite running Devign and Reveal models 5 times with different seeds, no uncertainty measures are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "All claims of outperformance (e.g., 'Injection outperforms NoAug, VulGen, VGX, and ROS by 30.80%, 27.48%, 27.93%, and 15.41%') are based on comparing point estimates without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports percentage improvements with baseline context throughout, e.g., 'Injection outperforms NoAug, Vulgen, VGX, and ROS by 30.80%, 27.48%, 27.93%, and 15.41% on average F1-score.' Full result tables provide baseline and proposed method numbers for computing effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 5K, 10K, and 15K augmented samples were chosen as experimental points. No power analysis or reasoning for these specific quantities is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "For Devign and Reveal DLVD models, the authors 'train both of these models 5 times with random seed values, test them and report the results with the highest F1-score achieved' (Section 4.3). Only the best-of-5 result is reported with no standard deviation or variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are included: NoAug (no augmentation), ROS (random oversampling), VulGen, and VGX. All are described in Section 4.6."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "VGX (Nong et al., 2024, ICSE '24) is contemporary and represents the latest state of the art in vulnerability augmentation. VulGen (2023, ICSE '23) is also recent."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 5.2) provides ablation studies removing the Retriever component (w/o Retriever) and the clustering phase (w/o Clustering) for both Injection and Extension strategies, with results in Tables 3 and 4."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Precision, Recall, and F1-Score, reported in all result tables (Tables 2-8)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 reports a manual assessment of 150 randomly selected generated samples (50 per strategy) to determine whether the LLM followed instructions and generated truly vulnerable code, finding 72-90% success rates."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper trains on augmented Devign and tests on BigVul's test set and Reveal dataset. 'We specifically train and test the models with different datasets, following the settings of VulGen and VGX, to ensure there is no information leakage' (Section 4.3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by test dataset (Reveal, BigVul), DLVD model (Devign, Reveal, LineVul), and LLM (ChatGPT, CodeQwen) in all tables, providing 12 experimental instances per strategy."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 discusses failure patterns for each strategy: Mutation fails by changing code in unspecified ways, Injection produces wrong or half-complete injections, and Extension removes input code instead of extending it."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Mutation does not consistently beat ROS (Section 5.1), ensemble of strategies does not always outperform single strategies (Section 6.1, 'M+I+E always fail to beat all of the single strategies'), and free-form extension without clean samples 'does not work well, and tends to hallucinate' (Section 3.1.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's specific numerical claims (e.g., '27.48%, 27.93%, and 15.41% in f1-score with 5K' and '53.84%, 54.10%, 69.90%, and 40.93% with 15K') are verified in Tables 2 and 5 respectively. The cost claim of '$1.88 per 1K samples' is confirmed in Section 6.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions (e.g., 'RAG component makes a significant contribution') are supported by controlled ablation studies in RQ2, where single variables (Retriever, Clustering) are systematically removed to measure their impact (Tables 3-4)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'our method significantly outperforms SOTA methods, and is suitable for large-scale vulnerability augmentation' without bounding to C/C++ code, the specific DLVD families tested, or the specific LLM capabilities used. All experiments use C/C++ datasets only, but claims are framed for vulnerability augmentation broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 (Threats to Validity) discusses specific alternative explanations: hyperparameter settings could yield different results, label noise in Devign/BigVul could affect results (addressed by PrimeVul experiment), and LLM hallucination could affect generated sample quality (addressed by verifier and manual inspection)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures F1-score on vulnerability detection tasks and frames results in terms of DLVD model effectiveness. The measurement (F1 on held-out vulnerability detection benchmarks) directly matches the claim (improved vulnerability detection performance). No broader proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'gpt-3.5-turbo', 'CodeQwen1.5-7B-Chat', and 'GPT4o-mini' but provides no snapshot dates or API versions for the OpenAI models. The gpt-3.5-turbo endpoint has been updated multiple times with different behaviors, and no date qualifier is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates are shown in Figures 2, 3, and 4. While the Mutation template truncates some transformation rules ('{...remaining_rules_are_hidden_to_save_space}'), the paper links to a public repository (reference [1]) containing the full prompts. Placeholders are data-dependent (code samples from datasets) and deterministically reconstructible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports temperature=0.5 for ChatGPT, default parameters for CodeQwen except max_new_tokens=4096, and GPT4o-mini temperature=0.5. Section 4.3 reports DLVD training settings (5 random seeds for Devign/Reveal, 10 epochs for LineVul)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a RAG-based generation pipeline with prompt templates, retrieval, and parsing-based filtering — not an agentic system with tool use, feedback loops, or memory management."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is described in detail: VGX's cleaned dataset versions with deduplicates removed (Section 4.2), SHA2 hash deduplication for PrimeVul, extraction of vulnerable lines via diff, clustering with CodeBERT embeddings and KMeans, BM-25 retrieval, and fuzzy parser filtering removing 2-13% of generated data."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 'Threats to Validity' provides substantive discussion of internal and external validity threats across multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 discusses study-specific threats: hyperparameter settings for DLVD models and LLMs not being tuned, label noise in Devign and BigVul datasets (citing references [11, 14, 54]), LLM hallucination affecting generated code quality, and limitation to two LLMs and three datasets for generalizability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The external validity section explicitly identifies what was not tested: 'Even though we used two different LLMs... evaluated on three commonly used datasets, and covered three SOTA DLVD models... our findings may not generalize.' It specifically identifies gaps: encoder-decoder LLMs, other datasets, and other DLVD models."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package (reference [1], GitHub) includes 'source code, experimental results, and the augmented datasets' making both generated data and results available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes dataset origins in detail: Devign from VGX's cleaned version, BigVul's training set with 6,610 samples that include vulnerable line metadata, Reveal as a testing set, and PrimeVul with SHA2 hash deduplication against Devign."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. All data comes from standard public vulnerability detection benchmarks (Devign, Reveal, BigVul, PrimeVul)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: Algorithm 1 describes pair retrieval with clustering, Section 3 describes the Retriever→Formulator→Generator→Verifier pipeline, Section 4.7 describes the approach for each RQ including specific sample counts (e.g., generate 6K to yield 5K after filtering)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper text. Funding sources are not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: University of Manitoba (Daneshvar, Yang, Wang), Washington State University (Nong), and University at Buffalo (Cai). No commercial affiliations with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence since funding is not disclosed. Absence of disclosure prevents evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. LLMs (ChatGPT, CodeQwen) are used as generation tools, and DLVD models are trained/fine-tuned on specific datasets then tested on separate held-out sets. The contamination concern is handled by train/test separation, not by pre-training cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — the paper trains DLVD models and tests them on separate datasets. Pre-trained model benchmark contamination is not the relevant concern here."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates data augmentation effectiveness for training classifiers, not a pre-trained model's zero-shot or few-shot capability on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The manual inspection of 150 generated samples is researcher evaluation work, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data is from public code repositories and vulnerability databases."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 reports 'generating every 1,000 vulnerable samples... costs about $1.88 with GPT3.5-Turbo and about 9 GPU hours with CodeQwen1.5-7B-Chat on two RTX3090s.' Also states 'less than $19 with ChatGPT to augment a dataset like Devign to twice its size.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.8 states 'We conducted a total of 781 experiments for all our RQs, which cost close to 2000 GPU hours solely for training and testing DLVD models' on 'four 24GB Nvidia RTX 3090s.' Generation costs are also quantified in Section 6.3."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "For Devign and Reveal models, the paper trains '5 times with random seed values' but reports only 'the results with the highest F1-score achieved' (Section 4.3). No mean/std across seeds is reported, and the best-of-5 selection masks seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly states '5 times with random seed values' for Devign and Reveal models, and '10 epochs' for LineVul with checkpoint selection on validation set."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper states it followed settings from previous studies (VulGen, VGX) for DLVD models and used mostly default LLM parameters, but does not state how many configurations were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "For Devign and Reveal models, the paper reports 'the results with the highest F1-score achieved' from 5 test-set runs — this is selection on the test set, not the validation set, which inflates reported performance. LineVul correctly selects on the validation set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous pairwise comparisons across strategies, models, datasets, and LLMs without applying any correction for multiple comparisons (no Bonferroni, Holm, or Benjamini-Hochberg mentioned)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge author-evaluation bias. While VGX/VulGen generated results are reused from original papers, the authors implement their own pipeline, DLVD training, and manual quality assessment without discussing self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute budget. The cost analysis in Section 6.3 reports generation cost but does not compare total compute (generation + training) across methods or discuss whether VulScribeR's improvements justify additional compute over baselines."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 explicitly discusses that Devign and BigVul are 'known to be noisy datasets... meaning that a considerable number of vulnerable samples are incorrectly labeled, which can affect the results.' They address this by adding PrimeVul, 'a more recent dataset with considerably less label noise,' in RQ4."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The DLVD models are standard classifiers (graph-based and transformer-based), not scaffolded agents."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The LLMs used for generation (ChatGPT, CodeQwen) may have seen the vulnerability datasets during pre-training, potentially meaning 'generated' samples are memorized. This temporal leakage concern is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the augmentation process introduces feature leakage — e.g., whether generated samples carry artifacts that the DLVD models could exploit as shortcuts rather than learning genuine vulnerability patterns."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses independence: 'we used the cleaned version from VGX, which had duplicates removed between the datasets' and 'we used the SHA2 hash to remove samples available in Devign... from all three sets of the PrimeVul dataset' (Section 4.2). They train and test on different datasets to prevent leakage."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "SHA2 hash deduplication is used to remove PrimeVul samples that overlap with Devign (Section 4.2), and VGX's cleaned versions with duplicates removed between datasets are used. These are concrete deduplication methods applied to prevent data leakage."
       }
     }
   },
   "claims": [
     {
       "claim": "VulScribeR's Injection strategy outperforms NoAug, VulGen, VGX, and ROS by 30.80%, 27.48%, 27.93%, and 15.41% in average F1-score with 5K generated samples.",
       "evidence": "Table 2 (Section 5.1) shows results across 12 experimental instances (3 DLVD models × 2 test datasets × 2 LLMs). Injection and Extension outperform all baselines in all 12 instances in terms of F1-score.",
       "supported": "moderate"
     },
     {
       "claim": "RAG component contributes 4.99% (Injection) and 10.77% (Extension) improvement in average F1-score.",
       "evidence": "Tables 3-4 (Section 5.2) compare full pipeline vs. w/o Retriever and w/o Clustering ablations. Injection w/ full Retriever improves by 4.99% over w/o Retriever; Extension improves by 10.77%.",
       "supported": "moderate"
     },
     {
       "claim": "VulScribeR scales to 15K augmented samples with continued improvement, while VulGen, VGX, and ROS degrade beyond 5K.",
       "evidence": "Table 5 and Figure 6 (Section 5.3) show Injection performance increasing from 5K to 15K in 11/12 instances, while VGX/VulGen/ROS decrease in 5/6, 4/6, and 4/6 instances respectively when going from 5K to 15K.",
       "supported": "moderate"
     },
     {
       "claim": "On PrimeVul (heavily imbalanced, 1:35 ratio), Extension outperforms NoAug, VulGen, VGX, and ROS by 14.54%, 21.14%, 17.73%, and 3.39% in average F1-score.",
       "evidence": "Table 6 (Section 5.4) shows results across 3 DLVD models using GPT4o-mini. Extension outperforms all baselines in all instances.",
       "supported": "moderate"
     },
     {
       "claim": "72-90% of generated samples are correctly vulnerable based on manual inspection (72% Mutation, 82% Injection, 90% Extension).",
       "evidence": "Section 6.4 reports manual assessment of 150 randomly selected samples (50 per strategy) from ChatGPT-3.5 RQ1 results.",
       "supported": "weak"
     },
     {
       "claim": "Generating 1K vulnerable samples costs approximately $1.88 with GPT-3.5-Turbo.",
       "evidence": "Section 6.3 directly reports this cost based on their experiments.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Best-of-5 test-set selection inflates results",
       "detail": "For Devign and Reveal DLVD models, the paper trains 5 times with different seeds and reports 'the results with the highest F1-score achieved' — this selects the best run on the test set rather than using the validation set for selection, inflating reported performance. This is particularly concerning because the same protocol is applied to both VulScribeR and baselines, but the variance may differ across methods."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All claims of outperformance across 781 experiments rely entirely on comparing point estimates. With no significance tests and no reported variance, it is impossible to assess whether observed differences are meaningful or within noise ranges."
     },
     {
       "flag": "No variance or uncertainty reporting despite multiple runs",
       "detail": "Despite training models 5 times with different seeds, only the maximum F1 is reported. Standard deviation, IQR, or any spread measure is absent, making it impossible to assess result stability."
     },
     {
       "flag": "Manual quality assessment by paper authors with no inter-rater reliability",
       "detail": "The 150-sample quality assessment in Section 6.4 was conducted by the authors evaluating their own system's output. No independent evaluators, inter-rater reliability metrics, or blinding procedures are described."
     },
     {
       "flag": "LLM memorization of vulnerability datasets not addressed",
       "detail": "ChatGPT and CodeQwen may have seen Devign, BigVul, and other vulnerability datasets during pre-training. Generated 'new' vulnerable samples could be memorized rather than genuinely novel, which would inflate the apparent effectiveness of augmentation. This is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "VULGEN: Realistic Vulnerability Generation Via Pattern Mining and Deep Learning",
       "authors": ["Yu Nong", "Yuzhe Ou", "Michael Pradel", "Feng Chen", "Haipeng Cai"],
       "year": 2023,
       "relevance": "Core baseline for vulnerability generation via pattern mining and injection, representing SOTA in single-statement vulnerability augmentation."
     },
     {
       "title": "VGX: Large-Scale Sample Generation for Boosting Learning-Based Software Vulnerability Analyses",
       "authors": ["Yu Nong", "Richard Fang", "Guangbei Yi", "Kunsong Zhao", "Xiapu Luo", "Feng Chen", "Haipeng Cai"],
       "year": 2024,
       "doi": "10.1145/3597503.3639116",
       "relevance": "Improved vulnerability generation method with larger-scale pattern mining; key baseline showing limitations of single-statement approaches."
     },
     {
       "title": "Vulnerability Detection with Code Language Models: How Far Are We?",
       "authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim"],
       "year": 2024,
       "arxiv_id": "2403.18624",
       "relevance": "Introduces PrimeVul dataset and evaluates LLM-based vulnerability detection, directly relevant to understanding limitations of current approaches."
     },
     {
       "title": "LineVul: A Transformer-based Line-Level Vulnerability Prediction",
       "authors": ["Michael Fu", "Chakkrit Tantithamthavorn"],
       "year": 2022,
       "doi": "10.1145/3524842.3528452",
       "relevance": "SOTA transformer-based DLVD model used as one of three evaluation models in VulScribeR experiments."
     },
     {
       "title": "Devign: Effective Vulnerability Identification by Learning Comprehensive Program Semantics via Graph Neural Networks",
       "authors": ["Yaqin Zhou", "Shangqing Liu", "Jingkai Siow", "Xiaoning Du", "Yang Liu"],
       "year": 2019,
       "relevance": "Foundational GNN-based vulnerability detection model used both as a DLVD evaluator and as the primary training dataset source."
     },
     {
       "title": "Deep Learning Based Vulnerability Detection: Are We There Yet?",
       "authors": ["Saikat Chakraborty", "Rahul Krishna", "Yangruibo Ding", "Baishakhi Ray"],
       "year": 2020,
       "relevance": "Introduces Reveal dataset and DLVD model, evaluating the state of deep learning for vulnerability detection."
     },
     {
       "title": "GRACE: Empowering LLM-based software vulnerability detection with graph structure and in-context learning",
       "authors": ["Guilong Lu", "Xiaolin Ju", "Xiang Chen", "Wenlong Pei", "Zhilong Cai"],
       "year": 2024,
       "doi": "10.1016/j.jss.2024.112031",
       "relevance": "LLM-based vulnerability detection using RAG and graph structures, representing the LLM-based alternative approach to DLVD."
     },
     {
       "title": "Vul-RAG: Enhancing LLM-based Vulnerability Detection via Knowledge-level RAG",
       "authors": ["Xueying Du", "Geng Zheng", "Kaixin Wang"],
       "year": 2024,
       "arxiv_id": "2406.11147",
       "relevance": "RAG-based approach for LLM vulnerability detection, closely related to VulScribeR's use of RAG for vulnerability understanding."
     },
     {
       "title": "LLM agents can autonomously exploit one-day vulnerabilities",
       "authors": ["Richard Fang", "Rohan Bindu", "Akul Gupta", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2404.08144",
       "relevance": "Demonstrates LLM agents' capability in vulnerability exploitation, relevant to understanding LLMs' security capabilities."
     },
     {
       "title": "Does data sampling improve deep learning-based vulnerability detection? Yeas! and Nays!",
       "authors": ["Xu Yang", "Shaowei Wang", "Yi Li", "Shaohua Wang"],
       "year": 2023,
       "relevance": "Studies data sampling effects on DLVD models, establishing that random oversampling can improve detection — used as a baseline context."
     },
     {
       "title": "Uncovering the limits of machine learning for automatic vulnerability detection",
       "authors": ["Niklas Risse", "Marcel Böhme"],
       "year": 2024,
       "relevance": "Examines fundamental limitations of ML-based vulnerability detection, relevant to understanding the ceiling for data augmentation approaches."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained code model used for embedding code samples in VulScribeR's clustering and retrieval pipeline."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a usable pipeline for augmenting vulnerability datasets with released code and datasets, applicable to practitioners training vulnerability detectors."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm expected intuition that LLMs with RAG produce better code augmentation than simpler pattern-mining approaches."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Generates vulnerable code at scale which has dual-use potential, though framed entirely for defensive improvement of vulnerability detectors."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Standard academic contribution with no controversy or conflict narrative."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Source code and augmented datasets are publicly released on GitHub, enabling replication of the pipeline."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From University of Manitoba, Washington State, and University at Buffalo — not high-profile AI labs."
     }
   }
 }