ai-research-survey

scan.json (32177B)

---

 {
   "paper": {
     "title": "DLAP: A Deep Learning Augmented Large Language Model Prompting Framework for Software Vulnerability Detection",
     "authors": [
       "Yanjing Yang",
       "Xin Zhou",
       "Runfeng Mao",
       "Jinwei Xu",
       "Lanxin Yang",
       "Yu Zhang",
       "Haifeng Shen",
       "He Zhang"
     ],
     "year": 2024,
     "venue": "Journal of Systems and Software",
     "arxiv_id": "2405.01202",
     "doi": "10.48550/arXiv.2405.01202"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DLAP combines DL model predictions with LLM prompting via ICL and COT to improve vulnerability detection. Linevul was the most effective DL model plugin among three tested, outperforming Devign by ~10% MCC on average. DLAP outperformed four prompting baselines (PRol, PAux, PCot, GRACE) across four C/C++ projects, with higher F1 and MCC. Compared to LoRA fine-tuning of Vicuna-13B, DLAP achieved comparable average F1 at significantly lower computational cost but underperformed fine-tuning on larger projects.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub URL is provided: https://github.com/Yang-Yanjing/DLAP.git, referenced in a footnote in Section 1 as 'Data and materials.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The datasets are built from public open-source projects (Chrome, Linux, Android, Qemu) and the GitHub repository is stated to contain data and materials. The underlying vulnerability datasets from Fan et al. and Chakraborty et al. are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 lists DL model hyperparameters including Java versions and specific tool versions (Joern 0.3.1, Neo4j), but no Python version, pip requirements, Docker configuration, or complete dependency list is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the DLAP algorithm (Algorithm 1) and experimental design, but provides no step-by-step reproduction instructions, no README description, and no commands to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3, 5, and 6 report only point estimates (e.g., '40.4', '73.3'). No confidence intervals, error bars, or ± notation appear anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests (p-values, t-tests, Wilcoxon, etc.) are reported. Claims like 'DLAP outperforms' are based solely on comparing raw metric values across frameworks."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage differences with baseline context: 'Linevul surpasses using Devign by an average of 7.2% and 10.5% on F1 and MCC' (Section 5.1), and '10% higher F1 score and a 20% higher MCC' (Section 1). Tables show both DLAP and baseline values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 describes three criteria for project selection (researched by related work, >3000 functions, traceable) but provides no statistical power analysis or justification for why four projects or the specific dataset sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or interquartile ranges are reported. All results appear to be from single experimental runs with no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four prompting baselines (PRol, PAux, PCot, GRACE) are compared in Table 5 (RQ2), and LoRA fine-tuning of Vicuna-13B is compared in Table 6 (RQ3). Three DL models are also compared in Table 3 (RQ1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GRACE (Lu et al. 2024) and the prompting frameworks from Zhang et al. (2023) are recent. LoRA (Hu et al. 2022) is the standard efficient fine-tuning technique. The baselines represent the state of the art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 (Section 5.1, Table 3) systematically compares three DL model variants (Sysevr, Devign, Linevul) as plugins for DLAP, effectively ablating the DL model component to determine its impact."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five evaluation metrics are used: Precision, Recall, F1-score, FPR, and MCC (Section 4.5). Results are reported across all five in Tables 3, 5, and 6."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of DLAP's outputs is conducted. Figure 8 shows one qualitative example of DLAP output matching a fix commit, but there is no systematic human evaluation of detection quality or explanatory text."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'we divided the dataset into training and testing sets with the 8:2 proportion. The training set was used to build DL models, while the testing set was used to evaluate the performance of DLAP.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported per-project (Chrome, Android, Linux, Qemu) across all metrics in Tables 3, 5, and 6, allowing comparison of performance variation across different project characteristics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No systematic failure case analysis is provided. Figure 8 shows one successful example. The paper does not discuss where DLAP produces incorrect predictions, what types of vulnerabilities it misses, or qualitative error analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Table 6) shows DLAP underperforms fine-tuning on Chrome (F1: 52.1 vs 82.0) and Linux (F1: 65.4 vs 70.3). RQ1 shows Sysevr performs poorly as a DLAP plugin (e.g., Chrome MCC 14.6% vs Linevul 37.6%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims DLAP 'outperforms state-of-the-art prompting frameworks...as well as fine-tuning on multiple metrics.' While DLAP does beat prompting baselines, Table 6 shows fine-tuning (Vicuna-13B) achieves higher F1 on Chrome (82.0 vs 52.1) and Linux (70.3 vs 65.4) and higher overall precision (88.7 vs 54.1). The abstract significantly overstates the fine-tuning comparison."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes central causal claims about 'implicit fine-tuning' of LLMs (Equations 3-5, Section 3.3) and that DLAP 'stimulates' LLMs to adapt. The authors acknowledge in construct validity (Section 7) that they 'can not strictly demonstrate that the stimulation produces gradient descent optimization loss.' The causal mechanism is theoretically argued but not empirically verified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Software Vulnerability Detection' generally, but experiments are limited to four C/C++ projects. Section 6.2 discusses generalization to other tasks (library identification, code smell detection) speculatively without bounding the actual results to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 discusses threats to validity but focuses on methodological limitations (DL model choice, LLM choice, observability of internals) rather than alternative explanations for the results. For example, DLAP's improvement could simply be due to providing more contextual information rather than 'implicit fine-tuning,' but this alternative is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures binary classification metrics (precision, recall, F1, FPR, MCC) for vulnerability detection and claims vulnerability detection performance. The measurement granularity matches the claims — they do not inflate function-level detection into broader security claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 specifies 'GPT-3.5-turbo-0125' (includes version date), and the fine-tuning comparison uses 'Llama-13b' / 'Vicuna-13B'. DL models are specified by name (Linevul, Devign, Sysevr) with architecture details in Table 2."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided for all four baselines (PRol, PAux, PCot in Section 4.4). DLAP prompt examples with actual content are shown in Figures 2, 3, 4, and 5. The COT template library is referenced on GitHub."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 provides extensive DL model hyperparameters (learning rates, batch sizes, optimizers, etc.), but no LLM API parameters are reported for GPT-3.5-turbo (temperature, top-p, max tokens, frequency penalty). These significantly affect generation quality."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "DLAP is a prompt construction pipeline, not agentic scaffolding. The LLM is called once with a pre-assembled prompt; there is no agent loop, tool calling by the LLM, retry logic, or memory management."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents the preprocessing: project selection criteria (3 criteria listed), random undersampling of non-vulnerable samples for class balance, and 8:2 train/test split. Table 1 provides raw function and vulnerability counts per project."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' is a dedicated section discussing internal validity, construct validity, and external validity, spanning approximately one page."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Internal validity discusses the specific risk of selecting an erroneous DL model. Construct validity specifically discusses inability to observe LLM internal outputs to verify implicit fine-tuning. External validity identifies GPT-3.5-turbo-0125 as a specific threat — different LLMs may yield different results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound findings to C/C++ projects, function-level detection, or the specific LLM used. Section 6.2 instead speculates about broader applicability without stating scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository (https://github.com/Yang-Yanjing/DLAP.git) is stated to contain data and materials. The underlying vulnerability datasets are from publicly available sources (Chrome, Linux, Android, Qemu projects used by Chakraborty et al. and Fan et al.)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes three criteria for project selection: (1) researched by related work, (2) >3,000 functions, (3) traceable vulnerability fix records. Table 1 provides basic statistics. Source references (Chakraborty et al. [4], Fan et al. [12], Zhou et al. [49]) are cited."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public open-source projects commonly used as benchmarks in vulnerability detection research."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes undersampling and 8:2 splitting but does not report the resulting dataset sizes after undersampling, how many examples were removed at each stage, or the final training/test set sizes per project."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Nanjing University (China) and Southern Cross University (Australia). The authors are academic researchers not affiliated with the evaluated products (GPT, Llama)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Without a funding statement, this criterion cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of GPT-3.5-turbo's training data cutoff date. The model may have been trained on data containing the public vulnerability datasets used for evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Chrome, Linux, Android, or Qemu vulnerability data (all publicly available) could have appeared in GPT-3.5-turbo's training corpus."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The vulnerability datasets from public projects were available online well before GPT-3.5-turbo's training cutoff. Vulnerability fix commits and CVE descriptions are widely indexed. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of automated vulnerability detection."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 reports memory usage (MB), wall-clock time (hours), and GPU memory (GB) for both DLAP and LoRA fine-tuning across all four datasets. Section 5.2 mentions 'cost constraints associated with OpenAI API calls.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 provides GPU memory requirements and training time for each dataset. DLAP ranges from 0.3-0.8 hours and 2.8-6.3 GB GPU, versus fine-tuning at 1.3-11.1 hours and 28.7-31.2 GB GPU."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds, seed sensitivity analysis, or results across different initializations. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from a single run or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 states hyperparameters were 'referenced the parameters reported in the respective research papers.' No search budget, search method, or number of configurations tried is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 (Section 5.1) systematically compares three DL models on the test set and selects Linevul based on superior performance across all metrics and datasets. The selection criterion (best overall F1 and MCC) is clear."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Comparisons are made across 4 datasets × 5 metrics × multiple frameworks with no statistical tests performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own versions of baseline prompting frameworks (PRol, PAux, PCot) and do not acknowledge the bias of evaluating their own system against their own baseline implementations. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 directly compares computational costs (memory, time, GPU) between DLAP and LoRA fine-tuning alongside performance results in Table 6, allowing readers to assess the cost-performance tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "While Section 4.2 references Croft et al. [10] on labeling bias, the paper does not discuss whether binary classification of vulnerability presence at function level actually measures vulnerability detection capability, or whether the chosen metrics capture real-world detection utility."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RQ3 compares DLAP (using GPT-3.5-turbo) against fine-tuning (using Vicuna-13B) — different models with different architectures and training. The performance difference cannot be attributed purely to prompting vs. fine-tuning since the underlying LLMs differ. This confound is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The vulnerability datasets contain historical vulnerability fixes from Chrome, Linux, Android, and Qemu that predate GPT-3.5-turbo's training. The model may have memorized vulnerability patterns or fix commits."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The ICL prompts provide similar code examples with DL model predictions (probabilities), which could leak information about the ground truth labels through the training-set examples."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of non-independence. Training and test examples come from the same projects, so functions may share structural similarities, common patterns, or even be near-duplicates from the same codebase."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Linevul is the most effective DL model for augmenting DLAP, outperforming Devign by 7.2% F1 and 10.5% MCC on average, and Sysevr by 28.4% F1 and 34.0% MCC on average.",
       "evidence": "Table 3 (Section 5.1) shows per-project results across three DL models on all five metrics. Linevul achieves the highest CV in probability distribution (Table 4), indicating more discrete predictions.",
       "supported": "moderate"
     },
     {
       "claim": "DLAP outperforms all state-of-the-art prompting frameworks (PRol, PAux, PCot, GRACE) across all metrics and datasets.",
       "evidence": "Table 5 (Section 5.2) shows DLAP achieves the highest F1 and MCC on all four projects. For example, Chrome F1: DLAP 52.1% vs GRACE 32.6%, Linux MCC: DLAP 56.4% vs PCot 6.5%.",
       "supported": "moderate"
     },
     {
       "claim": "DLAP achieves performance comparable to fine-tuning at significantly lower computational cost.",
       "evidence": "Table 6 (Section 5.3) shows fine-tuning achieves higher F1 on Chrome (82.0 vs 52.1) and Linux (70.3 vs 65.4), but DLAP wins on Android (49.3 vs 46.7) and Qemu (66.7 vs 12.1). Table 7 shows DLAP requires ~5x less GPU memory and ~5-14x less time.",
       "supported": "weak"
     },
     {
       "claim": "DLAP stimulates implicit fine-tuning of LLMs through ICL prompts, producing similar predictive distributions to explicit fine-tuning.",
       "evidence": "Theoretical analysis in Equations 3-5 and Appendix, plus Figure 7 showing similar probability distributions between DLAP and fine-tuning. Authors acknowledge in Section 7 they 'can not strictly demonstrate' the mechanism.",
       "supported": "weak"
     },
     {
       "claim": "DL models with more discrete probability distributions (higher CV) are more suitable as DLAP plugins.",
       "evidence": "Table 4 shows Linevul has the highest CV (2.7 average) and Figure 6 shows its more discrete distribution. This correlates with Linevul's superior performance in Table 3.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Confounded fine-tuning comparison",
       "detail": "RQ3 compares DLAP (GPT-3.5-turbo) against fine-tuning (Vicuna-13B) — different underlying LLMs. Performance differences could be due to model capability differences rather than prompting vs. fine-tuning. This makes the central claim about prompting rivaling fine-tuning unreliable."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All claims of superiority (across 4 datasets × 5 metrics × multiple comparisons) are based on comparing raw numbers with no significance tests, error bars, or variance reporting. With single-run results, observed differences may be within noise."
     },
     {
       "flag": "Abstract overclaims fine-tuning comparison",
       "detail": "The abstract states DLAP outperforms 'fine-tuning on multiple metrics,' but Table 6 shows fine-tuning achieves dramatically higher F1 on Chrome (82.0 vs 52.1) and Linux (70.3 vs 65.4). DLAP's advantage is limited to the smallest dataset (Qemu) and partially to Android."
     },
     {
       "flag": "Severe contamination risk",
       "detail": "GPT-3.5-turbo was likely trained on data containing the public vulnerability fix commits from Chrome, Linux, Android, and Qemu. The model may have memorized vulnerability patterns or even specific fix descriptions, inflating or confounding all results. This is never discussed."
     },
     {
       "flag": "Model selection on test set",
       "detail": "RQ1 selects Linevul as the best DL model based on test set performance (Table 3), then RQ2 evaluates DLAP with Linevul on the same test set. This introduces selection bias — the test set influenced the system configuration."
     },
     {
       "flag": "Single-run results with no variance",
       "detail": "DL model training involves random initialization, and undersampling involves randomness. All results appear to be from single runs with no seed sensitivity analysis or variance reporting, making reproducibility of exact numbers uncertain."
     }
   ],
   "cited_papers": [
     {
       "title": "Deep learning based vulnerability detection: Are we there yet",
       "authors": ["Saikat Chakraborty", "Rahul Krishna", "Yangruibo Ding", "Baishakhi Ray"],
       "year": 2022,
       "relevance": "Empirical study showing DL-based vulnerability detection performance drops 73% on real-world projects, demonstrating generalization challenges."
     },
     {
       "title": "Linevul: A transformer-based line-level vulnerability prediction",
       "authors": ["Michael Fu", "Chakkrit Tantithamthavorn"],
       "year": 2022,
       "relevance": "The DL model selected as the best plugin for DLAP; transformer-based vulnerability prediction achieving strong function-level and line-level results."
     },
     {
       "title": "An empirical study of deep learning models for vulnerability detection",
       "authors": ["Benjamin Steenhoek", "Md Mahbubur Rahman", "Richard Jiles", "Wei Le"],
       "year": 2023,
       "relevance": "Empirical study demonstrating variability between runs and low agreement among DL models for vulnerability detection."
     },
     {
       "title": "Prompt-enhanced software vulnerability detection using chatgpt",
       "authors": ["Chenyuan Zhang", "Hao Liu", "Jiutian Zeng", "Kejing Yang", "Yuhong Li", "Hui Li"],
       "year": 2023,
       "arxiv_id": "2308.12697",
       "relevance": "Baseline prompting framework using role-based and auxiliary information prompts for ChatGPT vulnerability detection."
     },
     {
       "title": "GRACE: Empowering LLM-based software vulnerability detection with graph structure and in-context learning",
       "authors": ["Guanjun Lu", "Xiaojun Ju", "Xiang Chen", "Wenhua Pei", "Zhiyong Cai"],
       "year": 2024,
       "relevance": "State-of-the-art baseline that combines graph structural information with ICL for LLM-based vulnerability detection."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma"],
       "year": 2022,
       "relevance": "Foundational work on chain-of-thought prompting, the core technique used in DLAP's COT component."
     },
     {
       "title": "Why can GPT learn in-context? Language models implicitly perform gradient descent as meta-optimizers",
       "authors": ["Damai Dai", "Yutao Sun", "Li Dong", "Yaru Hao"],
       "year": 2023,
       "relevance": "Theoretical basis for DLAP's claim that ICL produces implicit fine-tuning of LLM attention layers."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["Edward J. Hu", "Phillip Wallis", "Zeyuan Allen-Zhu"],
       "year": 2022,
       "relevance": "State-of-the-art efficient fine-tuning technique used as the fine-tuning baseline in RQ3."
     },
     {
       "title": "SySeVR: A framework for using deep learning to detect software vulnerabilities",
       "authors": ["Zhen Li", "Deqing Zou", "Shouhuai Xu", "Hai Jin"],
       "year": 2021,
       "relevance": "DL framework for vulnerability detection using syntactic/semantic code representation; one of three DL models tested in DLAP."
     },
     {
       "title": "Devign: Effective vulnerability identification by learning comprehensive program semantics via graph neural networks",
       "authors": ["Yaqin Zhou", "Shangqing Liu", "Jingkai Siow", "Xiaoning Du", "Yang Liu"],
       "year": 2019,
       "relevance": "Graph neural network approach to vulnerability detection; one of three DL models tested in DLAP."
     },
     {
       "title": "Software vulnerability detection using large language models",
       "authors": ["Moumita Das Purba", "Arpita Ghosh", "Benjamin J. Radford", "Bill Chu"],
       "year": 2023,
       "relevance": "Evaluation of LLMs for software vulnerability detection, establishing baseline LLM performance levels."
     },
     {
       "title": "Evaluation of ChatGPT model for vulnerability detection",
       "authors": ["Anton Cheshkov", "Pavel Zadorozhny", "Rodion Levichev"],
       "year": 2023,
       "arxiv_id": "2304.07232",
       "relevance": "Found that ChatGPT and GPT-3 do not outperform existing tools for Java vulnerability detection."
     },
     {
       "title": "Data quality for software vulnerability datasets",
       "authors": ["Roland Croft", "M. Ali Babar", "M. Mehdi Kholoosi"],
       "year": 2023,
       "relevance": "Identifies labeling bias in common vulnerability detection datasets, directly motivating DLAP's dataset selection criteria."
     },
     {
       "title": "Exploring distributional shifts in large language models for code analysis",
       "authors": ["Shushan Arakelyan", "Rishiraj Das", "Yi Mao", "Xiang Ren"],
       "year": 2023,
       "relevance": "Studies how distributional shifts affect LLM performance on code analysis tasks, relevant to prompt engineering effectiveness."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "DLAP provides a usable framework for vulnerability detection combining DL models with LLM prompting, with code available on GitHub."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Combines existing techniques (ICL, COT, DL models) in a novel way but does not challenge fundamental assumptions about LLM capabilities."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses software vulnerability detection which is security-relevant, but presents a defensive tool rather than demonstrating novel attack vectors."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no criticism of existing tools or companies, straightforward technical contribution."
     },
     "demo_ability": {
       "score": 1,
       "justification": "GitHub repository exists but requires DL model training, GPT API access, and static tool setup — not trivially runnable."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-3.5-turbo (recognizable product) but from a lesser-known university lab; not from a major AI company."
     }
   }
 }