ai-research-survey

scan.json (33419B)

---

 {
   "paper": {
     "title": "IRIS: LLM-Assisted Static Analysis for Detecting Security Vulnerabilities",
     "authors": [
       "Ziyang Li",
       "Saikat Dutta",
       "Mayur Naik"
     ],
     "year": 2025,
     "venue": "International Conference on Learning Representations (ICLR 2025)",
     "arxiv_id": "2405.17238",
     "doi": "10.48550/arXiv.2405.17238"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "IRIS, a neuro-symbolic system combining LLMs with CodeQL static analysis, detects 55/120 vulnerabilities in CWE-Bench-Java (vs. 27 for CodeQL alone) while reducing average false discovery rate by 5.21%. The approach uses LLMs to infer taint specifications (sources/sinks) and filter false positives via contextual analysis. Even small open-source models like DeepSeekCoder 7B detect 52 vulnerabilities, though contextual analysis only helps models with strong reasoning (GPT-4, Llama-3 70B). IRIS also discovered 4 previously unknown vulnerabilities in latest versions of Java projects.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Two public GitHub repositories are provided: https://github.com/iris-sast/iris for the IRIS tool and https://github.com/iris-sast/cwe-bench-java for the benchmark dataset. Both are referenced in the abstract and Section 4."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "CWE-Bench-Java is publicly available at https://github.com/iris-sast/cwe-bench-java, containing 120 vulnerabilities across 4 CWE classes with scripts to fetch, build, and analyze projects (Section 4)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies CodeQL 2.15.3, LLM model IDs (Table 6), hardware specs (Appendix C.1), and inference parameters (temperature=0, top_p=1, max_tokens=2048). However, no requirements.txt, Dockerfile, or comprehensive environment specification is provided. Java/Maven versions per project are handled by a semi-automated script but not listed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The public repositories include scripts to fetch, build, and analyze Java projects (Section 4: 'The dataset and the corresponding scripts to fetch, build, and analyze the Java projects are available publicly'). CodeQL queries are provided in the appendix (Listings 3-5)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1 and 2 are point estimates (e.g., '55 detected', '84.82% FDR', '0.177 F1') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims IRIS outperforms CodeQL and other baselines based solely on comparing raw numbers (e.g., 55 vs. 27 detected) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Improvements are reported with baseline context throughout: '55 (+28)', '84.82% (↓5.21)', '0.177 (↑0.101)' in Table 1, and per-CWE deltas in Table 2. Relative improvement is also stated: '103.7% more than CodeQL'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 120 vulnerabilities across 4 CWE classes (55/13/31/21 split). No power analysis or justification for why 120 is sufficient, and CWE-78 has only 13 instances which is very small."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs with temperature=0 and fixed seeds for GPT models. No standard deviation, variance, or spread measures are reported across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: CodeQL with built-in security queries, Facebook Infer, SpotBugs, and Snyk (Table 2, Section 5.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeQL 2.15.3 (2024), Snyk, Facebook Infer are contemporary and widely-used static analysis tools. These represent the state of the art in static vulnerability detection."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 ablates source/sink specifications (LLM-inferred vs. CodeQL), showing both are necessary. Figure 9 ablates contextual analysis, showing its impact varies by LLM reasoning capability."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: #Detected (recall), Average FDR (precision proxy), and Average F1 Score (Section 3.6, Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual analysis of 50 random alarms found 27/50 exhibit potential attack surfaces (Section 5.2). Additionally, 960 randomly selected LLM-inferred labels were manually evaluated for precision (Section 5.4, Fig. 7)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit train/dev/test split is described. The motivating example (cron-utils, Section 2) is from CWE-Bench-Java itself. Hyperparameters like context window size (±5 lines) and intermediate steps (S=10) were tuned with observations like 'we observed that setting S to 10 provides a good balance' (Appendix A.3), without specifying whether this tuning used benchmark data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-CWE breakdown of detected vulnerabilities for all baselines and IRIS variants (CWE-22, CWE-78, CWE-79, CWE-94)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses CWE-78 being 'particularly challenging' due to 'gadget-chains or external side effects.' Appendix C.7 provides detailed error analysis with three main causes of undetected vulnerabilities and two LLM-induced failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Llama-3 8B underperforms CodeQL on CWE-22 (19 vs. 22 detected, Table 2). Contextual analysis hurts smaller models' F1 scores (Fig. 9). CWE-78 remains challenging for all LLMs (max 3/13 detected)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'detects 55 (+28)' matches Table 1, 'improves upon CodeQL's average false discovery rate by 5% points' matches Table 1 (90.03% → 84.82%), '4 previously unknown vulnerabilities' is supported in Section 5.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('IRIS improves detection') are supported by controlled ablation studies: Table 3 shows removing LLM-inferred sources or sinks drastically reduces detection, and Fig. 9 isolates the effect of contextual analysis. Single-variable manipulation is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'LLM-Assisted Static Analysis for Detecting Security Vulnerabilities' is broader than what was tested: only Java projects, only 4 CWE classes, only CodeQL as the static analysis backbone. While limitations mention 'it is unknown if IRIS will perform well on other languages' (Section 7), the title and abstract frame the contribution more broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for IRIS's improvement: whether CodeQL's baseline configuration was suboptimal, whether CWE-Bench-Java selection favors specification-gap approaches, or whether improvements stem from expanded search space rather than LLM intelligence. Section 5.4 discusses over-approximation benefiting IRIS but this explains mechanism, not alternatives."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures vulnerability detection (known CVEs found) and FDR directly. These metrics closely match the claimed capability. The paper also acknowledges the FDR metric's limitation: 'the reported FDR is an upper bound' since true unknown vulnerabilities found by IRIS are counted as false positives (Section 5.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model IDs are specified in Table 6: gpt-4-0125-preview, gpt-3.5-turbo-0125, meta-llama/Meta-Llama-3-8B-Instruct, meta-llama/Meta-Llama-3-70B-Instruct, deepseek-ai/deepseek-coder-7b-instruct, Qwen/Qwen2.5-Coder-32B-Instruct, google/gemma-2-27b-it."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Listings 1 and 2 (Appendix A.2) for specification inference, and Fig. 4 shows the contextual analysis prompt structure. The IRIS source code repository (https://github.com/iris-sast/iris) is linked for complete prompt access. However, the specific few-shot examples and contextual analysis system prompt are not shown in the paper text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3 reports: temperature=0, max_tokens=2048, top_p=1 for all LLMs, batch sizes of 20 (internal) and 30 (external), few-shot counts per CWE (3-4 examples), and fixed seeds for GPT models."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline is thoroughly described: candidate extraction (Section 3.2), LLM specification inference (Section 3.3), CodeQL taint analysis (Section 3.4), and contextual analysis filtering (Section 3.5). Fig. 3 provides a visual overview. CodeQL queries are shown in Appendix A.4."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Fig. 5 document the full dataset curation pipeline with counts at each stage: 1065 initial CVEs → 430 with GitHub URL → 265 with fix commits → 149 compilable → 120 after manual validation. Filtering criteria are stated at each step."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Conclusion and Limitations' contains a dedicated limitations paragraph discussing undetected vulnerabilities, LLM call costs, language generalizability, and the gap between generated and developer-desired reports."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: 'many vulnerabilities that IRIS cannot detect' with error analysis in Appendix C.7, 'IRIS makes numerous calls to LLMs... increasing the potential cost of analysis', 'it is unknown if IRIS will perform well on other languages', CWE-78 being inherently difficult for static analysis (Section 5.2)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states: 'it is unknown if IRIS will perform well on other languages,' 'there is still a gap between the IRIS generated report and the report that the developers would like to see.' Section 5.2 notes that CWE-78 'highlights the inherent limitations of static analysis, as opposed to dynamic approaches—an area that we leave for future work.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "CWE-Bench-Java is publicly available with all project data, CVE metadata, fix commits, and build scripts at https://github.com/iris-sast/cwe-bench-java. The IRIS tool is also public at https://github.com/iris-sast/iris."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Fig. 5 describe the complete data collection procedure: starting from GitHub Advisory Database, cross-validating with libraries.io and Snyk, obtaining git information, building with Maven, and manual validation. Table 4 shows detailed statistics at each step."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The data selection process is thoroughly described: CVEs from GitHub Advisory Database filtered by 4 CWE classes and Java Maven packages, cross-validated with libraries.io and Snyk for fix information, compiled with semi-automated Java/Maven version selection, and manually validated by two co-authors with cross-checking (Section 4, Appendix B.1)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 5 and Table 4 document the full pipeline from 1065 initial CVEs through 6 filtering stages to 120 final vulnerabilities, with counts and criteria at each step. Appendix B.1 provides additional details on exclusion criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This research was supported by NSF award CCF 2313010.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Pennsylvania (Ziyang Li, Mayur Naik) and Cornell University (Saikat Dutta). No evaluated product affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is a government funding agency with no financial stake in whether IRIS outperforms CodeQL or any other tool."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the 7 LLMs used. GPT-4-0125-preview, Llama-3, DeepSeekCoder, etc. are used without stating when their training data ends. Since CVE information is public, the models may have seen vulnerability details during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "CWE-Bench-Java contains known CVEs with public descriptions, advisories, and fix commits. GPT-4 may have seen detailed information about these specific vulnerabilities during training (e.g., CVE descriptions, fix patches). This potential contamination is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The CVEs in CWE-Bench-Java are public and pre-date GPT-4's training. The LLMs may have seen vulnerability descriptions, source/sink patterns, or even fix commits for these exact CVEs. No contamination analysis or mitigation is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated with manual validation of benchmark and outputs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes open-source software projects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 (Appendix D) provides wall-clock runtime per project for all 120 projects using Llama 3 8B, ranging from 43 seconds to 4 hours. However, API costs for closed-source models (GPT-4, GPT-3.5) are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Intel Xeon machines with RTX 2080 Ti and A100 GPUs, Appendix C.1) and per-project runtimes are given (Table 10), but total computational budget (total GPU hours, total API spend) is not stated. The limitations section acknowledges 'IRIS makes numerous calls to LLMs... increasing the potential cost' without quantifying."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 and seeds are fixed for GPT models (Appendix C.3), but no sensitivity analysis across different seeds or configurations is reported. Only single-configuration results are presented."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state how many runs produced the results. The use of temperature=0 and fixed seeds implies single runs, but this is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters include batch sizes (20/30), context window (±5 lines), intermediate steps (S=10), and few-shot counts (3-4). Appendix A.3 states 'we observed that setting S to 10 provides a good balance' but no systematic search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "While all 7 LLM results are reported (not just the best), the selection of hyperparameters (batch size, context window, few-shot examples) is not justified through validation-set evaluation. It is unclear whether these were tuned using CWE-Bench-Java projects."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims across 7 LLMs, 4 CWE classes, and multiple ablation configurations without any statistical tests or multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors built both IRIS and CWE-Bench-Java, and wrote custom CodeQL queries for the evaluation. The potential for self-comparison bias (e.g., benchmark design favoring their approach, or baselines not being optimally configured) is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 7B to GPT-4 are compared without discussing compute differences. Table 10 shows runtimes only for Llama 3 8B. The significant cost difference between running a 7B model locally and calling GPT-4 API is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether detecting known CVEs in prepared benchmark projects generalizes to detecting unknown vulnerabilities in the wild. The 4 new vulnerabilities (Section 5.3) partially validate practical utility but no systematic construct validity analysis is provided."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The IRIS pipeline (candidate extraction → LLM labeling → CodeQL analysis → contextual filtering) is held constant across all 7 LLM comparisons in Table 1. Only the LLM component varies, making model comparisons fair with respect to scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CWE-Bench-Java contains public CVEs from before GPT-4's training cutoff. The LLMs may have seen vulnerability descriptions, fix patches, or even CodeQL discussions about these specific CVEs. This temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The LLMs are given CWE descriptions and API signatures as input features. If the models have memorized associations between specific APIs and vulnerability types from training data (e.g., from CVE databases), this constitutes feature leakage. Not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several projects appear multiple times in CWE-Bench-Java with different CVEs (e.g., multiple xstream, keycloak, DSpace entries in Table 10). Non-independence between these related evaluations is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference, temporal splits, or decontamination pipelines are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "IRIS with GPT-4 detects 55 vulnerabilities (103.7% more than CodeQL's 27) on CWE-Bench-Java",
       "evidence": "Table 1 shows 55 detected for IRIS+GPT-4 vs. 27 for CodeQL across 120 vulnerabilities. Table 2 provides per-CWE breakdown confirming the aggregate numbers.",
       "supported": "strong"
     },
     {
       "claim": "IRIS with GPT-4 improves average false discovery rate by 5.21 percentage points over CodeQL",
       "evidence": "Table 1 reports 84.82% AvgFDR for IRIS+GPT-4 vs. 90.03% for CodeQL. Manual analysis of 50 random alarms yields an estimated refined FDR of 46% (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "IRIS identifies 4 previously unknown vulnerabilities in latest versions of Java projects",
       "evidence": "Section 5.3 describes application to latest versions of 30 Java projects, finding 3 CWE-22 and 1 CWE-94 vulnerabilities. One example (alluxio Zip-Slip) is shown in Fig. 8. CodeQL alone could not detect them.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-inferred sink specifications can replace CodeQL sinks with high recall (87.11% for GPT-4)",
       "evidence": "Fig. 6 shows recall of LLM-inferred specifications against CodeQL's known specifications. GPT-4 achieves 87.11% sink recall. Fig. 7 shows manual precision evaluation of 960 samples with GPT-4 achieving >70% precision.",
       "supported": "strong"
     },
     {
       "claim": "Both LLM-inferred sources and sinks are necessary for IRIS's performance",
       "evidence": "Table 3 ablation: replacing either LLM sources or sinks with CodeQL specs reduces detection from 55 to 36 or 24 respectively. The ablation is clean single-variable manipulation.",
       "supported": "strong"
     },
     {
       "claim": "Contextual analysis effectiveness depends on the LLM's reasoning capability",
       "evidence": "Fig. 9 shows contextual analysis improves precision/F1 for GPT-4, GPT-3.5, and Llama-3 70B but hurts smaller models (Llama-3 8B, DeepSeekCoder 7B). 'Smaller models are more likely to respond with vulnerable than larger models' (Section 5.5).",
       "supported": "moderate"
     },
     {
       "claim": "Smaller specialized LLMs like DeepSeekCoder 7B can still detect 52 vulnerabilities, close to GPT-4's 55",
       "evidence": "Table 1 shows DSC 7B detecting 52 vs. GPT-4's 55, but with much higher FDR (95.40% vs. 84.82%) and lower F1 (0.062 vs. 0.177).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "CWE-Bench-Java contains known CVEs with public descriptions, fix commits, and advisory database entries. GPT-4 and other LLMs may have seen detailed information about these specific vulnerabilities during training (e.g., which APIs are sources/sinks for these CVEs). This could inflate IRIS's performance. The paper does not discuss or test for this contamination."
     },
     {
       "flag": "No statistical testing on any comparison",
       "detail": "All claims of superiority (IRIS vs. CodeQL, GPT-4 vs. other LLMs) are based on comparing raw numbers without significance tests, confidence intervals, or variance estimates. With only 120 vulnerabilities and small per-CWE counts (CWE-78: 13), differences may not be statistically significant."
     },
     {
       "flag": "Authors built both the system and the benchmark",
       "detail": "The same team created IRIS, CWE-Bench-Java, and the custom CodeQL queries used for evaluation. The benchmark curation process (which CVEs to include, how to label vulnerable locations) could unconsciously favor the approach. No independent evaluation or self-comparison bias acknowledgment."
     },
     {
       "flag": "Very high FDR even for best configuration",
       "detail": "IRIS with GPT-4 has an 84.82% average false discovery rate—meaning roughly 85% of alarms are false positives. While this is 5% better than CodeQL, it still represents a large developer burden. The manually estimated 46% FDR (from 50 samples) is substantially different from the automated metric, raising questions about metric validity."
     },
     {
       "flag": "Non-independent benchmark entries",
       "detail": "Multiple entries in CWE-Bench-Java come from the same project (e.g., keycloak, DSpace, xwiki-commons appear multiple times in Table 10). Performance on these correlated entries is treated as independent, which could bias aggregate results."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-Bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks; cited as evidence that LLMs struggle with project-level reasoning."
     },
     {
       "title": "Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities",
       "authors": ["Avishree Khare", "Saikat Dutta", "Ziyang Li", "Alaia Solko-Breslin", "Rajeev Alur", "Mayur Naik"],
       "year": 2023,
       "arxiv_id": "2311.16169",
       "relevance": "Directly evaluates LLMs on method-level vulnerability detection, showing LLMs fail at complex code reasoning without context—motivating IRIS."
     },
     {
       "title": "A Comprehensive Study of the Capabilities of Large Language Models for Vulnerability Detection",
       "authors": ["Benjamin Steenhoek", "Md Mahbubur Rahman", "Monoshi Kumar Roy", "Mirza Sanjida Alam", "Earl T. Barr", "Wei Le"],
       "year": 2024,
       "arxiv_id": "2403.17218",
       "relevance": "Studies LLMs' effectiveness at detecting vulnerabilities, demonstrating limitations that motivate combining LLMs with static analysis."
     },
     {
       "title": "Enhancing Static Analysis for Practical Bug Detection: An LLM-Integrated Approach",
       "authors": ["Haonan Li", "Yu Hao", "Yizhuo Zhai", "Zhiyun Qian"],
       "year": 2024,
       "relevance": "Concurrent work combining LLMs with static analysis for bug detection, directly related to the IRIS approach."
     },
     {
       "title": "LLMDFA: Analyzing Dataflow in Code with Large Language Models",
       "authors": ["Chengpeng Wang", "Wuqi Zhang", "Zian Su", "Xiangzhe Xu", "Xiaoheng Xie", "Xiangyu Zhang"],
       "year": 2024,
       "relevance": "Uses LLMs for dataflow analysis in code, a complementary approach to IRIS's neuro-symbolic pipeline."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-Trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Evaluates LLMs for automated program repair, demonstrating LLM capabilities in code understanding and modification."
     },
     {
       "title": "CODAMOSA: Escaping Coverage Plateaus in Test Generation with Pre-trained Large Language Models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K. Lahiri", "Siddhartha Sen"],
       "year": 2023,
       "relevance": "Combines LLMs with traditional test generation tools, exemplifying the neuro-symbolic approach to software engineering."
     },
     {
       "title": "Fuzz4All: Universal Fuzzing with Large Language Models",
       "authors": ["Chunqiu Steven Xia", "Matteo Paltenghi", "Jia Le Tian", "Michael Pradel", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Uses LLMs for universal fuzzing, demonstrating LLMs' utility in security testing—related to IRIS's vulnerability detection goals."
     },
     {
       "title": "Vulnerability Detection with Code Language Models: How Far Are We?",
       "authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim", "Chawin Sitawarin", "Xinyun Chen", "Basel Alomair", "David Wagner", "Baishakhi Ray", "Yizheng Chen"],
       "year": 2024,
       "arxiv_id": "2403.18624",
       "relevance": "Evaluates code language models for vulnerability detection, finding limited effectiveness—supporting IRIS's motivation for neuro-symbolic approaches."
     },
     {
       "title": "Large Language Models for Code: Security Hardening and Adversarial Testing",
       "authors": ["Jingxuan He", "Martin Vechev"],
       "year": 2023,
       "relevance": "Studies LLMs for code security, including hardening and adversarial testing, directly relevant to AI-assisted security analysis."
     },
     {
       "title": "Lost in Translation: A Study of Bugs Introduced by Large Language Models While Translating Code",
       "authors": ["Rangeet Pan", "Ali Reza Ibrahimzada", "Rahul Krishna"],
       "year": 2024,
       "relevance": "Studies bugs introduced by LLMs during code translation, relevant to understanding LLM reliability in code tasks."
     },
     {
       "title": "Large Language Models for Test-Free Fault Localization",
       "authors": ["Aidan ZH Yang", "Ruben Martins", "Claire Le Goues", "Vincent J. Hellendoorn"],
       "year": 2023,
       "arxiv_id": "2310.01726",
       "relevance": "Uses LLMs for fault localization without test cases, combining LLMs with program analysis for debugging."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "IRIS is an open-source tool that can be applied to real Java projects and found 4 real unknown vulnerabilities, but requires CodeQL setup and LLM API access."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Combining LLMs with static analysis is a natural idea; the specific 2x improvement over CodeQL is impressive but not contrarian."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Directly addresses security vulnerability detection and found previously unknown vulnerabilities in production software."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or dramatic claims; straightforward system paper showing improvements over baselines."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Open-source code and benchmark on GitHub, but requires CodeQL installation, Java project compilation, and LLM API access to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From UPenn and Cornell (respected but not headline labs); uses GPT-4 (well-known) but the tool itself is new."
     }
   }
 }