ai-research-survey

scan.json (30099B)

---

 {
   "paper": {
     "title": "(Security) Assertions by Large Language Models",
     "authors": [
       "Rahul Kande",
       "Hammond Pearce",
       "Benjamin Tan",
       "Brendan Dolan-Gavitt",
       "Shailja Thakur",
       "Ramesh Karri",
       "Jeyavijayan Rajendran"
     ],
     "year": 2023,
     "venue": "IEEE Transactions on Information Forensics and Security",
     "arxiv_id": "2306.14027",
     "doi": "10.1109/TIFS.2024.3372809"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section I.E lists 'Open-sourcing LLM-based framework and benchmarks' as a contribution, but no repository URL, GitHub link, or archive link appears anywhere in the paper text. A stated intention without a concrete link does not satisfy this criterion."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark suite is described in detail (Table I, Listings 1-5) and benchmarks derive from public sources (Hack@DAC, OpenTitan), but no download link or archive for the complete benchmark suite is provided in the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-B mentions '32-core, 2.6 GHz Intel Xeon with 512 GB of RAM running CentOS Linux release 7.9.2009' and 'Siemens Modelsim' as the simulator, but no software dependencies, library versions, Python version, or environment specification file is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The framework pipeline is described conceptually (Figure 1, Section III) but there are no runnable commands or a README-style guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table III and Figures 3-13 report only point estimates (percentages). No confidence intervals, error bars, or uncertainty measures are provided for any accuracy numbers."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'DetailedEx, DetailedCom performed the best with 72.16% accuracy') based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw accuracy percentages but does not compute formal effect sizes. Comparisons like '93.55% vs. 26.54% average' are given without standardized effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark suite contains 10 benchmarks. No justification is given for why 10 benchmarks are sufficient, and no power analysis or discussion of whether this sample size supports the generality of claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "With temperature 0.4 and top_p=1, the model is deterministic. For temperature 0.9, stochastic generation is used, but no variance, standard deviation, or spread measures are reported across repeated queries. n=10 assertions per query are generated but only aggregate percentages are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table V compares code-davinci-002 against three other LLMs: code-cushman-001, codegen-2b-ft, and ChatGPT (Jan 9 2023 version). Additionally, the systematic variation of prompt components serves as an internal comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The LLMs compared (Codex code-davinci-002, code-cushman-001, codegen-2b-ft, ChatGPT Jan 2023) were all contemporary at the time of experimentation (2022-2023). These represented the state of the art for code generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies individual prompt components: example assertions (4 types), comment strings (3 types), design source codes (3 types), assertion beginnings (3 types), synonyms (2 types), temperature (2 values), and frequency penalty (3 values). Sections IV-D.1 through IV-D.7 analyze the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table III reports four metrics: number of assertions generated, % generated assertions compiled, % compiled assertions simulated, and % correct simulated assertions. These capture different stages of the evaluation pipeline."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated through simulation against golden reference assertions. No human expert review of generated assertion quality, correctness, or security relevance is performed. The golden references are manually crafted beforehand but the evaluation itself is entirely automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The best prompt configurations (Table IV) are identified and reported on the same 10 benchmarks used for all evaluations. There is no separation between a validation set for prompt engineering and a held-out test set for final reporting."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table III provides per-benchmark breakdowns for all 10 benchmarks (BM1-BM10). Figures 4-8 and 9-13 show performance broken down by prompt component type across all benchmarks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-F provides detailed analysis of incorrect assertions with specific error types categorized in Listing 9: invalid Verilog syntax, invalid variables, incorrect variable indices, additional logic after assertion, incorrect logic, unwanted logic, and incorrect timing. Multiple concrete examples are shown."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that NoEx configurations produce <2% correct assertions, ChatGPT tends to explain rather than generate assertions (Table V), BM7 achieves only 4.46% correct simulated assertions, and 595 prompt configurations result in less than 5% accuracy. Multiple failure modes are documented."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the authors investigate LLMs for hardware assertion generation, design an evaluation framework, and create a benchmark suite. All three are substantiated in the paper: Table III shows LLM performance, Section III describes the framework, and Table I describes the benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about prompt components affecting performance (e.g., 'DetailedCom combinations performed the best because they elaborate all the details'). The systematic factorial design (2,268 configurations varying one component at a time across a grid) provides adequate support for these component-level causal claims, functioning as a controlled ablation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title '(Security) Assertions by Large Language Models' and conclusions reference 'LLMs' generally, but the main evaluation is on a single LLM (code-davinci-002) with only limited scalability testing on three others (Table V). The benchmark has only 10 designs covering a subset of CWE types. The paper does not explicitly bound its generalization claims to these specific models and benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes patterns in results (e.g., why DetailedCom works better) but does not discuss alternative explanations such as whether the LLM's success could be due to memorizing public Hack@DAC/OpenTitan code rather than genuine assertion generation capability."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section V.B explicitly acknowledges: 'it is possible a generated assertion could be a better representation of the security intent but is marked as incorrect by the simple criteria of mismatch with hand-crafted references.' They also discuss the parameterized signal width simplification in testbenches. The gap between proxy (matching golden reference) and actual outcome (security verification quality) is acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'code-davinci-002', 'code-cushman-001', 'codegen-2b-ft', and 'ChatGPT (Jan 9 2023 version)' (Section IV-A and IV-E)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt structures are provided in Listings 2 and 5. Listing 2 shows the prompt data file with all comment strings, example assertions, and beginning strings. Listing 5 shows a complete prompt string with all components. The template structure (Figure 2) is also detailed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B reports: max tokens=256, stop token='endmodule', top P=1, presence penalty=0, n=10. Temperature varied as {0.4, 0.9} and frequency penalty as {0, 0.5, 1}."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLM is queried directly with prompts and returns completions without any multi-step agent workflow, tool use, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C and Table II document the assertion file generation process including four automated syntax fixes (R1-R4): removing non-ASCII characters, removing characters after 'endmodule', removing triple-quoted strings, and adding missing 'endmodule'. Section III-A documents how benchmark designs were trimmed to <100 lines."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Discussion and Limitations' contains four substantive subsections: V-A Completeness of Benchmark Suite, V-B Simulation Testbench, V-C Use-cases of our Framework, and V-D Future Work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A discusses specific threats: benchmarks representing only a subset of CWE types, comment strings based on human judgment of complexity, reference assertions being 'only one of several possible ways to capture the desired security property.' Section V-B discusses signal width parameterization assumptions and simulation timeout handling."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C states: 'Our work focused on concurrent assertions in separate SVA files, keeping the design files unmodified. Generating assertions like immediate assertions that are written into the hardware designs could be future work.' Section V-A bounds the CWE coverage. Section V-C acknowledges that the framework 'as-is is not usable without some amount of security expertise.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate statistics (percentages, distributions) are reported in tables and figures. The individual 226,800 generated assertions and their evaluation results are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the benchmark suite construction in detail: two manually crafted designs and eight from Hack@DAC and OpenTitan (Table I). Section IV-B describes the experimental setup including hardware, LLM configuration, and the systematic prompt generation process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data sources are standard public benchmarks (Hack@DAC competition designs, OpenTitan open-source SoC) plus two manually crafted designs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the complete pipeline: benchmark suite → prompt generation → LLM query → assertion file generator (with syntax fixing) → simulator → scoreboard. Each stage is described in Sections III-A through III-E with concrete details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII: 'Our research work was partially funded by the US Office of Naval Research (ONR Award #N00014-18-1-2058). This research work is also supported in part by a gift from Intel Corporation.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Texas A&M University, University of New South Wales, University of Calgary, New York University. None of the authors are affiliated with OpenAI (whose product is evaluated)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "ONR (military research funder) and Intel (hardware company) fund the work. Neither is OpenAI. The paper explicitly states: 'This work does not in any way constitute an Intel endorsement of a product or supplier.' The funders do not have a direct stake in whether OpenAI's Codex succeeds at generating assertions."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The Intel gift is disclosed in acknowledgments but there is no explicit declaration of whether authors hold patents, equity, or other financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states Codex was trained on 'all the open-source code on GitHub' (Section II-B) but does not state a specific training data cutoff date for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark designs come from Hack@DAC (public competition) and OpenTitan (open-source SoC), both publicly available on GitHub. Since Codex was trained on GitHub code, there is substantial risk these designs were in the training data. This overlap is never discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Hack@DAC designs and OpenTitan source code were publicly available before Codex's training. The paper does not address whether the LLM may have seen these specific designs or similar assertions during training, which could inflate accuracy results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using LLM-generated assertions compared against golden references."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM outputs on hardware benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes 226,800 API calls to Codex (22,680 per benchmark × 10 benchmarks) plus additional calls for three other LLMs, but reports no API costs, token consumption, or wall-clock time for inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-B mentions the hardware (32-core, 2.6 GHz Intel Xeon, 512 GB RAM) but does not state total compute time, simulation hours, or API expenditure for the 226,800+ queries."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "For temperature=0.4 with top_p=1, the model is deterministic. For temperature=0.9, stochastic outputs are generated, but no seed sensitivity analysis or reporting of result variation across seeds is provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B clearly states: 'n is set to generate 10 assertions for every query' and '2,268 prompts for each benchmark,' totaling 22,680 assertions per benchmark and 226,800 total."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The full grid of configurations is explicitly stated: temperature {0.4, 0.9} × frequency penalty {0, 0.5, 1} = 6 parameter combinations per prompt string. All 2,268 configurations are reported, making the search budget fully transparent."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table IV shows the best configurations ranked by accuracy across all benchmarks. Figure 3 shows the full distribution of all 2,268 configurations. All results are shown, not just the best, making selection transparent."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims across 2,268 configurations and 10 benchmarks without any statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created both the benchmark suite (golden reference assertions, comment strings, prompt templates) and the evaluation framework. They do not acknowledge potential bias from designing both the test and evaluation criteria."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table V compares four LLMs of vastly different sizes (code-davinci-002 vs. codegen-2b-ft) without discussing compute differences. No performance-per-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section V discusses validity extensively: Section V-A acknowledges benchmark may not cover all CWE types, V-B discusses testbench limitations (parameterized signal widths), and V-C acknowledges that 'it is possible a generated assertion could be a better representation of the security intent but is marked as incorrect.' The gap between their evaluation metric and the actual goal is explicitly discussed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. LLMs are queried directly with prompts for single-shot completions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Hack@DAC designs (2019-2021) and OpenTitan code were publicly available before Codex's training. The paper does not discuss whether the LLM may have seen these exact designs or similar assertions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The GoldenDUT prompt configuration includes the correct design source code, which the LLM may have seen paired with assertions during training on GitHub. This potential feature leakage from training data is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 10 benchmarks include 6 from related Hack@DAC competitions and 2 from the same OpenTitan SoC. Structural similarities between benchmarks from the same source are not discussed, nor is potential non-independence between the training data and test benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, decontamination, or temporal analysis to check if benchmark code was in the training data."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs can generate correct hardware security assertions for all 10 benchmarks, with an average accuracy of 26.54% across all prompt configurations and up to 93.55% for the best configuration.",
       "evidence": "Table III shows correct assertions generated for all 10 benchmarks. Table IV shows the best configuration achieving 93.55%. Section IV-C reports 26.54% average accuracy.",
       "supported": "strong"
     },
     {
       "claim": "Detailed comment strings and example assertions are the most important prompt components for generating correct assertions.",
       "evidence": "Section IV-D.1 and Figure 4/11 show (DetailedEx, DetailedCom) achieves 72.16% accuracy. Section IV-D.7 identifies example assertions and comment strings as the dominant factors, with the top 3 combinations producing ~70% of correct assertions.",
       "supported": "moderate"
     },
     {
       "claim": "Design source code type has minimal impact when detailed comments are provided.",
       "evidence": "Section IV-D.2 and Figure 12 show GoldenDUT (48.79%), BuggyDUT (45.59%), and EmptyDUT (48.83%) all achieve similar accuracy with DetailedCom.",
       "supported": "moderate"
     },
     {
       "claim": "Temperature 0.9 produces higher accuracy but fewer total correct assertions compared to 0.4.",
       "evidence": "Section IV-D.5 and Figure 13 show (temp=0.9, freq_penalty=1) at 34.25% accuracy vs. (temp=0.4, freq_penalty=1) at 25.32%. Section IV-D.5 notes temp=0.4 generated ~10% more correct assertions in total.",
       "supported": "moderate"
     },
     {
       "claim": "code-davinci-002 outperforms other LLMs, generating correct assertions for 9/10 benchmarks vs. 7 (code-cushman-001), 4 (codegen-2b-ft), and 5 (ChatGPT).",
       "evidence": "Table V shows per-benchmark success/failure for all four LLMs on the most contextual prompt configuration.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OpenAI's Codex (code-davinci-002) can generate correct hardware security assertions with up to 93.55% accuracy when given detailed prompts, but averages only 26.54% across all 2,268 prompt configurations and 10 benchmarks. The quality of prompt engineering — particularly detailed comment strings and example assertions — is the dominant factor, while design source code type and assertion beginnings have relatively minor impact. The framework was validated across 10 benchmarks covering multiple CWE categories, and error analysis revealed common LLM failures including invalid Verilog syntax, incorrect signal references, and timing errors.",
   "red_flags": [
     {
       "flag": "Training data contamination unaddressed",
       "detail": "All benchmarks derive from publicly available sources (Hack@DAC, OpenTitan) that were likely in Codex's training data (trained on 'all open-source code on GitHub'). The LLM may be recalling or partially memorizing known assertion patterns rather than generating novel ones. This is never discussed."
     },
     {
       "flag": "Small benchmark suite",
       "detail": "Only 10 benchmarks: 2 manually crafted 'toy' designs, 6 from related Hack@DAC competitions, and 2 from OpenTitan. This limited diversity may not support generalizations about LLM capability for hardware security assertions broadly."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "Thousands of comparative claims are made across 2,268 configurations without any statistical significance tests, confidence intervals, or error bars. Differences between configurations may not be statistically meaningful."
     },
     {
       "flag": "Scalability evaluation limited",
       "detail": "The comparison with other LLMs (Table V) uses only a single prompt configuration, with binary correct/incorrect per benchmark. This provides very limited evidence for the claim about relative LLM performance."
     },
     {
       "flag": "Signal width simplification",
       "detail": "Section V-B acknowledges that 32-bit signals were parameterized to 2-bit for simulation feasibility, assuming assertion behavior holds across data widths. This assumption is not validated."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex evaluation paper; the primary LLM evaluated in this work."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["H. Pearce"],
       "year": 2022,
       "relevance": "Directly relevant study assessing security implications of LLM-generated code (Copilot)."
     },
     {
       "title": "Examining Zero-Shot Vulnerability Repair with Large Language Models",
       "authors": ["H. Pearce"],
       "year": 2022,
       "arxiv_id": "2112.02125",
       "relevance": "Evaluates LLMs for security vulnerability repair, closely related to LLM-based security verification."
     },
     {
       "title": "DAVE: Deriving Automatically Verilog from English",
       "authors": ["H. Pearce", "B. Tan", "R. Karri"],
       "year": 2020,
       "relevance": "Pioneering work on fine-tuning LLMs for hardware description language generation from natural language."
     },
     {
       "title": "Benchmarking Large Language Models for Automated Verilog RTL Code Generation",
       "authors": ["S. Thakur"],
       "year": 2023,
       "relevance": "Benchmark study for LLM-based hardware code generation, directly related evaluation methodology."
     },
     {
       "title": "LLM4SecHW: Leveraging Domain-Specific Large Language Models for Hardware Debugging",
       "authors": ["W. Fu"],
       "year": 2023,
       "relevance": "Uses trained LLMs for hardware security bug detection and fixing, a follow-on application of LLMs in hardware security."
     },
     {
       "title": "A Conversational Paradigm for Program Synthesis",
       "authors": ["E. Nijkamp"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "CodeGen model used as one of the baseline LLMs evaluated in this work."
     },
     {
       "title": "From RTL to SVA: LLM-assisted generation of Formal Verification Testbenches",
       "authors": ["M. Orenes-Vera", "M. Martonosi", "D. Wentzlaff"],
       "year": 2023,
       "arxiv_id": "2309.09437",
       "relevance": "Parallel work evaluating LLMs for hardware assertion generation using formal verification rather than simulation."
     },
     {
       "title": "LLM4DV: Using Large Language Models for Hardware Test Stimuli Generation",
       "authors": ["Z. Zhang"],
       "year": 2023,
       "arxiv_id": "2310.04535",
       "relevance": "Evaluates LLMs for hardware test generation, complementary to assertion generation in verification workflow."
     },
     {
       "title": "Can OpenAI Codex and Other Large Language Models Help Us Fix Security Bugs?",
       "authors": ["H. Pearce"],
       "year": 2022,
       "arxiv_id": "2112.02125",
       "relevance": "Evaluates LLM capability for security bug fixing, directly relevant to LLM-assisted security verification."
     }
   ]
 }