ai-research-survey

scan.json (20877B)

---

 {
   "paper": {
     "title": "A Survey on Large Language Model (LLM) Security and Privacy: The Good, the Bad, and the Ugly",
     "authors": ["Yifan Yao", "Jinhao Duan", "Kaidi Xu", "Yuanfang Cai", "Zhibo Sun", "Yue Zhang"],
     "year": 2023,
     "venue": "High-Confidence Computing",
     "arxiv_id": "2312.02003",
     "doi": "10.1016/j.hcc.2024.100211"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey of 281 papers categorizes LLM security/privacy research into beneficial applications ('the good', 83 papers), offensive uses ('the bad', 54 papers), and LLM vulnerabilities/defenses ('the ugly', 144 papers). The authors find that LLMs contribute more positively than negatively to cybersecurity, with 17/25 papers concluding LLM-based methods outperform traditional approaches for code security. User-level attacks are the most prevalent offensive application (33 papers), attributed to LLMs' human-like reasoning. Model extraction and parameter extraction attacks remain limited and primarily theoretical.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code release mentioned. A survey could release its paper collection pipeline or analysis scripts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset of the 281 collected papers or their metadata is released. The paper lists papers in tables but does not provide a structured, downloadable dataset."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey with no computational experiments requiring environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions provided for reproducing the literature search or paper collection process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey that does not run experiments or perform statistical aggregation."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are applicable to this narrative literature review."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative meta-analysis is performed; the survey is narrative."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted; paper counts (281) are reported but not as statistical samples."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with multiple runs; this is a literature survey."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare its survey against prior surveys in a structured way. §8 (Related Work) discusses other surveys but only narratively, not with a systematic comparison of coverage or methodology."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The related work section (§8) discusses contemporary surveys from 2023 on LLM security, including works by Caven, Al-Hawawreh, Gupta, Schwinn, and others."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate; this is a survey."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation metrics are applicable to this narrative literature survey."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; this is a survey paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Papers are broken down into Good (83), Bad (54), and Ugly (144), with further sub-categorization (e.g., hardware/OS/software/network/user-level attacks in §5, attack types in §6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where LLM-based methods did NOT outperform SOTA (4/25 papers in Table 2), and limitations of LLMs for various attacks (e.g., model extraction remaining theoretical, Finding V)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Papers with negative findings are reported: Cheshkov et al. found ChatGPT performed 'no better than a dummy classifier' for vulnerability detection; Moumita et al. found higher false positives/negatives (Table 2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about LLMs enhancing code security and data privacy, user-level attacks being most prevalent, and model extraction being limited are all supported by the categorized findings in §4-§6."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims; it categorizes and summarizes existing literature."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims like 'LLMs contribute more positively than negatively to security' (Finding I) based on paper counts (83 good vs 54 bad), without bounding this to its search scope or acknowledging that paper counts don't measure impact magnitude."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure survey/taxonomy paper presenting no empirical results of its own."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses paper counts as a proxy for the relative 'positive vs negative' impact of LLMs on security (Finding I: 83 good vs 54 bad), without acknowledging that the number of papers in a category does not measure actual impact or harm."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used in experiments; this is a survey."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used; this is a survey."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with hyperparameters; this is a survey."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used; this is a survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states it 'conducted a search on Google Scholar and compiled papers related to security and privacy involving LLMs' (§3.2) but does not document search queries, date ranges, inclusion/exclusion criteria, or filtering stages with counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The paper has a Discussion (§7) and Future Directions (§7.2), but no explicit limitations or threats-to-validity section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not acknowledge potential biases in its literature search or categorization methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 explicitly states: 'our primary focus remains steadfastly on matters of security and privacy' and acknowledges that 'LLMs wield multifaceted applications extending beyond security considerations (e.g., social and financial impacts)' which are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The full list of 281 papers and their categorizations is not available as downloadable data. Papers are referenced in-text and tables but not provided as a structured dataset."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'we conducted a search on Google Scholar' (§3.2) but provides no details on search terms, date ranges, or how many results were initially retrieved."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is published papers from Google Scholar."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how the 281 papers were filtered from initial search results. The categorization into good/bad/ugly is described conceptually but the pipeline from search to final corpus is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgement section states: 'This research was supported partly by the NSF award FMitF-2319242.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Drexel University, clearly stated in the author block."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is an independent government funding agency with no financial stake in the survey's conclusions about LLM security."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA diagram, no structured review protocol, no reproducible search queries. The paper states only that it 'conducted a search on Google Scholar' without specifying search terms, inclusion criteria, or screening stages."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey treats all 281 papers equally regardless of methodological quality. No quality scoring rubric or risk-of-bias assessment is applied to included studies."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether its sources are biased toward positive results or whether negative-result studies are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs contribute more positively than negatively to the security community (83 'good' papers vs 54 'bad' papers).",
       "evidence": "Paper counts in §3.2 and Finding I. 83 papers on beneficial applications vs 54 on offensive applications.",
       "supported": "weak"
     },
     {
       "claim": "17 out of 25 researchers concluded that LLM-based methods outperform traditional approaches for code security.",
       "evidence": "Table 2 and Finding II (§4.1). Tabulated comparison of papers with SOTA comparison results.",
       "supported": "moderate"
     },
     {
       "claim": "User-level attacks are the most prevalent LLM-enabled attack category with 33 papers.",
       "evidence": "Figure 3 and Finding IV (§5). Paper counts across hardware/OS/software/network/user attack levels.",
       "supported": "moderate"
     },
     {
       "claim": "Model extraction, parameter extraction, and similar attacks remain primarily theoretical with limited practical exploration.",
       "evidence": "Finding V (§6.1.1). Discussion of limited research on extraction attacks and barriers due to LLM scale and confidentiality.",
       "supported": "moderate"
     },
     {
       "claim": "ChatGPT is the predominant LLM extensively employed in diverse security applications.",
       "evidence": "Finding III, Tables 2 and 3 showing ChatGPT as the most frequently used model across reviewed papers.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No structured review methodology",
       "detail": "The survey says it 'conducted a search on Google Scholar' but provides no search terms, date ranges, inclusion/exclusion criteria, or PRISMA-style flow. The 281-paper corpus cannot be independently verified or reproduced."
     },
     {
       "flag": "Paper counts used as proxy for impact",
       "detail": "Finding I concludes 'LLMs contribute more positively than negatively' based solely on counting 83 'good' vs 54 'bad' papers. Paper counts do not measure the magnitude of impact — one successful malware campaign could outweigh dozens of incremental security improvements."
     },
     {
       "flag": "No quality assessment of included papers",
       "detail": "All 281 papers are treated equally regardless of venue, methodology, or rigor. A preprint describing a theoretical attack receives the same weight as a peer-reviewed empirical study published at IEEE S&P."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section, despite being a survey with significant methodological choices (search strategy, categorization scheme) that could affect conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "Lost at C: A User Study on the Security Implications of Large Language Model Code Assistants",
       "authors": ["G. Sandoval", "H. Pearce", "T. Nys", "R. Karri", "S. Garg", "B. Dolan-Gavitt"],
       "year": 2023,
       "arxiv_id": "2208.09727",
       "relevance": "User study (58 participants) measuring security implications of LLM code assistants, directly relevant to AI-assisted coding safety."
     },
     {
       "title": "Large Language Models for Code: Security Hardening and Adversarial Testing",
       "authors": ["J. He", "M. Vechev"],
       "year": 2023,
       "relevance": "Proposes SVEN for controlling LLMs to generate secure code via continuous prompts, relevant to LLM code generation quality."
     },
     {
       "title": "Examining Zero-Shot Vulnerability Repair with Large Language Models",
       "authors": ["H. Pearce", "B. Tan", "B. Ahmad", "R. Karri", "B. Dolan-Gavitt"],
       "year": 2023,
       "relevance": "Evaluates LLMs for zero-shot vulnerability repair across synthetic and real-world scenarios at IEEE S&P 2023."
     },
     {
       "title": "Universal Fuzzing via Large Language Models",
       "authors": ["C. S. Xia", "M. Paltenghi", "J. L. Tian", "M. Pradel", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2308.04748",
       "relevance": "Fuzz4All: uses LLMs for input generation and mutation in fuzzing, relevant to LLM-assisted testing."
     },
     {
       "title": "PentestGPT: An LLM-Empowered Automatic Penetration Testing Tool",
       "authors": ["G. Deng", "Y. Liu", "V. Mayoral-Vilches"],
       "year": 2023,
       "arxiv_id": "2308.06782",
       "relevance": "Automated penetration testing using LLM domain knowledge, relevant to agentic AI security applications."
     },
     {
       "title": "Jailbroken: How Does LLM Safety Training Fail?",
       "authors": ["A. Wei", "N. Haghtalab", "J. Steinhardt"],
       "year": 2023,
       "arxiv_id": "2307.02483",
       "relevance": "Analysis of how LLM alignment can be manipulated, directly relevant to AI safety evaluation."
     },
     {
       "title": "Extracting Training Data from Large Language Models",
       "authors": ["N. Carlini", "F. Tramer", "E. Wallace"],
       "year": 2021,
       "relevance": "Foundational work on training data extraction from LLMs, relevant to LLM privacy and data leakage."
     },
     {
       "title": "MASTERKEY: Automated Jailbreaking of Large Language Model Chatbots",
       "authors": ["G. Deng", "Y. Liu", "Y. Li"],
       "year": 2024,
       "relevance": "Automated jailbreak prompt generation at NDSS 2024, relevant to LLM safety evaluation methodology."
     },
     {
       "title": "More Than You've Asked For: A Comprehensive Analysis of Novel Prompt Injection Threats",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Comprehensive analysis of prompt injection threats to application-integrated LLMs."
     },
     {
       "title": "Safety-Tuned LLaMAs: Lessons from Improving the Safety of Large Language Models that Follow Instructions",
       "authors": ["F. Bianchi", "M. Suzgun", "G. Attanasio"],
       "year": 2023,
       "arxiv_id": "2309.07875",
       "relevance": "Shows that incorporating 3% safe examples during fine-tuning substantially improves LLM safety."
     },
     {
       "title": "ChatGPT for Vulnerability Detection, Classification, and Repair: How Far Are We?",
       "authors": ["M. Fu", "C. Tantithamthavorn", "V. Nguyen", "T. Le"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT on vulnerability-related tasks over 190,000 C/C++ functions, relevant to LLM code security capability."
     },
     {
       "title": "Large Language Models for Software Engineering: Survey and Open Problems",
       "authors": ["A. Fan", "B. Gokkaya", "M. Harman"],
       "year": 2023,
       "relevance": "Broad survey on LLMs for software engineering, relevant as a comparison survey in the same space."
     }
   ]
 }