ai-research-survey

scan.json (28082B)

---

 {
   "paper": {
     "title": "Review of Generative AI Methods in Cybersecurity",
     "authors": [
       "Yagmur Yigit",
       "William J Buchanan",
       "Madjid G Tehrani",
       "Leandros Maglaras"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2403.08701",
     "doi": "10.48550/arXiv.2403.08701"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "qualitative", "case-study"],
   "key_findings": "This narrative review surveys GenAI applications in cybersecurity across offensive (jailbreaking, phishing, malware generation, automated hacking) and defensive (threat intelligence, vulnerability detection, secure code generation) use cases. The authors informally demonstrate that ChatGPT-4 and Google Gemini remain susceptible to jailbreaking and reverse psychology manipulation despite built-in safeguards, and show GPT-4 can generate attack payloads and educational ransomware code. The paper emphasizes the dual-use nature of GenAI and calls for interdisciplinary collaboration on ethical frameworks, but lacks systematic methodology or quantitative analysis.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code, repository URL, or archive is provided anywhere in the paper. The jailbreaking and malware demonstrations are shown only as screenshots in figures."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset, corpus of reviewed papers, or structured analysis data is released. The paper does not provide any downloadable data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or tool versions are provided. The paper mentions using ChatGPT-4 and Google Gemini but provides no setup details for reproducing the demonstrations."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. Neither the literature review process nor the informal demonstrations include step-by-step instructions for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a narrative review/survey paper with no quantitative experiments producing statistical results."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons are made. The paper is a qualitative literature review with informal demonstrations."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted that would produce effect sizes. The paper reviews and summarizes existing literature."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experimental sample is defined. This is a survey paper."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted. This is a survey paper with qualitative demonstrations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare its scope, methodology, or coverage against prior surveys on GenAI in cybersecurity. Section 1.2 mentions related works but does not systematically compare against them."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No prior surveys are used as baselines for comparison. The paper does not position itself against existing reviews or explain what additional coverage it provides."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey paper — there is no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no experiments are conducted that would require evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not produce system outputs that require human evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no experiments with train/test splits are conducted."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper organizes findings into distinct categories: attacking GenAI (Section 2: jailbreaks, reverse psychology, prompt injection), cyber offense (Section 3: social engineering, phishing, automated hacking, payload generation, malware), cyber defense (Section 4: automation, reporting, threat intelligence, code security, vulnerability detection), and ethical/legal implications (Section 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where GenAI fails or has limitations: ChatGPT hallucinations (Section 5.6), limitations in dynamic code analysis (Section 4.4), ChatGPT's decreased accuracy with job specificity (Section 4.8), and Gemini producing the most code vulnerabilities (Section 4.6)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: existing DAN jailbreak techniques no longer work on current GPT-4 (Section 3.5), ChatGPT and GPT-3 matched only dummy classifier performance for vulnerability detection (Section 4.5, citing Cheshkov et al.), and GPT's accuracy decreases with job specificity (Section 4.8)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are generally matched by the paper content: it provides an overview of GenAI attacks (jailbreaking, prompt injection, reverse psychology in Section 2), cybercrimes (Section 3), defensive applications (Section 4), and ethical considerations (Section 5). The claims are descriptive rather than empirical, and the paper delivers on the described scope."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims like 'GenAI can significantly improve the automation of defensive cyber security processes' (abstract) and 'ChatGPT can simplify the process of launching complex phishing attacks' (citing ref [28]). These claims are supported only by narrative review and informal demonstrations, not by controlled experiments or rigorous causal evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to review 'Generative AI Methods in Cybersecurity' broadly, but the paper focuses almost exclusively on ChatGPT (GPT-3.5/4) and Google Gemini/Bard. Other GenAI systems (WormGPT, FraudGPT, XXXGPT, WolfGPT) are mentioned but not evaluated. The demonstrations are limited to a few informal tests, yet conclusions are drawn about GenAI capabilities in general."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for its observations. For example, when demonstrating jailbreaking, it does not consider whether the responses would actually be useful for real attacks, or whether the generated code would function in practice. No confounds or alternative interpretations are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows that GenAI can produce text that resembles malware code, attack payloads, and phishing emails, but does not distinguish between producing such text and its actual operational viability. Generating skeleton ransomware code (Fig 11) or a rootkit snippet (Fig A6) is treated as evidence that GenAI enables these attacks, without verifying whether the outputs are functional, evasive, or practically deployable."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'ChatGPT 4', 'GPT-4', 'GPT-3.5', and 'Google's Gemini' without specifying exact model versions, API snapshot dates, or access dates for the demonstrations. No version identifiers like 'gpt-4-0613' are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The jailbreaking prompt text is provided verbatim in Section 2.1 (the 'JailBreak' persona prompt). Conversation screenshots showing prompts and responses are included in Figures 2-8. The attack payload and malware prompts are shown in appendix figures."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the ChatGPT or Gemini interactions shown in the paper."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The demonstrations involve direct conversational prompting of ChatGPT and Gemini."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides no description of how the literature was collected, searched, or filtered. There is no search strategy, database selection, keyword specification, or inclusion/exclusion criteria for the reviewed papers."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Discussion (Section 6) and Conclusion (Section 7) mention broad challenges and future directions but do not substantively discuss limitations of the review itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not acknowledge limitations of its informal demonstration methodology, potential selection bias in reviewed literature, or the narrow scope of GenAI tools tested."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does not cover. There is no mention of excluded topics (e.g., GenAI for privacy, GenAI in non-English contexts, open-source LLMs beyond those mentioned), excluded time periods, or explicit non-claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The corpus of reviewed papers is not provided as a structured dataset, conversation logs from demonstrations are not released, and no supplementary materials are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the reviewed literature was collected. There is no mention of which databases were searched, what search queries were used, or what time period was covered."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The reviewed literature is the 'data', but the absence of systematic selection methodology is captured under data_collection_described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The paper goes from referencing works to presenting findings without describing any intermediate filtering, categorization, or analysis process for the reviewed literature."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed via numbered superscripts on the title page (institutions 1-3), following standard academic practice."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be verified. The absence of a funding statement makes this impossible to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/review paper. It does not evaluate a pre-trained model's capability on any benchmark — the informal demonstrations test model safety guardrails, not knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The paper does not evaluate model performance on benchmarks where train/test overlap would be a concern."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. No benchmark evaluation is conducted in this paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. It does not propose a method with its own inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no significant computational requirements beyond the informal demonstrations."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not follow PRISMA or any structured review protocol. There is no flow diagram, no protocol registration, no reproducible search strategy, and no systematic methodology for paper selection. It is a narrative review."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the quality of its source papers. All cited works are treated equally regardless of their methodological rigor — preprints, peer-reviewed papers, blog posts, and company announcements are presented alongside each other without quality differentiation."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper does not consider whether the literature it reviews skews toward positive results for GenAI capabilities or whether negative findings are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "ChatGPT-4 remains vulnerable to jailbreaking prompts despite improved safeguards, though it becomes more robust after exposure to similar prompts in the same chat session.",
       "evidence": "Informal demonstrations shown in Figures 2 and 3 where jailbreaking prompts elicit forbidden responses, and subsequent robustness within the same conversation.",
       "supported": "weak"
     },
     {
       "claim": "Google Gemini is more resistant to jailbreaking than ChatGPT-4, refusing all existing prompts and name-changing scenarios at the beginning of chat.",
       "evidence": "Informal comparison shown in Figure 4, where Gemini refused jailbreaking entries that ChatGPT-4 responded to.",
       "supported": "weak"
     },
     {
       "claim": "Both ChatGPT-4 and Google Gemini can be manipulated through reverse psychology to produce phishing email examples.",
       "evidence": "Demonstrations in Figures 5 and 6 showing both models eventually providing phishing email examples after conversational manipulation.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 can generate functional attack payloads and embed them into files using reverse proxies, leveraging frameworks like Metasploit, Veil, and TheFatRat.",
       "evidence": "Figure 10 shows a script for payload generation. The paper lists six frameworks GPT-4 can use for payload code generation (Section 3.4).",
       "supported": "weak"
     },
     {
       "claim": "GenAI can generate educational ransomware code with basic code obfuscation techniques like renaming and control flow flattening.",
       "evidence": "Figure 11 shows educational ransomware code generated by ChatGPT. Previous DAN jailbreak techniques from [48] were no longer functional at time of research (Section 3.5).",
       "supported": "moderate"
     },
     {
       "claim": "SecurityLLM achieves 98% overall accuracy in identifying 14 types of cyberattacks using LLMs combined with a basic classification model.",
       "evidence": "Cited from Ferrag et al. [86] in Section 4.9. This is a claim from a reviewed paper, not independently verified.",
       "supported": "weak"
     },
     {
       "claim": "LLift, combining static analysis with an LLM, achieves 50% precision and 100% recall on real-world UBI bugs, uncovering 13 new bugs in the Linux kernel.",
       "evidence": "Cited from Li et al. [71] in Section 4.5. This is a claim from a reviewed paper, not independently verified by the authors.",
       "supported": "weak"
     },
     {
       "claim": "GenAI has the potential to dramatically increase cybersecurity standards through automating defenses, enhancing threat intelligence, and improving cybersecurity protocols.",
       "evidence": "Narrative synthesis across Section 4 and the Discussion (Section 6). No original quantitative evidence provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic review methodology",
       "detail": "The paper claims to be a review but follows no PRISMA protocol, has no systematic search strategy, no inclusion/exclusion criteria, no quality assessment of sources, and no reproducible paper selection process. This is a narrative review that launders the signal-to-noise ratio of its heterogeneous sources."
     },
     {
       "flag": "Informal demonstrations presented as evidence",
       "detail": "The jailbreaking, reverse psychology, and code generation demonstrations (Figs 2-11) are ad-hoc conversational tests with no systematic methodology, no repeated trials, no controlled conditions, and no reproducibility information (model versions, dates, hyperparameters). These anecdotal demonstrations are presented as supporting broad claims about GenAI capabilities."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The paper makes sweeping claims about GenAI's transformative potential for both offense and defense in cybersecurity based on a narrative literature review and a handful of informal demonstrations. No original quantitative analysis supports the conclusions."
     },
     {
       "flag": "No quality differentiation among sources",
       "detail": "Peer-reviewed papers, arXiv preprints, blog posts, company press releases, and medium.com articles are cited alongside each other and treated with equal authority. For example, Yandex press releases [4], Microsoft blog posts [5], MIT project pages [6-7], and Medium articles [8] are presented alongside NeurIPS papers [19] without any quality weighting."
     },
     {
       "flag": "Proxy-outcome conflation",
       "detail": "The paper equates GenAI producing text that resembles malware code or phishing emails with GenAI enabling actual cyberattacks. Generating a skeleton rootkit or ransomware snippet (Figs A2, A6, 11) does not establish that these outputs are functional, evasive, or practically deployable, but the paper treats them as evidence of real-world threat."
     },
     {
       "flag": "Missing limitations section",
       "detail": "Despite being a review paper making broad claims about GenAI's impact on cybersecurity, there is no limitations section, no threats to validity, and no discussion of the review's own methodological weaknesses."
     }
   ],
   "cited_papers": [
     {
       "title": "From chatgpt to threatgpt: Impact of generative ai in cybersecurity and privacy",
       "authors": ["Maanak Gupta", "CharanKumar Akiri", "Kshitiz Aryal", "Eli Parker", "Lopamudra Praharaj"],
       "year": 2023,
       "relevance": "Directly examines ChatGPT's capabilities for social engineering attacks, phishing, automated hacking, and malware creation — core survey-scope topics."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "Introduces Codex and evaluates LLM code generation capability on HumanEval, foundational for understanding AI code generation quality."
     },
     {
       "title": "An Analysis of the Automatic Bug Fixing Performance of ChatGPT",
       "authors": ["Dominik Sobania", "Carol Hanna", "Martin Briesch", "Justyna Petke"],
       "year": 2023,
       "arxiv_id": "2301.08653",
       "relevance": "Evaluates ChatGPT for automated program repair on QuixBugs benchmark, relevant to LLM code repair capabilities."
     },
     {
       "title": "PentestGPT: An LLM-empowered Automatic Penetration Testing Tool",
       "authors": ["Gelei Deng"],
       "year": 2023,
       "relevance": "Describes an agentic LLM tool for automated penetration testing, directly relevant to LLM-based security tooling."
     },
     {
       "title": "Exploring the dark side of ai: Advanced phishing attack design and deployment using chatgpt",
       "authors": ["Nicolas Begou", "Jeremy Vinoy", "Andrzej Duda", "Maciej Korczynski"],
       "year": 2023,
       "arxiv_id": "2309.10463",
       "relevance": "Examines ChatGPT's role in automating sophisticated phishing campaigns including website cloning and credential theft."
     },
     {
       "title": "Evaluating llms for privilege-escalation scenarios",
       "authors": ["Andreas Happe", "Aaron Kaplan", "Juergen Cito"],
       "year": 2023,
       "arxiv_id": "2310.11409",
       "relevance": "Benchmarks LLMs on Linux privilege escalation tasks, finding GPT-4 achieves up to 100% success rate in some scenarios."
     },
     {
       "title": "Revolutionizing cyber threat detection with large language models",
       "authors": ["Mohamed Amine Ferrag"],
       "year": 2023,
       "arxiv_id": "2306.14263",
       "relevance": "Introduces SecurityLLM for cybersecurity threat identification achieving 98% accuracy across 14 attack types."
     },
     {
       "title": "The Hitchhiker's Guide to Program Analysis: A Journey with Large Language Models",
       "authors": ["Haonan Li", "Yu Hao", "Yizhuo Zhai", "Zhiyun Qian"],
       "year": 2023,
       "arxiv_id": "2308.00245",
       "relevance": "Introduces LLift combining LLMs with static analysis for bug detection, uncovering 13 new UBI bugs in the Linux kernel."
     },
     {
       "title": "THE FORMAI DATASET: GENERATIVE AI IN SOFTWARE SECURITY THROUGH THE LENS OF FORMAL VERIFICATION",
       "authors": ["Norbert Tihanyi", "Tamas Bisztray", "Ridhi Jain", "Mohamed Amine Ferrag"],
       "year": 2023,
       "arxiv_id": "2307.02192",
       "relevance": "Provides 112K AI-generated C programs with vulnerability classifications, relevant to understanding AI code security."
     },
     {
       "title": "Ocassionally secure: A comparative analysis of code generation assistants",
       "authors": ["Rami Elgedawy"],
       "year": 2024,
       "arxiv_id": "2402.00689",
       "relevance": "Compares security of code generated by GPT-3.5, GPT-4, Google Bard, and Gemini across multiple dimensions."
     },
     {
       "title": "The Scope of ChatGPT in Software Engineering: A Thorough Investigation",
       "authors": ["Wei Ma", "Shangqing Liu", "Wenhan Wang"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT's understanding of code semantics including syntax, static behavior, and dynamic behavior."
     },
     {
       "title": "Defending chatgpt against jailbreak attack via self-reminder",
       "authors": ["Yueqi Xie", "Jingwei Yi", "Jiawei Shao"],
       "year": 2023,
       "doi": "10.1038/s42256-023-00765-8",
       "relevance": "Proposes System-Mode Self-Reminder defense reducing jailbreak success from 67.21% to 19.34%, relevant to LLM safety."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Discusses real attack and defense techniques with GenAI that security practitioners should be aware of, though no tools or actionable methods are released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Mostly confirms widely expected concerns about GenAI dual-use in cybersecurity; does not challenge conventional wisdom."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Directly demonstrates AI-generated malware, rootkits, phishing attacks, data exfiltration scripts, and jailbreaking — highly fear-inducing content for general audiences."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Presents offense vs defense tension but avoids controversial claims about specific companies or products."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Shows conversation screenshots but releases no code, tools, or demos that readers could try."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Prominently features ChatGPT/GPT-4 and Google Gemini — well-known products — but the paper itself is from a non-famous lab."
     }
   }
 }