ai-research-survey

scan.json (26499B)

---

 {
   "paper": {
     "title": "AQUA-LLM: Evaluating Accuracy, Quantization, and Adversarial Robustness Trade-offs in LLMs for Cybersecurity Question Answering",
     "authors": [
       "Onat Gungor",
       "Roshan Sood",
       "Harold Wang",
       "Tajana Rosing"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2509.13514"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The authors reference external libraries (Unsloth, BitsAndBytes, DeepTeam) but do not release their own evaluation framework code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available benchmarks: CyberBench (reference [18]) and CyberMetric (reference [8]). These are standard public datasets that the authors did not modify in a proprietary way. The adversarial prompts are generated via the DeepTeam framework using a described procedure."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (NVIDIA A100 GPU, 80 GB, 16-core CPU, 32 GB RAM, Linux VM) in Section III-G and libraries used (Unsloth, BitsAndBytes, Transformers) but does not provide a requirements.txt, Dockerfile, or detailed dependency version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. While the methodology sections describe the experimental pipeline conceptually, there are no specific commands or reproduction guides."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '98.75% accuracy', '77.92% ASR'). No confidence intervals, error bars, or uncertainty measures are provided on any metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'fine-tuning substantially improves task performance', 'quantization accelerates inference at the cost of robustness') but relies entirely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'Mistral-7B reaches 100% accuracy on CyberMetric, compared to 70% in its base configuration' (Section IV.1), 'ASR increases from 52.25% in base models to 77.92% in quantized models' (Section IV.2), and 'up to 1.28× speedup' and '16× loss in robustness' (Table II)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 50 and 100 questions per benchmark subset and 50 or 100 adversarial prompts. These are very small sample sizes for the comparative claims being made, and no justification or power analysis is provided for why these sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures across experimental runs are reported. Results appear to be from single runs. There is no mention of multiple seeds or repeated trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares four configurations (base, quantized, fine-tuned, fine-tuned+quantized) across six models, where base models serve as baselines for each model's other configurations. Table I and Figures 2-3 show these comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The six evaluated models are all recent: Meta LLaMA-3.1-8B-Instruct, Mistral-7B-Instruct, Phi-3.5-Mini-Instruct, Foundation-Sec-8B, Qwen 2.5-7B-Instruct, and DeepSeek-R1-Distill. These are contemporary models as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The four configurations (base, quantized, fine-tuned, fine-tuned+quantized) effectively constitute an ablation study showing the individual and combined effects of quantization and fine-tuning on accuracy and robustness."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Accuracy (QA correctness), Attack Success Rate (ASR, adversarial robustness), and Inference Latency (efficiency), as described in Section III-G."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper measuring automated metrics (accuracy, ASR, latency) on multiple-choice QA tasks. Human evaluation of system outputs is not relevant to the claims being made."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C describes explicit train/test splits: CyberMetric uses 850 training and 150 testing samples; CyberBench fine-tunes on SecMMLU (116 questions) and evaluates on CyQuiz (128 questions), a separate subset."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (6 models), per configuration (4 configurations), per dataset (CyberBench and CyberMetric), and per evaluation size (50 and 100 questions) in Tables I and II and Figures 2-5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the expected pattern breaks down, e.g., Finding 5 notes Foundation-Sec-8B achieves 'up to 100% ASR' despite strong accuracy, and Finding 3 notes 'only a few exceptions across the evaluated LLMs, e.g., Meta-Llama-3.1-8B' where the FTQ robustness improvement does not hold."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are prominently reported: quantization alone degrades both accuracy and robustness (Finding 4), cybersecurity-specialized LLMs exhibit 'alarmingly poor robustness' (Finding 5), and fine-tuning alone 'frequently reduces robustness compared to base models' (Conclusion)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that (1) quantization alone yields lowest accuracy and robustness, (2) combining quantization with fine-tuning enhances both robustness and performance. Both claims are supported by the experimental results in Section IV (Tables I-II, Figures 2-5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about the effects of fine-tuning and quantization (e.g., 'fine-tuning substantially improves task performance'). These are supported by controlled comparisons — the same models are evaluated across four configurations that differ only in the applied treatment (fine-tuning, quantization, or both), constituting adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract make broad claims about 'LLMs for Cybersecurity Question Answering' but the study tests only six small open-source models (7-8B parameters) on two benchmarks with only multiple-choice QA format. The paper does not adequately bound its conclusions to this narrow scope — no mention that results may not generalize to larger models, closed-source models, or other QA formats."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, it speculates that FTQ may act as 'a regularization mechanism that enhances model robustness' (Section IV.3) without considering alternatives. No threats-to-validity section or systematic consideration of confounds is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by general names (e.g., 'Meta LLaMA-3.1-8B-Instruct', 'Mistral-7B-Instruct', 'Phi-3.5-Mini-Instruct') but without exact version hashes, checkpoint dates, or Hugging Face model IDs with specific revisions. The paper uses marketing/product names rather than precise reproducible identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes using 'model-specific chat templates' (Section III-C) and formatting prompts as 'instruction-response pairs' but does not provide the actual prompt text used for evaluation or fine-tuning. No prompt templates or examples are included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C and III-E report key hyperparameters: LoRA rank=16, alpha=16, no dropout, 60 training steps, AdamW optimizer, peak learning rate 2×10^-4. Section III-D specifies 4-bit NF4 quantization with BitsAndBytes."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is a standard benchmark evaluation pipeline with no agent loops, tool use, or multi-step reasoning scaffolds."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C documents data preprocessing: loading Hugging Face datasets, standardizing into instruction-response pairs using Unsloth, formatting with model-specific chat templates, and tokenization. Section III-B describes how CyberBench was adapted into multiple-choice QA format. Training/test splits are explicitly stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations, Threats to Validity, or similar section in the paper. The paper goes directly from experimental results to conclusions without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper, whether specific or generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge limitations such as: only small models tested, only multiple-choice QA format evaluated, only one type of adversarial attack used, or that results may not generalize to other cybersecurity tasks or larger models."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual model responses, per-question accuracy breakdowns, individual adversarial prompt results) is made available. Only aggregated metrics in tables and figures are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: CyberBench and CyberMetric are publicly available benchmarks, adversarial prompts are generated using the DeepTeam framework with the IllegalActivity vulnerability class targeting cybercrime scenarios (Section III-F), and GPT-3.5 generates 50 or 100 adversarial examples per configuration."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses standard public benchmarks and automated adversarial prompt generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented across Sections III-B through III-F: dataset loading → preprocessing into instruction-response pairs → tokenization → model inference → evaluation via accuracy and ASR metrics. The adversarial evaluation pipeline (prompt generation via DeepTeam → model querying → GPT-3.5 harmfulness assessment) is also described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding from NSF (with specific award numbers) and DARPA (via PRISM and CoCoSys centers in JUMP 2.0, an SRC program)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors list their affiliation as the Department of Computer Science and Engineering, University of California, San Diego. No commercial product of their own is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF and DARPA, which are government agencies without a direct financial interest in the specific outcome of whether fine-tuned quantized models outperform other configurations."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the six models evaluated. This is relevant because the models may have been trained on data that includes the benchmark questions."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The CyberMetric benchmark questions were generated by GPT-3.5 and the models evaluated could have been trained on similar or identical data. No contamination analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CyberBench and CyberMetric were published before 2025, and the evaluated models (especially 2024-2025 vintage) could have been trained on data containing these benchmarks. The paper does not address this contamination risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports per-question inference latency in seconds for each model configuration (base vs. quantized), with speedup ratios. For example, Meta-Llama-3.1-8B: 0.140s (base) vs. 0.112s (quantized), 1.25× speedup."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware is specified (NVIDIA A100 80GB GPU) but total compute budget (GPU hours for fine-tuning, total experiment wall time, API costs for GPT-3.5 calls used in adversarial prompt generation and harmfulness evaluation) is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Task-specific fine-tuning substantially improves QA accuracy, with fine-tuned models achieving 98.75% on CyberMetric and 97.17% on CyberBench compared to 70.42% and 69.83% for base models.",
       "evidence": "Table I shows accuracy before and after fine-tuning across all six models on both datasets. All models show large accuracy gains (e.g., Mistral-7B: 70% → 100% on CyberMetric). Section IV Finding 1.",
       "supported": "moderate"
     },
     {
       "claim": "Quantization accelerates inference by up to 1.28× but increases ASR from 52.25% (base) to 77.92% (quantized), representing up to 16× robustness loss.",
       "evidence": "Table II provides per-model latency and ASR comparisons between base and quantized configurations. The 16× figure comes from Meta-Llama-3.1-8B (5% → 80% ASR). Section IV Finding 2.",
       "supported": "moderate"
     },
     {
       "claim": "Combining fine-tuning with quantization (FTQ) preserves accuracy while improving robustness over fine-tuning alone, with average ASR of 64.58% vs. 84.08% on CyberBench.",
       "evidence": "Section IV Finding 3 and Figure 4 compare FT vs. FTQ configurations. The paper acknowledges exceptions (e.g., Meta-Llama-3.1-8B). Average FTQ accuracy is 94.92% vs. 97.17% for FT on CyberBench.",
       "supported": "moderate"
     },
     {
       "claim": "The cybersecurity-specialized Foundation-Sec-8B achieves top accuracy but exhibits 'alarmingly poor robustness' with ASR up to 100%.",
       "evidence": "Section IV Finding 5 and Figures 2-3 show Foundation-Sec-8B reaching 100% accuracy in FT/FTQ but ASR up to 100% on CyberMetric for fine-tuned configuration.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning combined with quantization may act as a regularization mechanism that enhances model robustness.",
       "evidence": "Section IV Finding 3 speculates this based on observed lower ASR for FTQ vs. FT models. No mechanistic analysis or theoretical justification is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AQUA-LLM evaluates six small open-source LLMs in four configurations (base, quantized, fine-tuned, fine-tuned+quantized) on cybersecurity QA accuracy, adversarial robustness (prompt injection ASR), and inference latency. Fine-tuning dramatically improves QA accuracy (from ~70% to ~98%), while quantization alone degrades both accuracy and robustness. Combining fine-tuning with quantization preserves most accuracy gains while improving robustness over fine-tuning alone, suggesting FTQ as the preferred deployment configuration. The cybersecurity-specialized Foundation-Sec-8B achieves top accuracy but exhibits severely poor adversarial robustness.",
   "red_flags": [
     {
       "flag": "Very small evaluation samples",
       "detail": "The evaluation uses only 50 and 100 questions per benchmark configuration, and 50 or 100 adversarial prompts. These sample sizes are too small for reliable comparative claims — a few changed answers could flip the relative rankings. No confidence intervals or significance tests compensate for this."
     },
     {
       "flag": "Single-run results with no variance reporting",
       "detail": "All results appear to be from single experimental runs with no repeated trials, seeds, or variance reporting. Given the stochastic nature of LLM inference and adversarial prompt generation, this makes the results unreliable."
     },
     {
       "flag": "Suspiciously high fine-tuning accuracy",
       "detail": "Several models achieve 98-100% accuracy after fine-tuning on small datasets (116-850 training samples) evaluated on 50-100 test questions. This raises concerns about potential train/test contamination, memorization, or the test sets being too easy. No contamination analysis is performed."
     },
     {
       "flag": "GPT-3.5 as adversarial judge",
       "detail": "Both adversarial prompt generation and harmfulness evaluation are performed by GPT-3.5 API calls. This introduces a dependency on a black-box third-party model whose behavior may change over time, and the reliability of GPT-3.5 as a harmfulness classifier is not validated."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or discussion of scope boundaries. This is a significant omission for a paper making broad claims about LLM deployment in safety-critical cybersecurity applications."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "CyberBench and CyberMetric were published before the evaluated models' likely training cutoffs. CyberMetric questions were generated by GPT-3.5, and the models may have been trained on similar or identical data. This is never discussed."
     },
     {
       "flag": "Overlapping test subsets",
       "detail": "Section III-C states that for CyberBench, 'the CyQuiz test set is further partitioned into overlapping subsets of 50 and 100 questions.' The 50-question subset is thus a subset of the 100-question set, meaning the two reported numbers are not independent evaluations."
     }
   ],
   "cited_papers": [
     {
       "title": "CyberBench: A Multi-Task Benchmark for Evaluating Large Language Models in Cybersecurity Applications",
       "authors": ["Z. Liu", "J. Shi", "J. F. Buford"],
       "year": 2024,
       "relevance": "Cybersecurity-specific LLM benchmark used as one of the two evaluation datasets in this study."
     },
     {
       "title": "CyberMetric: A Benchmark Dataset Based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge",
       "authors": ["N. Tihanyi"],
       "year": 2024,
       "relevance": "RAG-based cybersecurity QA benchmark used as one of the two evaluation datasets."
     },
     {
       "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models",
       "authors": ["R. Bhatt", "S. Chennabasappa"],
       "year": 2024,
       "arxiv_id": "2404.13161",
       "relevance": "Comprehensive cybersecurity evaluation suite that identifies safety-utility trade-offs in LLMs, directly relevant to LLM safety benchmarking."
     },
     {
       "title": "Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": ["K. Greshake"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLMs, relevant to adversarial robustness evaluation."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs That Persist Through Safety Training",
       "authors": ["E. Hubinger"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistent deceptive behavior in LLMs surviving safety training, relevant to LLM safety and adversarial robustness."
     },
     {
       "title": "Exploiting LLM Quantization",
       "authors": ["K. Egashira"],
       "year": 2024,
       "arxiv_id": "2405.18137",
       "relevance": "Shows that quantized models can exhibit adversarial behavior not present in full-precision counterparts, directly relevant to quantization safety."
     },
     {
       "title": "Fine-tuning, Quantization, and LLMs: Navigating Unintended Outcomes",
       "authors": ["D. Kumar"],
       "year": 2024,
       "arxiv_id": "2404.04392",
       "relevance": "Demonstrates that fine-tuning and quantization combined can degrade robustness, the key prior work this paper extends."
     },
     {
       "title": "PromptBench: Towards Evaluating the Robustness of Large Language Models on Adversarial Prompts",
       "authors": ["Y. Zhu"],
       "year": 2023,
       "arxiv_id": "2306.04528",
       "relevance": "Benchmark for evaluating LLM robustness to adversarial prompts, relevant to prompt injection evaluation methodology."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu"],
       "year": 2022,
       "relevance": "The parameter-efficient fine-tuning method used in this study, widely used in LLM adaptation research."
     },
     {
       "title": "QLoRA: Efficient Finetuning of Quantized LLMs",
       "authors": ["T. Dettmers", "A. Pagnoni", "A. Holtzman", "L. Zettlemoyer"],
       "year": 2023,
       "relevance": "The quantization library and approach (BitsAndBytes, NF4) used in this paper's quantization pipeline."
     },
     {
       "title": "CyberLLMInstruct: A New Dataset for Analyzing Safety of Fine-Tuned LLMs Using Cyber Security Data",
       "authors": ["A. ElZemity", "B. Arief", "S. Li"],
       "year": 2025,
       "relevance": "Investigates safety risks of fine-tuning LLMs on cybersecurity data, finding increased vulnerability to adversarial prompts."
     },
     {
       "title": "Exploring the Role of Large Language Models in Cybersecurity: A Systematic Survey",
       "authors": ["S. Tian"],
       "year": 2025,
       "arxiv_id": "2504.15622",
       "relevance": "Systematic survey of LLMs in cybersecurity, relevant as a comprehensive review of the field this paper contributes to."
     }
   ]
 }