ai-research-survey

scan.json (29198B)

---

 {
   "paper": {
     "title": "Enhancing Android Malware Detection with Retrieval-Augmented Generation",
     "authors": [
       "Saraga S.",
       "Anagha M. S.",
       "Dincy R. Arikkat",
       "Rafidha Rehiman K. A.",
       "Serena Nicolazzo",
       "Antonino Nocera",
       "Vinod P."
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2506.22750",
     "doi": "10.48550/arXiv.2506.22750"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or archive is provided anywhere in the paper. The system components (AgenticRAG pipeline, BERT classifiers) are described but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The raw data source (AndroZoo) is publicly available, but the authors' specific compiled dataset of 10,000 benign and 8,000 malicious APKs, their VirusTotal labels, and their generated functional descriptions are not released. The Data Corpus of Android feature descriptions is also not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, hardware descriptions, or library versions are provided. The paper mentions using Androguard, FAISS, BM25, HuggingFace, and various LLMs but specifies no versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are provided. A researcher would have to reconstruct the entire pipeline from the prose description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 5-8 are reported as point estimates only (e.g., '92.89% accuracy'). No confidence intervals, error bars, or uncertainty measures are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AgenticRAG outperforms Gemini Fusion and CySecBERT outperforms SecBERT based solely on comparing raw numbers (e.g., 92.89% vs 91.36%). No statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 5-8 report absolute performance values for all compared systems (e.g., AgenticRAG 92.89% vs Gemini Fusion 91.36% accuracy), providing baseline context for interpreting differences across accuracy, precision, recall, and F1-score."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset comprises 10,000 benign and 8,000 malicious samples, but no justification is given for why this size was chosen or whether it is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run numbers. No standard deviations, variance across seeds, or multiple-run statistics are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares AgenticRAG vs Gemini Fusion (Table 7), CySecBERT vs SecBERT (Tables 5-6), and three fusion models (Gemini, Llama, Mistral) in Table 8, providing internal baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper cites AppPoet (2025) achieving 99.3% detection accuracy and HinDroid achieving 98.3%, but does not benchmark against any external prior methods on the same dataset. All comparisons are internal between the authors' own system variants."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (RAG retrieval, cache, fallback LLM querying, BERT classifier) but no ablation study isolates the contribution of individual components. The comparison between AgenticRAG and Gemini Fusion is between two different systems, not an ablation of one."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used throughout: accuracy, precision, recall, and F1-score (Section 4.2, Tables 5-8)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated descriptions or classification outputs. All evaluation is automated via metrics on the test set."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'the dataset is partitioned into training, validation, and testing subsets with a distribution ratio of 70:10:20' and 'comprehensive evaluation is performed on the segregated test set.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are only reported as aggregate binary classification (benign vs malicious). No per-malware-family, per-category, or per-difficulty breakdown is provided. Confusion matrices show TP/FP/FN/TN but no finer granularity."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases, error analysis, or qualitative examples of misclassified samples are discussed. The paper only reports aggregate metrics."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 shows that Llama fusion (87.97% accuracy) and Mistral fusion (88.89%) substantially underperform Gemini fusion (91.36%), and that SecBERT underperforms CySecBERT on key metrics. These are configurations that were tried and found inferior."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'improving detection accuracy over conventional feature-based methods for malware detection,' but the paper never compares against conventional feature-based methods. It only compares its own two description generation approaches. The contribution list claims 'outperforms state-of-the-art baselines' (Section 1) but no external SOTA baselines are tested."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3.3 claims AgenticRAG's superiority is due to 'Enhanced Contextual Specificity,' 'Improved Edge Case Handling,' 'Semantic Coherence,' and 'Malware-Specific Terminology.' These are post-hoc explanations with no controlled experiments to isolate each factor. The paper uses causal language ('can be attributed to') without adequate causal design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims the approach addresses 'increasingly complex challenges in security-critical applications' and mentions extension to 'network traffic analysis, vulnerability assessment, and threat intelligence' (Section 1). Results are on a single dataset from one source (AndroZoo) with one labeling scheme. No bounding of generalization to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for why AgenticRAG outperforms Gemini Fusion. The difference could be due to RAG grounding, the number of LLM calls, prompt design differences, or random variation, but none are explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses VirusTotal with a threshold of ≥1 antivirus detection as ground truth for 'malicious.' This is a proxy—a single AV detection may be a false positive—but the paper does not acknowledge this gap. Additionally, static-analysis-based classification is framed as 'malware detection' without discussing the gap between classification of APK descriptions and real-world malware detection."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Gemini 2.0 Flash Lite,' 'LLaMA2,' and 'Mistral' without snapshot dates or API versions. CySecBERT and SecBERT are referenced by HuggingFace links (footnotes 5-6) but no specific model checkpoint versions. Marketing names without version snapshots do not count as specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 3, and 4 provide the actual prompt text used for AgenticRAG description generation, Llama/Mistral description generation, and Gemini fusion respectively, including the full templates with variable placeholders and response format instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM inference hyperparameters (temperature, top-p, max tokens) are reported. BERT fine-tuning hyperparameters (learning rate, batch size, epochs, optimizer) are not specified. Only 'early stopping mechanism implemented based on validation loss trajectory' is mentioned with no details."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 and Figure 2 describe the AgenticRAG architecture in detail: feature normalization, FAISS + BM25 ensemble retrieval with configurable weights, advanced fuzzy matching (Levenshtein distance, 65% threshold), cache memory, fallback LLM querying, and structured output generation pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 documents NLP preprocessing (text cleaning, lowercasing, stopword removal, Porter stemming). Section 3.2 describes static feature extraction categories (permissions, receivers, services, intent actions). Section 3.1 describes the VirusTotal labeling protocol."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion (Section 5) mentions future work directions (dynamic analysis, multi-platform) which implicitly acknowledge limitations, but there is no substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues such as the aggressive VirusTotal labeling threshold, dataset representativeness, temporal validity of malware labels, or the reliability of LLM-generated descriptions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what results do NOT show. The future work section mentions extending to dynamic analysis and other platforms, but does not explicitly bound the current claims to the tested setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "AndroZoo is publicly available, but the authors' specific selection of 10,000 benign and 8,000 malicious APKs, their VirusTotal labels, the generated functional descriptions, and the Data Corpus are not released. Results cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data acquisition from AndroZoo using SHA256 hashes cross-referenced with VirusTotal, with a threshold-based labeling criterion (≥1 detection = malicious, 0 detections = benign)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from AndroZoo, a standard public APK repository."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the overall pipeline stages are described (AndroZoo → VirusTotal labeling → static analysis → description generation → classification), key details are missing: how the specific 10,000 benign and 8,000 malicious samples were selected from AndroZoo's millions of APKs, what time period the APKs cover, and what filtering criteria were applied."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed on the first page: Cochin University of Science and Technology, University of Milan, and University of Pavia. No commercial product affiliations are relevant."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed, independence of funder from outcome cannot be determined."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's zero-shot capability on an existing benchmark. It fine-tunes BERT variants on a custom dataset of LLM-generated descriptions. The LLMs are used as feature-engineering tools, not evaluated on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above—the paper fine-tunes classifiers on a custom dataset, not evaluating pre-trained model knowledge on an established benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above—no pre-trained model benchmark evaluation is being conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported. The paper even notes AppPoet's 5-second inference limitation (Section 2) but does not report its own system's cost or latency."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, API costs, or hardware specifications are provided. The system uses multiple LLM API calls (Gemini, LLaMA, Mistral) and fine-tunes BERT models, but no computational budget is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or results across multiple seeds. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented as single values without indication of how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The fuzzy matching threshold of 65% and the 70:10:20 split appear chosen without justification, and no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares multiple configurations (6 in Table 8) but does not explain how model selection was performed or whether validation performance guided the final configuration choice."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 6 model-classifier combinations (Table 8) plus additional comparisons in Tables 5-7, but no correction for multiple comparisons is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All baselines (Gemini Fusion, Llama Fusion, Mistral Fusion) are the authors' own implementations. No acknowledgment of self-evaluation bias, and no independent evaluation or external baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "AgenticRAG involves retrieval (FAISS + BM25), cache lookups, and LLM calls, while Gemini Fusion uses multiple LLM calls. These likely have very different compute costs, but no comparison at matched compute budgets is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether VirusTotal-based binary labeling (≥1 detection = malicious) actually measures malware detection capability. A single AV engine flagging an APK may be a false positive, and VirusTotal thresholds are known to be noisy."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "AgenticRAG (retrieval + cache + LLM generation) vs Gemini Fusion (multi-LLM fusion) differ substantially in scaffolding. The paper attributes performance differences to the approach rather than the scaffolding confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of the time period of the APK dataset or whether temporal ordering matters. The LLMs generating descriptions may have seen information about these APKs during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLM-generated descriptions inadvertently encode label information or whether the prompts leak classification-relevant information beyond the static features."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test APKs may share structural similarities (same developers, same malware families, near-duplicate apps). The 70:10:20 split appears random without stratification by app family."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "claims": [
     {
       "claim": "AgenticRAG-generated descriptions achieve 92.89% accuracy and 92.86% F1-score for malware classification using CySecBERT, outperforming Gemini Fusion (91.36% accuracy, 91.25% F1-score).",
       "evidence": "Tables 5 and 7 (Section 4.3) show direct comparison. AgenticRAG achieves higher recall (96.69% vs 90.50%) and F1-score (92.86% vs 91.25%), though precision is lower (88.40% vs 90.07%).",
       "supported": "weak"
     },
     {
       "claim": "CySecBERT outperforms SecBERT for malware classification due to its specialized cybersecurity pre-training.",
       "evidence": "Tables 5-6 show CySecBERT achieves higher recall (96.69% vs 93.00% for AgenticRAG; 90.50% vs 89.25% for Gemini Fusion) and higher F1-score in both settings. However, SecBERT achieves higher accuracy (93.31% vs 92.89%) in the AgenticRAG setting.",
       "supported": "weak"
     },
     {
       "claim": "Gemini 2.0 Flash Lite is the best fusion model, outperforming Llama2 and Mistral across all evaluation metrics.",
       "evidence": "Table 8 shows Gemini fusion at 91.36% accuracy vs Llama at 87.97% and Mistral at 88.89% when classified with CySecBERT. Consistent pattern with SecBERT.",
       "supported": "moderate"
     },
     {
       "claim": "The proposed system outperforms state-of-the-art baselines in detection accuracy.",
       "evidence": "Section 1 contribution list claims this, but no external state-of-the-art methods are tested. The paper cites AppPoet at 99.3% accuracy (Section 2) which substantially exceeds their 92.89%.",
       "supported": "unsupported"
     },
     {
       "claim": "RAG integration mitigates LLM hallucinations in description generation.",
       "evidence": "The abstract states RAG enables 'the LLM to ground its output in relevant context' but no quantitative comparison of hallucination rates with vs without RAG is provided.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes AgenticRAG for Android malware detection, combining FAISS+BM25 retrieval with Gemini 2.0 Flash Lite to generate functional descriptions of APKs from static features. These descriptions are classified by fine-tuned CySecBERT, achieving 92.89% accuracy on an 18,000-sample dataset from AndroZoo. AgenticRAG marginally outperforms a Gemini Fusion alternative (91.36%), and Gemini outperforms Llama2 and Mistral as fusion models. However, no external state-of-the-art baselines are tested, no statistical significance tests are performed, and the claimed advantages lack controlled experimental support.",
   "red_flags": [
     {
       "flag": "Unsupported SOTA claims",
       "detail": "The paper claims to 'outperform state-of-the-art baselines' (Section 1) but tests no external baselines. The cited AppPoet achieves 99.3% accuracy (Section 2), far exceeding the proposed system's 92.89%. All comparisons are between the authors' own system variants."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Performance differences between systems (e.g., 92.89% vs 91.36%) are small and presented without any significance tests, error bars, or multiple-run statistics. These differences could easily be within random variation."
     },
     {
       "flag": "Questionable ground truth labeling",
       "detail": "Applications with ≥1 VirusTotal detection (out of 60+ engines) are labeled malicious. A single antivirus flagging is extremely noisy—false positive rates for individual AV engines are well-documented. This is not discussed."
     },
     {
       "flag": "No error analysis or failure cases",
       "detail": "The paper reports only aggregate metrics without examining what types of malware are missed, what benign apps are misclassified, or why. The 7.11% error rate is uncharacterized."
     },
     {
       "flag": "Overclaiming in abstract and conclusion",
       "detail": "The abstract claims 'improving detection accuracy over conventional feature-based methods' without testing any conventional methods. The conclusion extrapolates to 'network traffic analysis, vulnerability assessment, and threat intelligence' from a single Android classification experiment."
     },
     {
       "flag": "Single-run results",
       "detail": "All results appear to be from single experimental runs with no variance reporting. BERT fine-tuning is sensitive to random initialization and data ordering, making single-run comparisons unreliable."
     }
   ],
   "cited_papers": [
     {
       "title": "AppPoet: Large Language Model based Android Malware Detection via Multi-view Prompt Engineering",
       "authors": ["Wenxiang Zhao", "Juntao Wu", "Zhaoyi Meng"],
       "year": 2025,
       "relevance": "Directly relevant as LLM-based malware detection achieving 99.3% accuracy via multi-view prompt engineering, the key prior work in this space."
     },
     {
       "title": "CySecBERT: A Domain-Adapted Language Model for the Cybersecurity Domain",
       "authors": ["Markus Bayer", "Philipp Kuehn", "Ramin Shanehsaz", "Christian Reuter"],
       "year": 2024,
       "relevance": "Domain-adapted BERT model for cybersecurity used as the primary classifier; demonstrates effectiveness of domain-specific fine-tuning for security NLP tasks."
     },
     {
       "title": "SecBERT: Privacy-Preserving Pre-Training Based Neural Network Inference System",
       "authors": ["Hai Huang", "Yongjian Wang"],
       "year": 2024,
       "relevance": "Security-oriented BERT variant used as comparative classifier in the malware detection pipeline."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Documents position-dependent bias and verbosity effects in LLM-based evaluation, relevant to understanding LLM limitations in automated assessment."
     },
     {
       "title": "Summary of ChatGPT-Related Research and Perspective Towards the Future of Large Language Models",
       "authors": ["Yiheng Liu"],
       "year": 2023,
       "relevance": "Overview of LLM research trends including how LLMs are transforming various research approaches."
     },
     {
       "title": "Agentic Information Retrieval",
       "authors": ["Weinan Zhang", "Junwei Liao", "Ning Li"],
       "year": 2025,
       "relevance": "Foundational work on combining RAG with agentic behavior, the core architectural concept used in this paper's AgenticRAG system."
     },
     {
       "title": "Yes, Machine Learning Can Be More Secure! A Case Study on Android Malware Detection",
       "authors": ["Ambra Demontis", "Marco Melis", "Battista Biggio"],
       "year": 2017,
       "relevance": "Demonstrates ML-based android malware detection with adversarial robustness considerations, relevant to understanding evasion resilience."
     },
     {
       "title": "Drebin: Effective and Explainable Detection of Android Malware in Your Pocket",
       "authors": ["Daniel Arp", "Michael Spreitzenbarth", "Malte Hubner"],
       "year": 2014,
       "relevance": "Seminal work on explainable Android malware detection using static features, foundational to the feature-based approach used in this paper."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Android malware detection is practically relevant and the RAG-based approach could be applied by security practitioners, though no code or tool is released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected result that RAG+LLM can improve classification; no conventional wisdom challenged."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses Android security but does not reveal novel attack vectors or raise new safety concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no critique of existing systems or companies."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or installable tool is released."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From relatively unknown university labs; uses Gemini but is not from Google."
     }
   }
 }