ai-research-survey

scan-v5.json (26335B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Enhancing Android Malware Detection with Retrieval-Augmented Generation",
     "authors": [
       "S. Saraga",
       "S. AnaghaM.",
       "Dincy R. Arikkat",
       "A. RafidhaRehimanK.",
       "S. Nicolazzo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2506.22750",
     "doi": "10.48550/arXiv.2506.22750"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims improvement 'over conventional feature-based methods' but the experimental section only compares two description generation approaches (AgenticRAG vs Gemini Fusion); no direct comparison to raw feature-based classifiers is performed.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes performance differences to specific AgenticRAG properties (enhanced contextual specificity, semantic coherence) but the comparison is not an ablation that isolates these factors; Gemini Fusion uses different underlying models, confounding the causal attribution.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims the approach addresses 'increasingly complex challenges in security-critical applications' broadly, but results are from a single AndroZoo snapshot with no discussion of temporal drift, malware family diversity, or applicability outside this dataset.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Performance differences between AgenticRAG and Gemini Fusion are attributed to AgenticRAG properties without considering alternatives such as differences in the underlying LLMs used for generation, dataset characteristics, or label noise from the VirusTotal threshold.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses accuracy/F1 on a held-out slice of the same AndroZoo distribution as a proxy for real-world malware detection capability, without distinguishing this from deployment performance against obfuscated, polymorphic, or zero-day variants.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the conclusion frames gaps only as 'future work directions' (adding dynamic analysis, multi-OS coverage) rather than honest acknowledgment of current limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed at all — not the VirusTotal labeling threshold, dataset temporal skew, class imbalance handling, or single-run evaluation without variance estimation.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes no explicit statement about what its results do not show; generalization to other malware families, time periods, or operating systems is left unaddressed.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed on the title page (Cochin University of Science and Technology, University of Milan, University of Pavia).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, making this criterion not assessable.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial disclosure statement in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are adequately defined: 'AgenticRAG' is described in detail in Section 3.3, 'static analysis' is contrasted with dynamic analysis, and BERT variants are introduced with references.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists three contributions: dataset compilation with static analysis, the AgenticRAG-based detection system, and an extensive experimental evaluation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 traces the evolution from signature-based to deep learning approaches and explicitly positions the work relative to AppPoet (a direct predecessor using multi-view prompt engineering for Android malware detection).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository or link is provided; the paper describes the system architecture but makes no mention of a public release.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses a custom slice from AndroZoo but does not release the specific APK list, SHA256 hashes, extracted features, or generated descriptions used in experiments.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements file, Dockerfile, or dependency list is provided; specific library versions for HuggingFace, FAISS, BM25, Androguard, or the training framework are not stated.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided; the paper describes the pipeline conceptually but gives insufficient detail to reproduce experiments without guessing.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are reported as single point estimates (e.g., accuracy 92.89%); no confidence intervals or error bars are provided for any metric.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any of the comparative results; differences between AgenticRAG and Gemini Fusion are presented without testing whether they exceed noise.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute performance differences are reported (e.g., recall 96.69% vs 90.50%), giving the reader enough context to assess practical magnitude even without formal effect-size measures.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of 10,000 benign and 8,000 malicious samples is not justified; no power analysis or discussion of why this size is sufficient is provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Results are from single training runs; no variance across seeds, folds, or repeated runs is reported.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": false,
           "justification": "The only comparison is between two LLM-based description generation approaches; the abstract claims improvement over 'conventional feature-based methods' but no such baseline is included in the experiments.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Gemini 2.0 Flash Lite, LLaMA2, and Mistral are all contemporary models used as the comparative baselines within the paper's scope.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "There is no ablation study isolating individual components (e.g., the RAG retrieval vs. the LLM generation vs. the agentic planning); comparisons are between full systems, not components.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Four metrics are reported: accuracy, precision, recall, and F1-score, across all experimental conditions.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Human evaluation is not relevant for an automated malware classification task where ground truth labels come from VirusTotal.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "The dataset is split 70:10:20 with a held-out test set, and results are reported on the test partition.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": false,
           "justification": "No breakdown by malware family, permission category, or any other subgroup is provided; all results are aggregate binary classification metrics.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Failure cases are not discussed; confusion matrices are shown but no qualitative analysis of misclassified samples is provided.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not report any negative results; the brief note that CySecBERT didn't beat SecBERT on every metric is presented as confirmation of CySecBERT's selection, not as a substantive negative finding.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Gemini 2.0 Flash Lite is named, but LLaMA2 and Mistral versions are not specified (no model size, quantization level, or snapshot date); CySecBERT and SecBERT have HuggingFace links but no version pinning.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full prompts for AgenticRAG (Table 2), LLaMA/Mistral generation (Table 3), and Gemini fusion (Table 4) are provided with template placeholders whose fill values are described.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "No training hyperparameters are reported (learning rate, batch size, number of epochs, optimizer); only that early stopping on validation loss and class weighting are used.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The AgenticRAG pipeline is described in substantial detail: feature normalization, FAISS+BM25 ensemble retrieval, fuzzy matching with Levenshtein distance, fallback LLM querying, and cache memory are all explained.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.4 documents NLP preprocessing steps: text cleaning, lowercasing, stopword removal (with examples), and Porter stemming applied before classification.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw data (the specific APK subset, extracted features, or generated descriptions) is not released; only the source repository (AndroZoo) is referenced.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 describes the collection protocol: AndroZoo as source, SHA256 cross-referencing, VirusTotal API for labeling, and the threshold criterion (≥1 engine detection = malicious).",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; samples are drawn from a public APK repository.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "The pipeline from AndroZoo download to VirusTotal labeling to Androguard feature extraction is described, but the date range of APKs, deduplication procedure, and version-specific filtering are not documented.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The training data cutoff of Gemini 2.0 Flash Lite (used for description generation) is not stated; CySecBERT and SecBERT pre-training cutoffs are also not mentioned.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the AndroZoo APKs used for evaluation may have been present in pre-training corpora for Gemini 2.0 Flash Lite or the BERT variants.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The dataset is constructed from AndroZoo without verifying whether those APKs or their metadata appear in the training data of any of the LLMs used.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper notes in related work that prior LLM-based approaches exceed 5 seconds per application, but does not report inference latency or cost for the proposed AgenticRAG system itself.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No GPU hours, API call counts, or total computational budget for training or evaluation are reported.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "AgenticRAG achieves 92.89% accuracy and 96.69% recall, outperforming Gemini Fusion (91.36% accuracy, 90.50% recall) on malware classification.",
       "evidence": "Table 7 and Table 5 show direct comparisons of the two description generation approaches using CySecBERT classifier on the same held-out test set.",
       "supported": "moderate"
     },
     {
       "claim": "CySecBERT outperforms SecBERT for malware detection tasks due to its cybersecurity-domain pre-training.",
       "evidence": "Tables 5 and 6 show CySecBERT with higher recall and F1 than SecBERT in both description settings, but the margin is small and no significance test is run.",
       "supported": "weak"
     },
     {
       "claim": "Gemini 2.0 Flash Lite is the best fusion model, outperforming LLaMA2 and Mistral across all metrics.",
       "evidence": "Table 8 shows Gemini fusion achieving 91.36% accuracy vs 87.97% (LLaMA) and 88.89% (Mistral) with CySecBERT; result is consistent but based on a single run without variance.",
       "supported": "moderate"
     },
     {
       "claim": "The proposed approach improves malware detection accuracy over conventional feature-based methods.",
       "evidence": "This claim is made in the abstract and conclusion but no direct comparison against feature-based baselines (e.g., Drebin, DroidEcho) is performed in the experimental section.",
       "supported": "unsupported"
     },
     {
       "claim": "RAG mitigates LLM hallucinations in generating application functional descriptions.",
       "evidence": "This is stated as a design motivation but no hallucination rate or factual accuracy metric is measured; the claim is qualitative and unverified.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes an Android malware detection pipeline that uses AgenticRAG to convert static APK features into natural language descriptions, which are then classified by fine-tuned BERT variants. AgenticRAG-generated descriptions yield 92.89% accuracy and 96.69% recall versus 91.36% and 90.50% for a Gemini Fusion alternative on 18,000 APKs from AndroZoo. Gemini 2.0 Flash Lite outperforms LLaMA2 and Mistral as a fusion model. However, no direct comparison to non-LLM baselines is performed, no statistical significance is tested, and the claimed improvement over 'conventional feature-based methods' is not empirically validated.",
   "red_flags": [
     {
       "flag": "Missing baseline comparison",
       "detail": "Abstract and conclusion claim improvement over 'conventional feature-based methods' but no such baseline appears in the experimental section; the only comparison is between two LLM-based approaches."
     },
     {
       "flag": "No statistical tests or confidence intervals",
       "detail": "All comparative claims are based on single-run point estimates; differences of 1-5pp between methods are presented without significance testing, making it impossible to assess whether differences exceed noise."
     },
     {
       "flag": "No code or data release",
       "detail": "The system is fully proprietary; no code, feature extraction scripts, APK lists, or generated descriptions are released, making reproduction impossible."
     },
     {
       "flag": "No limitations section",
       "detail": "There is no dedicated limitations or threats-to-validity section; gaps are framed exclusively as future work, not as limitations of the current results."
     },
     {
       "flag": "VirusTotal labeling threshold noise",
       "detail": "Any single antivirus engine flagging an APK labels it malicious; this one-engine threshold is conservative but known to generate false positives and is not discussed as a threat to ground-truth validity."
     },
     {
       "flag": "Dataset temporality undisclosed",
       "detail": "The time range and collection date of the AndroZoo APKs are not reported, making it impossible to assess dataset freshness or whether train/test samples share malware family distributions."
     },
     {
       "flag": "Stopword removal and stemming applied to BERT input",
       "detail": "Section 3.4 applies stopword removal and Porter stemming before feeding text to BERT — preprocessing steps that are generally harmful for transformer models and may have suppressed performance."
     }
   ],
   "cited_papers": [
     {
       "title": "AppPoet: Large Language Model Based Android Malware Detection via Multi-View Prompt Engineering",
       "relevance": "Direct predecessor using LLMs for Android malware detection; the paper explicitly positions itself relative to AppPoet's multi-view prompt engineering approach."
     },
     {
       "title": "Drebin: Effective and Explainable Detection of Android Malware in Your Pocket",
       "relevance": "Classic feature-based Android malware detection baseline using static permissions and intents; foundational work in the area."
     },
     {
       "title": "HinDroid: An Intelligent Android Malware Detection System Based on Structured Heterogeneous Information Network",
       "relevance": "GNN-based approach achieving 98.3% detection accuracy; represents the state-of-the-art deep learning baseline discussed in related work."
     },
     {
       "title": "CySecBERT: A Domain-Adapted Language Model for the Cybersecurity Domain",
       "relevance": "The primary classification model used; cybersecurity-domain BERT variant central to the paper's methodology."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "relevance": "Used to discuss position bias and verbosity effects in LLM-based systems; motivates design choices in evaluation methodology."
     },
     {
       "title": "DroidScope: Seamlessly Reconstructing the OS and Dalvik Semantic Views for Dynamic Android Malware Analysis",
       "relevance": "Represents the dynamic analysis line of work that the paper's static analysis approach is contrasted against."
     },
     {
       "title": "FlowDroid: Precise Context, Flow, Field, Object-Sensitive and Lifecycle-Aware Taint Analysis for Android Apps",
       "relevance": "Foundational static taint analysis work for Android security context."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Android malware detection is a real and growing problem affecting billions of devices; the RAG-based approach is potentially deployable."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Using RAG to generate textual APK descriptions before classification is a novel framing but builds predictably on existing LLM-for-security trends."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Android malware threatening user privacy and financial security is a genuine concern with clear user impact."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward system paper."
     },
     "demo_ability": {
       "score": 1,
       "justification": "AndroZoo is publicly accessible so the dataset can be obtained, but no code is released making replication difficult."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from Cochin University of Science and Technology and Italian universities; no famous lab or industrial affiliation."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44675438",
         "title": "A Photonic SRAM with Embedded XOR Logic for Ultra-Fast In-Memory Computing",
         "points": 57,
         "comments": 16,
         "url": "https://news.ycombinator.com/item?id=44675438"
       },
       {
         "hn_id": "43131809",
         "title": "Cache Is King: Smart Page Eviction with eBPF",
         "points": 8,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43131809"
       },
       {
         "hn_id": "43142367",
         "title": "Cache Is King: Smart Page Eviction with eBPF",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43142367"
       },
       {
         "hn_id": "42984804",
         "title": "Cache Is King: Smart Page Eviction with eBPF",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42984804"
       },
       {
         "hn_id": "45781713",
         "title": "Consequences of Undecidability in Physics on the Theory of Everything",
         "points": 1,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=45781713"
       },
       {
         "hn_id": "44291959",
         "title": "Improving Brain-to-Image Reconstruction via Fine-Grained Text Bridging",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44291959"
       }
     ],
     "top_points": 57,
     "total_points": 74,
     "total_comments": 18
   }
 }