ai-research-survey

scan.json (21953B)

---

 {
   "paper": {
     "title": "Multi-Agent Collaborative Fuzzing with Continuous Reflection for Smart Contracts Vulnerability Detection",
     "authors": ["Jie Chen", "Liangmin Wang"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.12164",
     "doi": "10.48550/arXiv.2511.12164"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SmartFuzz, an LLM-driven multi-agent collaborative fuzzer for smart contracts, detects 5.8%-74.7% more vulnerabilities than existing tools within 30 minutes. The continuous reflection process is critical, with a 90.3% performance drop when disabled. Code-specialized LLMs (CodeGemma, CodeQwen, CodeLlama) outperform general models by 10.4%-15.6%. On real-world contracts, SmartFuzz detects 97.2% of true vulnerabilities with only 3 false negatives across 108 contracts.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The source code of the SmartFuzz is available at' but the URL appears to be missing/redacted from the paper text. No working repository link is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "D1 is derived from a labeled dataset [22] and D2 from MuFuzz [15], both publicly available. The 34 DApp projects are collected from GitHub and Etherscan with sources listed in Table 3."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Intel Xeon E5-2678 v3, 128GB RAM, four 2080Ti GPUs) and Ollama and CrewAI are mentioned, but no requirements.txt, library versions, or dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the system architecture but lacks a README or commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2 and 3 report only point estimates (TP, FN counts). No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SmartFuzz outperforms baselines but provides no statistical significance tests. Comparisons are based solely on raw count differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '74.7% more vulnerabilities than Mythril' with the calculation shown (150-35)/154, and '80% reduction in false negatives' ((5-1)/5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 85 contracts in D1 or 108 in D2 were selected beyond tool compatibility. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. Results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: Mythril, SmarTest, Smartian, ILF, RLF for RQ1; Oyente, sFuzz, ConFuzzius, MuFuzz for RQ3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent tools like MuFuzz (2024), RLF (2022), and ConFuzzius (2021), which represent state-of-the-art in smart contract fuzzing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 ablates the continuous reflection process (SmartFuzzwor with max_reflection_round=0) and tests multiple LLM engines (CodeLlama, LLaMA3, CodeQwen, CodeGemma)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports true positives (TP), false negatives (FN), timeout/error cases (TE), and detection time, across multiple vulnerability categories."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "For DApp projects, the authors state 'we implement SmartFuzz to analyze more than 100 DApps in total and then conduct a manual audit for each project.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "2-fold cross-validation is used for D1. For RQ3, D2 contracts are 'treated as unseen contracts' separate from the 10 D1 examples used for learning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks down by EL and SC vulnerability types. Table 3 provides breakdown across 6 vulnerability types (BD, UE, UD, EF, RE, TO)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The DApp projects table shows failure counts (#Fail column) and the paper notes 'an average of 12.36% of the analyzed contract files failed, with most failures resulting from exceeding the time limit.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that SmartFuzzlm (LLaMA3) performs 10.4-15.6% worse than code-specialized models, and that hallucinations increase beyond 5 reflection rounds for some models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '5.8%-74.7% more vulnerabilities within 30 minutes' and 'reduces false negatives by up to 80%' are supported by Tables 2 and 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (SmartFuzzwor) provides controlled single-variable manipulation to justify the causal claim that the reflection process drives performance. The 90.3% drop with reflection disabled supports this."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Smart Contracts Vulnerability Detection' generally, but results are limited to Ethereum/Solidity contracts with specific vulnerability types. No discussion of whether results extend to other blockchain platforms."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for performance gains. Could the improvement come from simply using LLMs rather than the specific multi-agent architecture? No confound analysis."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures true positive vulnerability counts against labeled ground truth, which directly matches the claimed outcome of vulnerability detection. No proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'CodeGemma', 'CodeLlama', 'LLaMA3', 'CodeQwen', 'DeepSeek-R1' with rough parameter sizes (7b, 8b, 16b) but no specific version strings or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes agent actions and roles but does not provide the actual prompt text used. Actions are described in natural language (e.g., 'findFuncs', 'pickVulFuncs') without showing the prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Max reflection rounds (10) and timeout (30 minutes) are stated, but no LLM hyperparameters (temperature, top-p, max tokens) are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent architecture is described in detail: 6 agents (TxSeqDrafter, TxSeqRefiner, FunChecker, ArgChecker, SNDChecker, AMTChecker), their roles, permission-aware actions (Table 6), the RCC workflow, and feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "D1 filtering is described: 'we select only the 85 vulnerable contracts that can be successfully analyzed by all the evaluated tools.' D2 source and selection from MuFuzz is documented. DApp collection sources are listed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what settings, platforms, or vulnerability types are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (execution logs, per-contract results) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "D1 source is described as derived from labeled dataset [22], D2 from MuFuzz [15], and DApp projects collected from GitHub and Etherscan platforms."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and public repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from dataset selection (85 from D1 filtered by tool compatibility, 108 from D2, 34 DApps from public platforms) through execution and oracle-based verification is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Southeast University and Engineering Research Center of BASAM of Ministry of Education."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a fuzzing tool's ability to generate attack sequences, not a pre-trained model's knowledge of benchmark answers. The LLMs are used as reasoning engines, not evaluated on benchmark knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates a fuzzing tool, not model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the evaluation measures whether generated transaction sequences trigger known vulnerabilities, not whether the model has memorized solutions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Wall-clock time is reported: 30-minute timeout for main experiments, and per-contract analysis times for DApps (ranging from 1m 48s to 5m 7s). Figure 4 shows vulnerability detection over time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified: two Intel Xeon E5-2678 v3 CPUs (24 cores, 48 threads), 128GB RAM, four 2080Ti GPUs. Local Ollama deployment is described."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No reporting of results across multiple random seeds. The 2-fold cross-validation splits the dataset but does not assess seed sensitivity of the LLM-based fuzzing process."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is not stated. It is unclear if results are from single runs or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The max reflection round of 10 and RLF reward of 0.7 appear chosen without documented search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek-R1 is used as the 'default' LLM engine without justification for why it was selected over the other tested models."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons across many tool×vulnerability-type comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their system against baselines without acknowledging potential bias from implementing/configuring the evaluation themselves."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SmartFuzz uses LLM inference (multiple agents, multiple reflection rounds) which is far more compute-intensive than traditional fuzzers, but this compute difference is never discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the labeled vulnerability datasets adequately represent real-world vulnerability detection needs, or whether TP/FN on labeled data measures practical security impact."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SmartFuzz bundles a specific multi-agent scaffold with specific LLMs. When comparing against non-LLM baselines, the scaffold effect is completely confounded with the LLM effect."
       }
     }
   },
   "claims": [
     {
       "claim": "SmartFuzz detects 5.8%-74.7% more vulnerabilities than existing state-of-the-art tools within 30 minutes",
       "evidence": "Table 2 shows SmartFuzz finds 150/154 true vulnerabilities vs Mythril (35), SmarTest (103), Smartian (43), ILF (129), RLF (141). Section 4.2.",
       "supported": "moderate"
     },
     {
       "claim": "SmartFuzz reduces false negatives by up to 80%",
       "evidence": "Table 3 shows BD false negatives reduced from 5 (MuFuzz) to 1 (SmartFuzz), i.e., 80% reduction. Section 4.4.",
       "supported": "moderate"
     },
     {
       "claim": "The continuous reflection process is critical, with 90.3% performance drop when disabled",
       "evidence": "Figure 5 shows SmartFuzzwor (no reflection) detects only 11 vulnerabilities vs 150 with reflection. Section 4.3.",
       "supported": "moderate"
     },
     {
       "claim": "Code-specialized LLMs outperform general models by 10.4%-15.6% in vulnerability detection",
       "evidence": "Figure 6 shows CodeLlama (134/154), CodeQwen (137/154), CodeGemma (129/154) vs LLaMA3 (113/154). Section 4.3.",
       "supported": "moderate"
     },
     {
       "claim": "SmartFuzz detects 97.2% of true vulnerabilities on real-world contracts",
       "evidence": "Table 3 shows 105/108 true positives across 6 vulnerability categories on D2. Section 4.4.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests",
       "detail": "All performance comparisons are based on raw counts with no statistical significance testing, despite the stochastic nature of LLM-based fuzzing."
     },
     {
       "flag": "No variance or multiple runs reported",
       "detail": "LLM-based generation is inherently stochastic, but results appear to be from single runs with no variance reporting. The 2-fold CV does not address run-to-run variability."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no discussion of limitations, threats to validity, or scope boundaries."
     },
     {
       "flag": "Unfair compute comparison",
       "detail": "SmartFuzz uses multiple LLM agents with multiple reflection rounds (substantial GPU compute) while baselines use traditional algorithms. The 30-minute wall-clock comparison masks vastly different compute costs."
     },
     {
       "flag": "Missing code repository",
       "detail": "The paper appears to reference a code repository but the URL is missing/redacted, preventing verification of claims."
     },
     {
       "flag": "Selection bias in D1 dataset",
       "detail": "D1 filters to only 85 contracts 'that can be successfully analyzed by all the evaluated tools,' potentially excluding contracts where SmartFuzz might struggle."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent llm systems fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Directly studies failure modes in multi-agent LLM systems, relevant to understanding agentic AI reliability."
     },
     {
       "title": "FuzzGPT: Large language models are edge-case generators: Crafting unusual programs for fuzzing deep learning libraries",
       "authors": ["Y. Deng", "C. S. Xia", "C. Yang"],
       "year": 2024,
       "relevance": "Uses LLMs for fuzzing deep learning libraries, directly relevant to LLM-driven testing and code generation."
     },
     {
       "title": "Fuzzing javascript interpreters with coverage-guided reinforcement learning for LLM-based mutation",
       "authors": ["J. Eom", "S. Jeong", "T. Kwon"],
       "year": 2024,
       "relevance": "Combines LLMs with reinforcement learning for fuzzing, relevant to LLM-augmented software testing."
     },
     {
       "title": "On the reliability of coverage-based fuzzer benchmarking",
       "authors": ["M. Böhme", "L. Szekeres", "J. Metzman"],
       "year": 2022,
       "relevance": "Meta-research on benchmarking methodology for fuzzers, relevant to evaluation rigor in software testing research."
     },
     {
       "title": "Mufuzz: Sequence-aware mutation and seed mask guidance for blockchain smart contract fuzzing",
       "authors": ["P. Qian", "H. Wu", "Z. Du"],
       "year": 2024,
       "relevance": "State-of-the-art smart contract fuzzer used as primary baseline, relevant to AI-augmented security testing."
     },
     {
       "title": "Smartian: Enhancing smart contract fuzzing with static and dynamic data-flow analyses",
       "authors": ["J. Choi", "D. Kim", "S. Kim"],
       "year": 2021,
       "relevance": "Smart contract fuzzing tool combining static and dynamic analysis, relevant to automated software testing."
     },
     {
       "title": "Effectively generating vulnerable transaction sequences in smart contracts with reinforcement learning-guided fuzzing",
       "authors": ["J. Su", "H. Dai", "L. Zhao"],
       "year": 2022,
       "relevance": "RL-guided fuzzing for smart contracts, directly relevant to AI-driven security testing."
     }
   ]
 }