scan.json (22453B)
1 { 2 "paper": { 3 "title": "A Survey of Adversarial Examples in Computer Vision: Attack, Defense, and Beyond", 4 "authors": ["Xu Keyizhi", "Lu Yajuan", "Wang Zhongyuan", "Liang Chao"], 5 "year": 2025, 6 "venue": "Wuhan University Journal of Natural Sciences", 7 "doi": "10.1051/wujns/2025301001" 8 }, 9 "scan_version": 3, 10 "active_modules": ["survey_methodology"], 11 "methodology_tags": ["meta-analysis"], 12 "key_findings": "This survey taxonomizes adversarial attacks in computer vision by four standards (targets, knowledge, perturbation structure, bounds) and defenses by five categories (adversarial training, robust network design, input transformation, certified defenses, ensemble defenses). It discusses theoretical explanations for adversarial examples (low-probability, linearity, off-manifold, manifold geometry hypotheses), the accuracy-robustness tradeoff, and benign uses of adversarial techniques such as privacy protection and preventing malicious AI-generated content. Three open problems are identified: scalability of defenses, cross-domain generalization, and understanding transferability.", 13 "checklist": { 14 "artifacts": { 15 "code_released": { 16 "applies": true, 17 "answer": false, 18 "justification": "No code, analysis scripts, or repository link is provided. A survey can release search/analysis code, but this paper does not." 19 }, 20 "data_released": { 21 "applies": true, 22 "answer": false, 23 "justification": "No dataset, search corpus, or structured extraction of reviewed papers is released. The collected paper inventory and comparison tables exist only in the paper text." 24 }, 25 "environment_specified": { 26 "applies": true, 27 "answer": false, 28 "justification": "No environment or dependency specifications are provided. No computational tools or analysis pipelines are described." 29 }, 30 "reproduction_instructions": { 31 "applies": true, 32 "answer": false, 33 "justification": "No instructions are provided for reproducing the survey's paper selection or analysis process." 34 } 35 }, 36 "statistical_methodology": { 37 "confidence_intervals_or_error_bars": { 38 "applies": false, 39 "answer": false, 40 "justification": "This is a survey paper that conducts no experiments and reports no original quantitative results." 41 }, 42 "significance_tests": { 43 "applies": false, 44 "answer": false, 45 "justification": "Survey paper with no statistical comparisons of its own." 46 }, 47 "effect_sizes_reported": { 48 "applies": false, 49 "answer": false, 50 "justification": "Survey paper with no experiments." 51 }, 52 "sample_size_justified": { 53 "applies": false, 54 "answer": false, 55 "justification": "Survey paper with no experimental sample." 56 }, 57 "variance_reported": { 58 "applies": false, 59 "answer": false, 60 "justification": "Survey paper with no experimental runs." 61 } 62 }, 63 "evaluation_design": { 64 "baselines_included": { 65 "applies": true, 66 "answer": true, 67 "justification": "Table 1 explicitly compares this survey against 6 prior surveys (Serban et al. 2020, Machado et al. 2021, Long et al. 2022, Wang et al. 2022, Li et al. 2024, Costa et al. 2024) across multiple dimensions including year, venue, scope, perspective, and highlights." 68 }, 69 "baselines_contemporary": { 70 "applies": true, 71 "answer": true, 72 "justification": "The compared surveys include Li et al. 2024 and Costa et al. 2024, which are contemporary works." 73 }, 74 "ablation_study": { 75 "applies": false, 76 "answer": false, 77 "justification": "Survey paper with no system components to ablate." 78 }, 79 "multiple_metrics": { 80 "applies": false, 81 "answer": false, 82 "justification": "Survey paper with no experiments requiring evaluation metrics." 83 }, 84 "human_evaluation": { 85 "applies": false, 86 "answer": false, 87 "justification": "Survey paper with no system outputs to evaluate." 88 }, 89 "held_out_test_set": { 90 "applies": false, 91 "answer": false, 92 "justification": "Survey paper with no experiments." 93 }, 94 "per_category_breakdown": { 95 "applies": true, 96 "answer": true, 97 "justification": "Tables 2 and 3 provide detailed per-category breakdowns of attack methods (by target, knowledge, perturbation structure, bounds) and defense methods (by category), covering 30+ attack algorithms and 25+ defense methods." 98 }, 99 "failure_cases_discussed": { 100 "applies": true, 101 "answer": true, 102 "justification": "The paper discusses limitations and failure modes of each defense category: adversarial training compromises clean accuracy (§3.1.1), input transformations can degrade clean performance (§3.1.3), certified defenses have conservative bounds and scalability issues (§3.1.4), and ensemble defenses have high computational cost (§3.1.5)." 103 }, 104 "negative_results_reported": { 105 "applies": true, 106 "answer": true, 107 "justification": "The paper reports negative findings: the accuracy-robustness tradeoff (§4.2), that certified defenses guarantee stability only within small perturbation ranges (§3.1.4), and that the off-manifold hypothesis was questioned by Carlini et al. who developed attacks circumventing adversarial detection (§4.1)." 108 } 109 }, 110 "claims_and_evidence": { 111 "abstract_claims_supported": { 112 "applies": true, 113 "answer": true, 114 "justification": "The abstract claims the survey covers adversarial attacks, defenses, and issues beyond attack/defense including theoretical explanations, trade-offs, and benign attacks. These are all substantiated by Sections 2, 3, and 4 respectively." 115 }, 116 "causal_claims_justified": { 117 "applies": false, 118 "answer": false, 119 "justification": "This survey makes no causal claims of its own. It describes and categorizes existing work without claiming causal relationships from its own analysis." 120 }, 121 "generalization_bounded": { 122 "applies": true, 123 "answer": true, 124 "justification": "The paper explicitly bounds its scope to computer vision and states: 'we only present the most classic, representative and illustrative related works' and 'highly ad-hoc designs of attacks or defenses on specific tasks will not be discussed, for their impact is limited to a certain smaller branch of computer vision.'" 125 }, 126 "alternative_explanations_discussed": { 127 "applies": false, 128 "answer": false, 129 "justification": "This is a pure survey/taxonomy paper with no empirical results of its own, so alternative explanations for observed results are not applicable." 130 }, 131 "proxy_outcome_distinction": { 132 "applies": false, 133 "answer": false, 134 "justification": "Survey paper with no measurements of its own." 135 } 136 }, 137 "setup_transparency": { 138 "model_versions_specified": { 139 "applies": false, 140 "answer": false, 141 "justification": "Survey paper that does not use any AI models." 142 }, 143 "prompts_provided": { 144 "applies": false, 145 "answer": false, 146 "justification": "Survey paper that does not use prompting." 147 }, 148 "hyperparameters_reported": { 149 "applies": false, 150 "answer": false, 151 "justification": "Survey paper with no experiments." 152 }, 153 "scaffolding_described": { 154 "applies": false, 155 "answer": false, 156 "justification": "No agentic scaffolding is used in this survey." 157 }, 158 "data_preprocessing_documented": { 159 "applies": true, 160 "answer": false, 161 "justification": "The paper does not document a systematic paper selection pipeline. It states it presents 'the most classic, representative and illustrative related works' but provides no search queries, databases searched, inclusion/exclusion criteria with counts, or screening process." 162 } 163 }, 164 "limitations_and_scope": { 165 "limitations_section_present": { 166 "applies": true, 167 "answer": false, 168 "justification": "There is no dedicated limitations section for the survey itself. Section 5 ('Discussion') discusses open problems in the field of adversarial examples, but these are about the research area, not about the survey's own methodological limitations." 169 }, 170 "threats_to_validity_specific": { 171 "applies": true, 172 "answer": false, 173 "justification": "No threats to the validity of the survey itself are discussed. The paper does not address potential biases in its paper selection, coverage gaps, or limitations of its classification scheme." 174 }, 175 "scope_boundaries_stated": { 176 "applies": true, 177 "answer": true, 178 "justification": "The paper states: 'we only present the most classic, representative and illustrative related works to form a well-structured taxonomy' and 'some related works might be purposefully omitted in this survey. For example, highly ad-hoc designs of attacks or defenses on specific tasks will not be discussed.' The focus is explicitly on computer vision." 179 } 180 }, 181 "data_integrity": { 182 "raw_data_available": { 183 "applies": true, 184 "answer": false, 185 "justification": "No supplementary data, paper lists, or structured extraction files are made available." 186 }, 187 "data_collection_described": { 188 "applies": true, 189 "answer": false, 190 "justification": "The paper does not describe how it identified and collected the papers it reviews. No search strategy, databases, queries, or time period are documented." 191 }, 192 "recruitment_methods_described": { 193 "applies": true, 194 "answer": false, 195 "justification": "For a survey paper, the 'sample' is the set of reviewed papers. The selection method is not described beyond stating they chose 'classic, representative and illustrative' works, with no explanation of how representativeness was determined." 196 }, 197 "data_pipeline_documented": { 198 "applies": true, 199 "answer": false, 200 "justification": "No pipeline from paper collection to the final taxonomy and analysis is documented. The paper does not describe stages of screening or how many papers were initially identified versus finally included." 201 } 202 }, 203 "conflicts_of_interest": { 204 "funding_disclosed": { 205 "applies": true, 206 "answer": true, 207 "justification": "Funding is disclosed: 'Supported by the National Natural Science Foundation of China (U1903214, 62372339, 62371350, 61876135), the Ministry of Education Industry-University Cooperative Education Project, and the Fundamental Research Funds for the Central Universities.'" 208 }, 209 "affiliations_disclosed": { 210 "applies": true, 211 "answer": true, 212 "justification": "All authors are affiliated with Wuhan University, with specific schools and research centers listed (School of Computer Science, NERCMS, School of Cyber Science and Engineering)." 213 }, 214 "funder_independent_of_outcome": { 215 "applies": true, 216 "answer": true, 217 "justification": "The funders are the National Natural Science Foundation of China and the Ministry of Education — government research funding bodies with no financial interest in the survey's conclusions." 218 }, 219 "financial_interests_declared": { 220 "applies": true, 221 "answer": false, 222 "justification": "No competing interests or financial interests statement is present in the paper." 223 } 224 }, 225 "contamination": { 226 "training_cutoff_stated": { 227 "applies": false, 228 "answer": false, 229 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 230 }, 231 "train_test_overlap_discussed": { 232 "applies": false, 233 "answer": false, 234 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 235 }, 236 "benchmark_contamination_addressed": { 237 "applies": false, 238 "answer": false, 239 "justification": "Survey paper that does not evaluate any pre-trained model on a benchmark." 240 } 241 }, 242 "human_studies": { 243 "pre_registered": { 244 "applies": false, 245 "answer": false, 246 "justification": "Survey paper with no human participants." 247 }, 248 "irb_or_ethics_approval": { 249 "applies": false, 250 "answer": false, 251 "justification": "Survey paper with no human participants." 252 }, 253 "demographics_reported": { 254 "applies": false, 255 "answer": false, 256 "justification": "Survey paper with no human participants." 257 }, 258 "inclusion_exclusion_criteria": { 259 "applies": false, 260 "answer": false, 261 "justification": "Survey paper with no human participants." 262 }, 263 "randomization_described": { 264 "applies": false, 265 "answer": false, 266 "justification": "Survey paper with no human participants." 267 }, 268 "blinding_described": { 269 "applies": false, 270 "answer": false, 271 "justification": "Survey paper with no human participants." 272 }, 273 "attrition_reported": { 274 "applies": false, 275 "answer": false, 276 "justification": "Survey paper with no human participants." 277 } 278 }, 279 "cost_and_practicality": { 280 "inference_cost_reported": { 281 "applies": false, 282 "answer": false, 283 "justification": "Survey paper with no computational method of its own." 284 }, 285 "compute_budget_stated": { 286 "applies": false, 287 "answer": false, 288 "justification": "Survey paper with no computational experiments." 289 } 290 }, 291 "survey_methodology": { 292 "prisma_or_structured_protocol": { 293 "applies": true, 294 "answer": false, 295 "justification": "No PRISMA diagram, no structured review protocol, no reproducible search queries, no database listing. The paper states it selects 'the most classic, representative and illustrative related works' without any systematic methodology." 296 }, 297 "quality_assessment_of_sources": { 298 "applies": true, 299 "answer": false, 300 "justification": "The survey does not assess the methodological quality of its source papers. All reviewed attack and defense papers are treated equally regardless of their experimental rigor. Table 1 compares prior surveys but does not assess the quality of primary research." 301 }, 302 "publication_bias_discussed": { 303 "applies": true, 304 "answer": false, 305 "justification": "No discussion of publication bias. The survey does not consider whether its sources skew toward positive results or whether unpublished negative findings exist." 306 } 307 } 308 }, 309 "claims": [ 310 { 311 "claim": "Adversarial attacks can be categorized by 4 standards: adversarial targets (targeted/untargeted), adversarial knowledge (white-box/black-box/gray-box), perturbation structures (noise/spatial/patch/semantic), and adversarial bounds (norm/perceptual/unrestricted).", 312 "evidence": "Section 2.1 presents the four-standard taxonomy with detailed definitions. Table 2 classifies 30+ representative attack algorithms across all four dimensions.", 313 "supported": "moderate" 314 }, 315 { 316 "claim": "Adversarial defenses can be divided into five categories: adversarial training, robust network design, input transformation, certified defenses, and ensemble defenses.", 317 "evidence": "Section 3.1 describes each category in detail. Table 3 lists 25+ representative defense methods organized by these categories.", 318 "supported": "moderate" 319 }, 320 { 321 "claim": "There is an inherent trade-off between accuracy and robustness in adversarial defense.", 322 "evidence": "Section 4.2 discusses this trade-off from three perspectives (generalization trade-off, robust loss functions, model capacity), citing Tsipras et al. as a key reference.", 323 "supported": "moderate" 324 }, 325 { 326 "claim": "Adversarial attacks can be harnessed for beneficial purposes including privacy protection and preventing malicious AI-generated content.", 327 "evidence": "Section 4.3 discusses Hu et al.'s make-up transfer attack for facial privacy and Salman et al.'s adversarial immunization against malicious diffusion-model editing.", 328 "supported": "moderate" 329 }, 330 { 331 "claim": "Existing surveys on adversarial examples lack harmonized classification standards.", 332 "evidence": "Table 1 compares 6 prior surveys and the paper notes 'the taxonomies vary in existing surveys, lacking a harmonized classification standard.'", 333 "supported": "weak" 334 } 335 ], 336 "red_flags": [ 337 { 338 "flag": "No systematic review methodology", 339 "detail": "The paper acknowledges selecting 'the most classic, representative and illustrative related works' but provides no systematic search strategy, no databases queried, no inclusion/exclusion criteria, and no PRISMA flow diagram. Papers appear to be selected ad-hoc." 340 }, 341 { 342 "flag": "No quality assessment of reviewed papers", 343 "detail": "The survey does not assess the methodological quality of the primary research papers it reviews. All papers are presented as equally valid regardless of experimental rigor. This launders the signal-to-noise ratio of the source literature." 344 }, 345 { 346 "flag": "No discussion of publication bias", 347 "detail": "The survey does not consider whether published adversarial attack/defense papers skew toward positive results, potentially missing failed approaches or overstating effectiveness of methods." 348 }, 349 { 350 "flag": "No self-limitations discussed", 351 "detail": "While Section 5 discusses open problems in the field, the paper never examines limitations of its own survey methodology, coverage gaps, or potential biases in paper selection." 352 } 353 ], 354 "cited_papers": [ 355 { 356 "title": "Is BERT really robust? A strong baseline for natural language attack on text classification and entailment", 357 "authors": ["Di Jin", "Zhijing Jin", "Joey Tianyi Zhou", "Peter Schiber"], 358 "year": 2020, 359 "relevance": "Extends adversarial robustness concerns to NLP/language models, directly relevant to LLM safety research." 360 }, 361 { 362 "title": "Black-box generation of adversarial text sequences to evade deep learning classifiers", 363 "authors": ["Ji Gao", "Jack Lanchantin", "Mary Lou Soffa", "Yanjun Qi"], 364 "year": 2018, 365 "relevance": "Black-box adversarial attacks on text classifiers, foundational for understanding LLM adversarial vulnerabilities." 366 }, 367 { 368 "title": "Generating natural language adversarial examples through probability weighted word saliency", 369 "authors": ["Shuhuai Ren", "Yihe Deng", "Kun He", "Wanxiang Che"], 370 "year": 2019, 371 "relevance": "Natural language adversarial example generation, relevant to prompt injection and LLM robustness." 372 }, 373 { 374 "title": "Raising the cost of malicious AI-powered image editing", 375 "authors": ["Hadi Salman", "Alaa Khaddaj", "Guillaume Leclerc"], 376 "year": 2023, 377 "arxiv_id": "2302.06588", 378 "relevance": "Uses adversarial techniques to defend against malicious AI content generation, directly relevant to AI safety." 379 }, 380 { 381 "title": "Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks", 382 "authors": ["Francesco Croce", "Matthias Hein"], 383 "year": 2020, 384 "relevance": "AutoAttack benchmark for reliable robustness evaluation, foundational methodology for evaluating AI model safety." 385 }, 386 { 387 "title": "Adversarial attacks on medical machine learning", 388 "authors": ["Samuel G. Finlayson", "John D. Bowers", "Joichi Ito"], 389 "year": 2019, 390 "relevance": "Demonstrates adversarial attack risks in healthcare AI deployment, key AI safety concern." 391 }, 392 { 393 "title": "Denoising diffusion probabilistic models", 394 "authors": ["Jonathan Ho", "Ajay Jain", "Pieter Abbeel"], 395 "year": 2020, 396 "arxiv_id": "2006.11239", 397 "relevance": "Foundational diffusion model paper; diffusion models are now central to both generative AI capabilities and adversarial purification defenses." 398 }, 399 { 400 "title": "Diffusion models for adversarial purification", 401 "authors": ["Weili Nie", "Brandon Guo", "Yujia Huang"], 402 "year": 2022, 403 "relevance": "Uses diffusion models as a defense mechanism against adversarial attacks, relevant to AI safety and robustness." 404 }, 405 { 406 "title": "Adversarial policies: Attacking deep reinforcement learning", 407 "authors": ["Adam Gleave", "Michael Dennis", "Cody Wild"], 408 "year": 2020, 409 "relevance": "Adversarial attacks on RL agents, directly relevant to agentic AI safety." 410 }, 411 { 412 "title": "Towards evaluating the robustness of neural networks", 413 "authors": ["Nicholas Carlini", "David Wagner"], 414 "year": 2017, 415 "relevance": "C&W attack is a foundational adversarial attack method; paper established rigorous robustness evaluation methodology." 416 } 417 ], 418 "engagement_factors": { 419 "practical_relevance": { 420 "score": 1, 421 "justification": "Provides a useful taxonomy overview but no immediately usable tools or techniques; primarily educational." 422 }, 423 "surprise_contrarian": { 424 "score": 0, 425 "justification": "Confirms existing knowledge and organizes known work; does not challenge conventional wisdom." 426 }, 427 "fear_safety": { 428 "score": 1, 429 "justification": "Discusses adversarial attacks on CV systems including physical-world threats, but these are well-known concerns, not novel." 430 }, 431 "drama_conflict": { 432 "score": 0, 433 "justification": "No controversy or provocative claims; straightforward literature review." 434 }, 435 "demo_ability": { 436 "score": 0, 437 "justification": "No code, demo, or tool released." 438 }, 439 "brand_recognition": { 440 "score": 0, 441 "justification": "From Wuhan University, not a widely recognized AI lab in Western tech media." 442 } 443 } 444 }