ai-research-survey

scan-v5.json (26496B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Exploring adversarial robustness of JPEG AI: methodology, comparison and new methods",
     "authors": [
       "Egor Kovalev",
       "Georgii Bychkov",
       "Khaled Abud",
       "A. Gushchin",
       "A. Chistyakova",
       "Sergey Lavrushkin",
       "Dmitriy Vatolin",
       "Anastasia Antsiferova"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2411.11795",
     "doi": "10.48550/arXiv.2411.11795"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (first large-scale JPEG AI robustness evaluation, comparison across 10 codecs, defense strategies) are all demonstrated in Results sections 5.1–5.7.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Paper makes comparative claims ('JPEG AI is more robust than Cheng2020') but doesn't justify causation—no ablation studies isolating architectural features responsible for robustness differences.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Results presented across 4 datasets but scope not explicitly bounded; paper doesn't discuss whether findings generalize to out-of-distribution images or different compression ratios.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Paper explains that adversarial noise alters rate-distortion tradeoff but doesn't discuss why HOP variants are less robust than BOP or propose alternative mechanistic explanations for codec differences.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Paper clearly distinguishes measurement (∆PSNR, ∆VMAF quality drops) from claim (robustness)—the delta-metrics directly operationalize the robustness construct.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Conclusion mentions that 'assessing attack success in NICs remains challenging' but does not systematically discuss scope boundaries or threats to validity.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats discussed (e.g., whether white-box attacks overestimate real-world risk, whether 4 attack runs are sufficient, whether standard datasets represent production image distributions).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Paper focuses on white-box attacks (justified by 'compression is purification') and 4 standard datasets, but doesn't explicitly state what the results do NOT show (e.g., black-box robustness, defenses against adaptive attacks).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section visible in paper. Authors are from MSU, ISP RAS, and Innopolis but no funding source stated.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list institutional affiliations (MSU, ISP RAS, Innopolis University) with email addresses.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "NA—no funding disclosed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosures statement present.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Neural image compression (Section 3: analysis transform, quantization, entropy coding, synthesis transform), adversarial attack (Eq. 2: perturbation δ constrained by ε), and white-box attack motivation are precisely defined.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions stated: (1) extended methodology with 4 quality metrics; (2) first large-scale JPEG AI evaluation on 10 codecs × 6 attacks; (3) defense evaluation. Clearly positioned as methodology + empirical study.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 reviews neural image compression evolution, JPEG AI standardization, and prior adversarial robustness work (Kang et al., Chen & Ma). Paper positions itself as first large-scale JPEG AI robustness study.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "Abstract states 'code are publicly available online (link is hidden for a blind review)'—promise made but URL withheld, so reproducibility cannot be verified at submission.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All four datasets are publicly standard (KODAK Photo CD, CITYSCAPES, NIPS 2017 Adversarial Learning, BSDS) without custom modifications.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Section 4.6 lists hardware (120 × Tesla A100, Intel Xeon) and mentions 'source code of JPEG AI' but no requirements.txt, Docker, or Python version specs provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Methodology describes attacks, datasets, and metrics but lacks step-by-step runnable instructions. Attack parameters ('learning rate, number of iterations, perturbation bound') mentioned but not instantiated.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Figures 2–9 report point estimates (mean ∆VMAF, average BSQ-rate). Section 4.6 notes 'applied each attack four times...and averaged' but no CI or error bars shown.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No p-values, t-tests, or statistical significance tests reported. Results presented as descriptive comparisons across methods.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "∆PSNR, ∆MSE, ∆MS-SSIM, ∆VMAF, BSQ-rate, and artifact metrics (Color, Texture) all quantify effect magnitude with baseline context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "Four attack runs per codec-attack pair mentioned, but no power analysis or justification that n=4 is sufficient to estimate robust delta-metrics.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Paper averages 4 attack runs but reports only means; no standard deviations, confidence intervals, or per-image variance across the three 4-dataset split.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Compares JPEG AI (3 versions) against 10 other neural compression methods: Balle 2018, CDC, Cheng2020, ELIC, EVC, HiFiC, Li-TCM, mbt2018 variants, QRES-VAE.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Models range 2018–2024; most comparisons (Cheng2020-attn, EVC, HiFiC, ELIC) are from 2020–2022, contemporary to JPEG AI 4.1–6.1 (2023–2024).",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "Paper compares different attack loss functions and defenses but does not ablate individual architectural components (e.g., attention, context modeling) within JPEG AI to isolate robustness drivers.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Four quality metrics (PSNR, MSE, MS-SSIM, VMAF), two artifact detectors (Color, Texture), BPP, transferability metric (∆̂VMAF), and defense comparison metrics.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "NA—paper evaluates automatic image quality metrics, not human perceptual judgments. Human evaluation not required for compression robustness assessment.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Four separate standard datasets (KODAK, CITYSCAPES, NIPS, BSDS) used; no data leakage across train/test splits within the benchmarks.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results broken down by codec (10 types), attack method (6 + random), loss function (6 targets), and dataset implicitly in aggregation ('Averaged for all tested datasets').",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.4 analyzes artifact types (color vs. texture distortions) and shows CDC codec 'may be less robust by design.' Section 5.6 shows some defenses only partially effective.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Figure 8 shows Geometric self-ensemble and DiffPure defenses offer minimal protection; reconstruction-based losses shown less effective than FTDA default; some attacks fail on JPEG AI.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "JPEG AI versions named (4.1, 5.1, 6.1) with HOP/BOP variants. Other codecs identified by paper + year (Cheng2020, ELIC 2022, etc.) per Table 2.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "NA—not an LLM evaluation study.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Section 4.6 states 'varied attack parameters (learning rate, number of iterations, perturbation bound)' but specific values (e.g., lr=0.01, iterations=100, ε=8/255) not listed in text.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "NA—no agentic scaffolding; pure adversarial attack evaluation.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "Standard datasets used without custom preprocessing. No mention of resizing, normalization, or other data pipeline steps before attack/defense evaluation.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All four datasets are publicly available standard benchmarks; no custom data collection.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.4 describes the four benchmark sources (KODAK Photo CD, CITYSCAPES, NIPS 2017, BSDS) with resolution and purpose; these are well-established datasets.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human participants.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "Pipeline described at high level (compress image, apply attack, measure quality drop) but implementation details (quantization settings, compression ratio choices) not fully documented.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "NA—paper evaluates pre-trained models, does not train new ones on benchmarks.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "NA—same as above.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "NA—standard compression benchmarks used; models pre-trained before paper submission.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human subjects.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human subjects.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human subjects.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human subjects.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human subjects.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human subjects.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "NA—no human subjects.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference time, latency, or memory footprint reported for attacks or defenses. Only hardware (120 A100 GPUs) mentioned but not total compute hours or cost.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Section 4.6 lists hardware resources but no total GPU-hours, wall-clock time, or budget breakdown across 10 codecs × 6 attacks × 4 datasets.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "JPEG AI shows relatively high robustness compared to other neural image compression models",
       "evidence": "Figure 3 shows ∆VMAF (quality drop under attack) for all 10 codecs; JPEG AI variants rank in top tier for most attack types.",
       "supported": "strong"
     },
     {
       "claim": "HOP variants of JPEG AI are less robust than BOP variants",
       "evidence": "Figure 3 and Section 5.2 explicitly state 'High-operation point versions of JPEG AI are less robust than base-operation point'; consistent across all attacks.",
       "supported": "strong"
     },
     {
       "claim": "JPEG AI robustness improves with newer versions (6.1 > 5.1 > 4.1)",
       "evidence": "Section 5.2: 'robustness of JPEG AI improved with a newer version (6.1 compared to 5.1)'; Figure 3 shows ordering.",
       "supported": "strong"
     },
     {
       "claim": "Adversarial attacks increase the size of compressed images even without BPP-targeted optimization",
       "evidence": "Figure 4 shows increased bitrate (positive ∆BPP) for attacks not optimizing BPP; Section 5.3 explains via altered rate-distortion tradeoff.",
       "supported": "strong"
     },
     {
       "claim": "Different codecs are vulnerable to different attack types",
       "evidence": "Section 5.2: 'Cheng2020 is subject to I-FGSM and FTDA attacks, which are ineffective against JPEG AI'; codec-specific vulnerability patterns evident in Figure 3.",
       "supported": "strong"
     },
     {
       "claim": "Simple reversible defenses (flip, roll, rotate) can partially mitigate adversarial attacks",
       "evidence": "Figure 8 shows Flip, Random Ensemble, and Random Roll reduce ∆PSNR by 5–20 points on FTDA/I-FGSM attacks.",
       "supported": "moderate"
     },
     {
       "claim": "Adversarial attacks transfer between JPEG AI versions, especially from lower to higher bitrates",
       "evidence": "Section 5.5 and Figure 7 show high transferability between JPEG AI versions, with stronger transfer from lower bitrates (b0002) to higher ones (b05).",
       "supported": "strong"
     },
     {
       "claim": "Color artifacts are a major driver of quality degradation under attack, more so than texture artifacts",
       "evidence": "Figure 5 shows Color metric correlates r=0.72 with ∆PSNR while Texture metric shows minimal correlation; Section 5.4 confirms artifacts on reconstructed images show stronger color distortions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational",
     "case-study"
   ],
   "key_findings": "This empirical evaluation demonstrates that JPEG AI achieves >50% bitrate savings vs. legacy codecs while maintaining competitively high adversarial robustness. The paper systematically compares 10 neural compression models across 6 white-box attacks and multiple quality metrics. Key findings: (1) JPEG AI 6.1 is more robust than earlier versions, with BOP variants outperforming HOP; (2) different codecs show codec-specific vulnerability patterns, suggesting architecture influences robustness; (3) simple reversible defenses (spatial transforms) offer partial mitigation; (4) attacks transfer effectively between JPEG AI versions, raising standardization concerns; (5) color artifacts dominate quality degradation under attack, not texture.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All results reported as point estimates; 4 attack runs averaged without confidence intervals or variance reporting, making it unclear if differences are robust."
     },
     {
       "flag": "Missing limitations section",
       "detail": "No dedicated discussion of scope boundaries, threat to validity, or generalization limits. Conclusion mentions challenges but does not systematically address what the study does NOT show."
     },
     {
       "flag": "Funding source not disclosed",
       "detail": "No funding acknowledgments or conflicts of interest statement despite institutional affiliations with Russian research centers."
     },
     {
       "flag": "Code reproducibility delayed",
       "detail": "Link to code hidden for blind review; reproducibility cannot be verified at submission time."
     },
     {
       "flag": "Incomplete hyperparameter specification",
       "detail": "Attack learning rates, iteration counts, and perturbation bounds mentioned as varied but specific values not provided in text."
     },
     {
       "flag": "No mechanistic explanation for robustness differences",
       "detail": "Paper documents that HOP is less robust than BOP and CDC is weakest, but does not isolate architectural features (attention, context modeling) responsible for these differences."
     },
     {
       "flag": "Limited defense evaluation",
       "detail": "Evaluated defenses are reversible image transforms and one diffusion-based method; no adversarially-trained defenses or certified robustness approaches explored."
     },
     {
       "flag": "Environment specs incomplete",
       "detail": "Hardware listed but no Python version, JPEG AI version numbers for training, or Docker/conda environment file provided for reproduction."
     }
   ],
   "cited_papers": [
     {
       "title": "End-to-end optimized image compression",
       "relevance": "Foundational neural image compression work (Ballé et al. 2016); baseline codec architecture."
     },
     {
       "title": "Variational image compression with a scale hyperprior",
       "relevance": "Introduces hyperprior entropy model used in multiple evaluated codecs; key compression technique."
     },
     {
       "title": "Toward robust neural image compression: Adversarial attack and model finetuning",
       "relevance": "Prior work on NIC adversarial robustness (Chen & Ma 2023); defines ∆PSNR metric extended in this paper."
     },
     {
       "title": "Manipulation attacks on learned image compression",
       "relevance": "Early adversarial attack on neural compression (Liu et al. 2023); establishes attack methodology."
     },
     {
       "title": "The jpeg ai standard: Providing efficient human and machine visual data consumption",
       "relevance": "Official JPEG AI standardization paper (Ascenso et al. 2023); primary subject of evaluation."
     },
     {
       "title": "Towards deep learning models resistant to adversarial attacks",
       "relevance": "PGD attack introduction (Madry et al. 2018); foundational adversarial robustness methodology."
     },
     {
       "title": "Adversarial examples in the physical world",
       "relevance": "I-FGSM attack (Kurakin et al. 2018); one of six attacks evaluated."
     },
     {
       "title": "Diffusion models for adversarial purification",
       "relevance": "DiffPure defense (Nie et al. 2022); defense baseline used in Section 5.6."
     },
     {
       "title": "Comparing the robustness of modern no-reference image- and video-quality metrics to adversarial attacks",
       "relevance": "Related work on adversarial robustness of quality metrics themselves (Antsiferova et al. 2024); metric validation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "JPEG AI is a real ISO/IEC standard for consumer devices, giving practical stakes; however, adversarial attacks on image compression codecs are low-probability real-world threats vs. other security concerns."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Results align with expected findings: newer codec versions are more robust, different architectures have different robustness profiles. No surprising reversals or counterintuitive claims."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Paper addresses adversarial robustness but in a niche domain (image compression security). No broader AI safety or alignment implications discussed."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Technical benchmarking paper with no controversy, disputes, or conflicting stakeholders. Straightforward empirical evaluation."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Could produce visual demos of adversarial attacks and defenses on JPEG AI outputs; code promised but currently unavailable. Requires GPU and specialized setup."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "JPEG AI is an official standard with real-world deployment; authors from reputable institutions (MSU, ISP RAS). Moderate credibility but niche audience (compression researchers)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "41947355",
         "title": "Universal optimality of Dijkstra via beyond-worst-case heaps",
         "points": 203,
         "comments": 47,
         "url": "https://news.ycombinator.com/item?id=41947355"
       },
       {
         "hn_id": "44742187",
         "title": "Deploying Large Language Models with Retrieval Augmented Generation (2024)",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44742187"
       },
       {
         "hn_id": "42185072",
         "title": "An Internet Voting System Fatally Flawed in Creative New Ways [pdf]",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42185072"
       },
       {
         "hn_id": "39198471",
         "title": "Image Conditioned Inpainting in Latent Diffusion Models for Virtual Try-All",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39198471"
       },
       {
         "hn_id": "39132573",
         "title": "ZkLogin: Privacy-Preserving Blockchain Authentication with Existing Credentials",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39132573"
       }
     ],
     "top_points": 203,
     "total_points": 207,
     "total_comments": 47
   }
 }