ai-research-survey

scan.json (17518B)

---

 {
   "paper": {
     "title": "Are GANs Created Equal? A Large-Scale Study",
     "authors": ["Mario Lucic", "Karol Kurach", "Marcin Michalski", "Olivier Bousquet", "Sylvain Gelly"],
     "year": 2018,
     "venue": "NeurIPS",
     "arxiv_id": "1711.10337"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we open-sourced our experimental setup and model implementations at goo.gl/G8kf5J' (Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (MNIST, Fashion-MNIST, CIFAR10, CelebA) and proposes new synthetic datasets (convex polygons) with the open-sourced code."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is described in the paper. Only mentions NVIDIA P100 GPUs."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released, the paper itself does not contain step-by-step reproduction instructions. Hyperparameter ranges are in the appendix but no explicit reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports mean FID ± standard deviation. Figure 5 shows 95% confidence intervals. Figure 3 shows one standard deviation intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes that 'any statistically significant comparison of the models is unattainable' with limited budget but does not perform formal significance tests between models. Comparisons are based on overlapping distributions."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "FID scores with baselines are reported (e.g., NS GAN 6.8 vs WGAN 6.7 on MNIST, Table 2), giving concrete magnitudes of differences between models."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses computational budget constraints ('approximately 6.85 GPU years') and uses 100 hyperparameter samples in wide search, 50 in narrow search, 50 random seeds for stability, and 5000 bootstrap resamples for confidence intervals."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports standard deviations across 50 random seeds. Figures 3 and 5 show variance via standard deviation intervals and confidence intervals respectively."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven GAN variants are compared (MM GAN, NS GAN, LSGAN, WGAN, WGAN GP, DRAGAN, BEGAN) plus VAE as a non-GAN baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All compared models were state-of-the-art or recent at the time (2017-2018 publications: WGAN GP, DRAGAN, BEGAN)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies hyperparameters (learning rate, batch norm, discriminator iterations, beta1, lambda, clipping) and measures their individual impact on FID (Figures 6-7, scatter plots)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "FID, precision, recall, and F1 score are all used as evaluation metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated samples is performed. All evaluation is automated via FID, precision, recall, and F1."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "FID is computed between generated samples and 10k samples from the test set (Section 6: 'compute the FID between the 10k samples generated by the model and the 10k samples from the test set')."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (MNIST, Fashion-MNIST, CIFAR10, CelebA) in Table 2 and per model across all metrics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses mode collapse failures (asterisks in Table 2 indicating 'significant outlier runs, usually severe mode collapses or training failures'), and notes BEGAN and VAE's 'underwhelming performance' on the polygon task."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The central finding is a negative result: no GAN algorithm consistently outperforms the original NS GAN. The paper also reports that hyperparameter transfer does not always work (e.g., WGAN is sensitive)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'most models can reach similar scores with enough hyperparameter optimization' and 'we did not find evidence that any of the tested algorithms consistently outperforms the non-saturating GAN' — both supported by Table 2 and Figure 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are modest and supported by controlled experiments. The claim that 'improvements can arise from a higher computational budget' is supported by Figure 3 showing budget-FID curves with controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations) explicitly states that results are limited to the tested architecture, datasets, and optimization settings, and acknowledges that 'some models significantly outperform others under currently unexplored conditions.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses alternative explanations: architecture choice could change results, different optimization methods could matter, and FID limitations could affect conclusions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The models compared are fully specified via their loss functions (Table 1). The architecture is specified as InfoGAN-based with exact details (latent code size 64, batch size 64, training epochs, etc.)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting — it trains generative models from scratch."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameter ranges are reported in Section 6 and Appendix A. Learning rate, beta1, batch norm, discriminator iterations, and model-specific parameters (lambda, gamma, clipping) are all specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard training and evaluation study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The datasets used are standard benchmarks (MNIST, Fashion-MNIST, CIFAR10, CelebA). The paper describes how FID samples are computed (10k samples), how the precision/recall datasets are constructed (convex polygons), and the train/test partition procedure."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations of the Study' is a dedicated section covering three categories of limitations: datasets/architectures, metrics, and hyperparameter exploration."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: single architecture may not generalize to higher-resolution tasks, FID may be fooled by encoding-specific artifacts, FID cannot detect overfitting, hyperparameter ranges from one dataset may not transfer."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'We cannot exclude the possibility that some models significantly outperform others under currently unexplored conditions' and notes results are limited to the InfoGAN architecture and four specific datasets."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The open-sourced code at goo.gl/G8kf5J includes the experimental setup, and the datasets used are all publicly available standard benchmarks."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "All datasets are standard public benchmarks (MNIST, Fashion-MNIST, CIFAR10, CelebA). The novel convex polygon dataset generation procedure is described in Section 5."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: hyperparameter sampling → training → FID evaluation every 5 epochs → early stopping based on best FID → re-running best model 50 times with different seeds (Section 6)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgment of grants. The acknowledgments section only thanks individuals for discussions."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Google Brain affiliates in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Google Brain is not evaluating its own GAN product. The study compares published GAN algorithms neutrally; Google has no financial stake in one algorithm winning over another."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper trains models from scratch on specified datasets. It does not evaluate pre-trained model capabilities on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Models are trained from scratch; no pre-trained model knowledge contamination applies."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Models are trained from scratch; benchmark contamination from pre-training is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No per-model inference cost or training cost per run is reported, only the total computational budget."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Reproducing these experiments requires approximately 6.85 GPU years (NVIDIA P100)' (Section 1, footnote 2)."
       }
     }
   },
   "claims": [
     {
       "claim": "Most GAN models can reach similar FID scores with enough hyperparameter optimization and random restarts.",
       "evidence": "Table 2 shows overlapping FID distributions across models. Figure 3 shows convergence of minimum FID as computational budget increases.",
       "supported": "strong"
     },
     {
       "claim": "No tested algorithm consistently outperforms the non-saturating GAN (NS GAN).",
       "evidence": "Table 2 shows NS GAN is competitive or best on multiple datasets. Figure 3 shows no model clearly dominates across budgets.",
       "supported": "strong"
     },
     {
       "claim": "A 'bad' model with higher computational budget can outperform a 'good' model with lower budget.",
       "evidence": "Figure 3 shows crossing curves where models with worse expected performance at low budgets achieve better FID at higher budgets.",
       "supported": "strong"
     },
     {
       "claim": "FID is sensitive to mode dropping and robust to encoding network choice.",
       "evidence": "Figure 1b shows FID increases sharply with mode dropping. Figure 1c shows Spearman's rank correlation of 0.9 between InceptionNet and VGG-based FID.",
       "supported": "strong"
     },
     {
       "claim": "GAN training is extremely sensitive to hyperparameter settings.",
       "evidence": "Figure 4 shows wide variance in FID across hyperparameter samples. Figure 14 shows even narrow-range search produces significant variance.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This large-scale study (6.85 GPU years on P100s) finds that no GAN algorithm consistently outperforms the original non-saturating GAN when given fair hyperparameter optimization budgets. Performance differences between state-of-the-art GANs are largely attributable to computational budget and hyperparameter tuning rather than algorithmic innovations. The paper also proposes convex polygon datasets enabling precision/recall measurement and demonstrates that FID is a robust evaluation metric.",
   "red_flags": [],
   "cited_papers": [
     {
       "title": "Wasserstein generative adversarial networks",
       "authors": ["Martín Arjovsky", "Soumith Chintala", "Léon Bottou"],
       "year": 2017,
       "relevance": "Foundational GAN variant evaluated in the study; demonstrates importance of rigorous model comparison."
     },
     {
       "title": "Generalization and equilibrium in generative adversarial nets (GANs)",
       "authors": ["Sanjeev Arora", "Rong Ge", "Yingyu Liang", "Tengyu Ma", "Yi Zhang"],
       "year": 2017,
       "relevance": "Theoretical analysis of GAN convergence and generalization relevant to evaluation methodology."
     },
     {
       "title": "Many paths to equilibrium: GANs do not need to decrease a divergence at every step",
       "authors": ["William Fedus", "Mihaela Rosca", "Balaji Lakshminarayanan", "Andrew M. Dai", "Shakir Mohamed", "Ian Goodfellow"],
       "year": 2018,
       "relevance": "Concurrent GAN comparison study using IS; this paper extends it with FID and precision/recall."
     },
     {
       "title": "GANs trained by a two time-scale update rule converge to a local Nash equilibrium",
       "authors": ["Martin Heusel", "Hubert Ramsauer", "Thomas Unterthiner", "Bernhard Nessler", "Sepp Hochreiter"],
       "year": 2017,
       "relevance": "Proposes FID metric that this paper extensively evaluates and validates."
     },
     {
       "title": "Improved techniques for training GANs",
       "authors": ["Tim Salimans", "Ian Goodfellow", "Wojciech Zaremba", "Vicki Cheung", "Alec Radford", "Xi Chen"],
       "year": 2016,
       "relevance": "Proposes Inception Score metric whose limitations motivate this study's focus on FID."
     }
   ]
 }