ai-research-survey

scan.json (18797B)

---

 {
   "paper": {
     "title": "Attention Is All You Need",
     "authors": ["Ashish Vaswani", "Noam Shazeer", "Niki Parmar", "Jakob Uszkoreit", "Llion Jones", "Aidan N. Gomez", "Łukasz Kaiser", "Illia Polosukhin"],
     "year": 2017,
     "venue": "NeurIPS 2017 (NIPS 2017)",
     "arxiv_id": "1706.03762"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/tensorflow/tensor2tensor (Section 7, Conclusion)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard public benchmarks: WMT 2014 English-German, WMT 2014 English-French, and Penn Treebank WSJ, all publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 8 NVIDIA P100 GPUs (Section 5.2) but provides no dependency specifications, library versions, or environment setup instructions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code link is given but no README-level guidance for replicating specific experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, and 4 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims state-of-the-art results by comparing BLEU scores directly without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context, e.g., 'improving over the existing best results, including ensembles, by over 2 BLEU' (abstract), and Table 2 provides full baseline comparisons enabling effect size assessment."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes or discussion of whether the training data is sufficient. Standard benchmarks are used but no power or sample size analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run results are reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares against ByteNet, GNMT+RL, ConvS2S, MoE, and their ensemble variants. Table 4 compares against multiple parsing baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include ConvS2S (2017), MoE (2017), GNMT+RL (2016) — all contemporary or near-contemporary at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides extensive ablations varying number of attention heads (A), key dimension (B), model size (C), dropout and label smoothing (D), and positional encoding (E)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both BLEU and perplexity (PPL) for translation experiments (Table 3), and F1 for constituency parsing (Table 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of translation quality is included. All evaluation is automated (BLEU, PPL, F1). For a machine translation paper claiming state-of-the-art quality, human evaluation is relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Main results (Table 2) are on newstest2014 test set. Ablations (Table 3) are explicitly on newstest2013 development set. Parsing results on Section 23 of WSJ (standard test split)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (EN-DE, EN-FR translation, constituency parsing) and by model variant (Table 3 ablations)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is presented. The paper does not discuss where the Transformer fails or produces poor translations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows configurations that hurt performance: single attention head loses 0.9 BLEU (row A), reducing dk hurts quality (row B), removing dropout (row D) hurts. Label smoothing 'hurts perplexity' (Section 5.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 28.4 BLEU on EN-DE and 41.8 on EN-FR are supported by Table 2. Training time of 3.5 days on 8 GPUs is stated in Section 5.2. Constituency parsing claim is supported by Table 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by controlled ablation studies in Table 3 (single-variable manipulations of heads, dimensions, dropout, etc.)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Attention Is All You Need' and abstract claim that the Transformer 'generalizes well to other tasks' based on only one additional task (English constituency parsing) overstate the evidence. The paper tests on two translation language pairs and one parsing task."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the performance gains. Could the improvement be due to increased parallelism enabling better hyperparameter tuning rather than architectural superiority? No confounds are discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper proposes its own architecture; it does not evaluate pre-existing versioned models/APIs."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a neural architecture paper with standard supervised training."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive hyperparameters: N=6 layers, dmodel=512, dff=2048, h=8 heads, dk=dv=64, Pdrop=0.1, εls=0.1, Adam with β1=0.9, β2=0.98, warmup=4000, beam size=4, length penalty α=0.6 (Sections 3, 5, 6)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a neural network architecture paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes byte-pair encoding with ~37K shared vocabulary for EN-DE, 32K word-piece vocabulary for EN-FR, batching by approximate sequence length with ~25K source and target tokens per batch."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The conclusion mentions plans to extend to other modalities but does not explicitly bound what the current results do NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "WMT 2014 and Penn Treebank are standard public benchmarks with publicly available data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the datasets: WMT 2014 EN-DE (~4.5M sentence pairs), WMT 2014 EN-FR (36M sentences), Penn Treebank WSJ (~40K training sentences)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 documents the pipeline: raw sentences → byte-pair/word-piece encoding → batching by approximate sequence length. Checkpoint averaging and beam search parameters are documented in Section 6.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The paper has an Acknowledgements section thanking individuals but no funding statement. All authors are affiliated with Google Brain/Research or University of Toronto."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Google Brain, Google Research, University of Toronto. Footnotes note work performed while at Google."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Google employees proposing a Google architecture have an inherent interest in its success."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains its own models from scratch on specified datasets; it does not evaluate a pre-trained model's knowledge on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the model is trained from scratch on specified training data with standard train/dev/test splits."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no pre-trained model is evaluated on benchmarks that could have been in its training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens-per-second metrics are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training compute is well documented: base model trained for 12 hours on 8 P100 GPUs (100K steps), big model for 3.5 days (300K steps). Table 2 provides estimated FLOPs for all models."
       }
     }
   },
   "claims": [
     {
       "claim": "The Transformer (big) achieves 28.4 BLEU on WMT 2014 English-to-German, improving over existing best results including ensembles by over 2 BLEU.",
       "evidence": "Table 2 shows Transformer (big) at 28.4 BLEU vs. previous best ensemble (GNMT+RL Ensemble) at 26.30 BLEU.",
       "supported": "strong"
     },
     {
       "claim": "The Transformer (big) achieves a new single-model state-of-the-art BLEU score of 41.8 on WMT 2014 English-to-French.",
       "evidence": "Table 2 shows 41.8 BLEU. However, Section 6.1 reports 41.0 BLEU for this task, creating an inconsistency with the abstract's 41.8 claim.",
       "supported": "moderate"
     },
     {
       "claim": "The Transformer requires significantly less training cost than competitive models.",
       "evidence": "Table 2 shows Transformer (big) EN-DE at 2.3×10^19 FLOPs vs. GNMT+RL Ensemble at 1.8×10^20 FLOPs — roughly 8x less. Base model at 3.3×10^18 is even cheaper.",
       "supported": "strong"
     },
     {
       "claim": "The Transformer generalizes well to English constituency parsing.",
       "evidence": "Table 4 shows 91.3 F1 (WSJ only) and 92.7 F1 (semi-supervised), competitive with or exceeding most baselines except RNNG generative (93.3).",
       "supported": "moderate"
     },
     {
       "claim": "Multi-head attention is beneficial; single-head attention is 0.9 BLEU worse.",
       "evidence": "Table 3 row (A) shows single-head at 24.9 BLEU vs. base (8 heads) at 25.8 BLEU on the dev set.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces the Transformer architecture, which relies entirely on self-attention mechanisms and dispenses with recurrence and convolution. On WMT 2014 machine translation, it achieves state-of-the-art BLEU scores (28.4 EN-DE, 41.8 EN-FR) while requiring a fraction of the training compute of competitive models. Ablation studies demonstrate the importance of multi-head attention, model size, and regularization. The architecture also transfers to English constituency parsing with competitive results.",
   "red_flags": [
     {
       "flag": "No variance or multi-run results",
       "detail": "All results appear to be single runs with no error bars, standard deviations, or confidence intervals. For a paper making state-of-the-art claims, this makes it impossible to assess whether observed differences are within noise."
     },
     {
       "flag": "Abstract/body BLEU inconsistency on EN-FR",
       "detail": "The abstract claims 41.8 BLEU on EN-FR, but Section 6.1 states 'our big model achieves a BLEU score of 41.0' for the same task. This discrepancy is unexplained."
     },
     {
       "flag": "Overly broad title and generalization claims",
       "detail": "The title 'Attention Is All You Need' and claims of generalization are based on only two translation language pairs and one parsing task. The scope of evidence is narrower than the scope of claims."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no discussion of limitations, threats to validity, or scope boundaries despite making broad architectural claims."
     },
     {
       "flag": "Conflict of interest: Google evaluating Google architecture",
       "detail": "All authors are Google employees (with one University of Toronto affiliate working at Google). The paper proposes and evaluates a Google-developed architecture with no independent evaluation or conflict disclosure."
     }
   ],
   "cited_papers": [
     {
       "title": "Neural machine translation by jointly learning to align and translate",
       "authors": ["Dzmitry Bahdanau", "Kyunghyun Cho", "Yoshua Bengio"],
       "year": 2014,
       "arxiv_id": "1409.0473",
       "relevance": "Foundational attention mechanism paper for sequence-to-sequence models, directly motivating the Transformer's attention-only approach."
     },
     {
       "title": "Convolutional sequence to sequence learning",
       "authors": ["Jonas Gehring", "Michael Auli", "David Grangier", "Denis Yarats", "Yann N. Dauphin"],
       "year": 2017,
       "arxiv_id": "1705.03122",
       "relevance": "Key baseline: convolutional approach to sequence transduction that the Transformer outperforms."
     },
     {
       "title": "Google's neural machine translation system: Bridging the gap between human and machine translation",
       "authors": ["Yonghui Wu", "Mike Schuster", "Zhifeng Chen"],
       "year": 2016,
       "arxiv_id": "1609.08144",
       "relevance": "GNMT is a primary baseline; represents the recurrent approach the Transformer replaces."
     },
     {
       "title": "Outrageously large neural networks: The sparsely-gated mixture-of-experts layer",
       "authors": ["Noam Shazeer", "Azalia Mirhoseini", "Krzysztof Maziarz"],
       "year": 2017,
       "arxiv_id": "1701.06538",
       "relevance": "Mixture-of-experts baseline in Table 2; relevant to scaling and efficiency of large models."
     },
     {
       "title": "Sequence to sequence learning with neural networks",
       "authors": ["Ilya Sutskever", "Oriol Vinyals", "Quoc VV Le"],
       "year": 2014,
       "relevance": "Foundational encoder-decoder architecture that the Transformer builds upon and replaces recurrence in."
     },
     {
       "title": "Deep residual learning for image recognition",
       "authors": ["Kaiming He", "Xiangyu Zhang", "Shaoqing Ren", "Jian Sun"],
       "year": 2016,
       "relevance": "Residual connections used in the Transformer architecture are adopted from this work."
     }
   ]
 }