ai-research-survey

scan.json (21451B)

---

 {
   "paper": {
     "title": "Show Your Work: Improved Reporting of Experimental Results",
     "authors": ["Jesse Dodge", "Suchin Gururangan", "Dallas Card", "Roy Schwartz", "Noah A. Smith"],
     "year": 2019,
     "venue": "EMNLP",
     "arxiv_id": "1909.03004"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code released at https://github.com/allenai/allentune (footnote 1) and https://github.com/allenai/show-your-work (Section 4.2, footnote 11)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (SST, SciTail, SQuAD). Hyperparameter search results are implicitly available through the code repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Computing infrastructure is specified per experiment in the appendix tables (e.g., 'GeForce GTX 1080 GPU', '3.1 GHz Intel Core i7 CPU', 'NVIDIA Titan Xp GPU'). AllenNLP platform is referenced. However, no requirements.txt or detailed library versions are provided, so this is borderline. The GPU and software platform (AllenNLP, scikit-learn) are stated."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix tables B-D provide complete hyperparameter search spaces, best assignments, number of trials, search strategy, and computing infrastructure for every experiment. Code is linked. This is sufficient for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported as shaded regions in figures (e.g., Fig. 4: 'shaded area represents the expected performance ±1 standard deviation'). Section 3.3 discusses variance estimation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about model rankings changing with budget but does not use statistical significance tests for these comparisons. Rankings are compared visually from expected validation performance curves."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete performance differences with context, e.g., models require budgets ranging from 2 to 20 hyperparameter trials to match reported performance (Section 4.4), and 18 GPU days for BiDAF on SQuAD (Section 4.4). The EMNLP survey reports specific percentages (e.g., '74% reported best hyperparameter assignments', '10% or fewer reported search bounds')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The EMNLP 2018 survey uses 50 randomly sampled papers (Section 5) but does not justify why 50 was chosen or discuss statistical power for the prevalence estimates."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 is dedicated to computing variance of the expected maximum. Fig. 4 shows ±1 standard deviation. The paper explicitly advocates for reporting variance (Section 3.3: 'Expected performance becomes more useful with an estimate of variation')."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its expected maximum technique against the bootstrap method (Section 3.2) and against current practice of reporting single best test scores."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The bootstrap is the standard non-parametric method for estimating statistics without closed form. The paper also compares against models from contemporary work (Peters et al. 2019, Khot et al. 2018)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The proposed technique (expected validation performance) is a single mathematical method with no separable components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy on SST (both fine-grained and binary), accuracy on SciTail, and exact match on SQuAD across different experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper proposes a reporting methodology based on mathematical computation of expected validation performance. Human evaluation of outputs is not relevant to these claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes validation and test performance. The expected maximum is computed on validation data, and test results are reported separately (Section 4, Fig. 1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model family (LR vs CNN, DGEM vs DAM vs ESIM vs n-gram, BCN with different embeddings) and per dataset (SST, SciTail, SQuAD). The EMNLP survey provides per-item breakdown in Table 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses limitations of the approach, including that fixing the same number of hyperparameter trials does not imply fair comparison, and that manual tuning history cannot be captured. Section 4.4 notes that hyperparameter search space choices affect results."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper shows cases where model rankings reverse with different budgets (Fig. 1, Fig. 3), which is itself a negative result about current reporting practices. Section 5 acknowledges 'there may be no simple way to make a comparison fair.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that (1) test-set scores alone are insufficient (supported by Figs. 1, 3 showing ranking reversals), (2) they find multiple cases where conclusions change with budget (Sections 4.2-4.4), and (3) massive variation in compute required (Section 4.4, 'from hours to weeks'). All are supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that varying computational budget causes changes in relative model rankings. This is demonstrated through controlled experiments where the same models are evaluated under systematically varied budgets, which is adequate causal design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper focuses on NLP and frames its contributions accordingly. Section 5 discusses limitations of the approach (e.g., manual tuning cannot be captured, search space choices matter). Claims are bounded to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses alternative explanations: different hyperparameter spaces could change results, prior human experience affects search bounds for popular models, implementation differences affect runtime comparisons. Footnote 13 acknowledges 'different choices could result in better or worse expected performance.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model implementations are cited (AllenNLP BCN, scikit-learn logistic regression, implementations from Khot et al. 2018, Seo et al. 2017). For NLP models in 2019, these are well-defined architectures, not API-based models with versioning issues."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It trains traditional NLP models (CNNs, LSTMs, logistic regression) with hyperparameter search."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix tables B-D provide exhaustive hyperparameter search spaces and best assignments for every experiment. This is exemplary reporting — the paper practices what it preaches."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper trains traditional NLP models."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Standard datasets are used (SST, SciTail, SQuAD) with standard splits. The EMNLP survey describes sampling 50 random papers. Preprocessing for the models follows the cited implementations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('Recommendations') contains substantive discussion of limitations, including the difficulty of fair comparisons when hyperparameter spaces differ, the inability to account for manual tuning history, and implementation differences."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: 'the two models in Fig. 1 have hyperparameter spaces that are different, so fixing the same number of hyperparameter trials for both models does not imply a fair comparison.' Footnote 13 acknowledges search space choices affect results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states 'there may be no simple way to make a comparison fair' and discusses specific things the approach does NOT address: manual tuning history, implementation-dependent runtime differences, and adaptive search strategies."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at github.com/allenai/allentune and github.com/allenai/show-your-work, which would contain the raw hyperparameter search results. Standard public datasets are used."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "For experiments: uniform random hyperparameter sampling with specified search spaces and number of trials (50-128). For the EMNLP survey: '50 random EMNLP 2018 papers that include experimental results' (Section 5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The EMNLP paper survey sampled papers, not people, and the selection method (random from EMNLP 2018) is described in the paper."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from hyperparameter sampling to expected maximum computation is mathematically specified in Section 3. The EMNLP survey pipeline is straightforward (random sample, evaluate checklist items)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments state 'This work was completed while the first author was an intern at the Allen Institute for Artificial Intelligence.' Authors are affiliated with AI2, CMU, and UW."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: CMU, Allen Institute for AI, UW. The paper does not evaluate any product from these organizations, so no conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "AI2 and university affiliations have no financial stake in whether reporting practices improve. The paper advocates for methodological reform, not a commercial product."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains traditional NLP models from scratch (CNNs, LSTMs, logistic regression). It does not evaluate pre-trained model capabilities on benchmarks in a way where training data contamination is relevant."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — models are trained from scratch on standard splits. No pre-trained model benchmark evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — not applicable to this paper type."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training durations are reported per experiment in appendix tables (e.g., 39 sec for CNN on SST, 31617 sec for BiDAF on SQuAD). The paper's central argument is about computational cost of hyperparameter search."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Number of search trials (50-128), GPU hardware, and per-trial training durations are all reported in the appendix. Total compute is derivable (e.g., 128 trials × ~8.8 hours = ~47 GPU days for SQuAD BiDAF)."
       }
     }
   },
   "claims": [
     {
       "claim": "Test-set performance scores alone are insufficient for drawing accurate conclusions about which model performs best.",
       "evidence": "Figures 1 and 3 show that relative model rankings change depending on computational budget (number of hyperparameter trials). On SST, LR outperforms CNN with fewer than 10 trials but CNN is better with larger budgets.",
       "supported": "strong"
     },
     {
       "claim": "Multiple recent model comparisons would have reached different conclusions with more or less computation.",
       "evidence": "Section 4.4 shows SciTail model rankings vary with budget (Fig. 3), and Section 4.3 shows ELMo embedding approaches have different budget-performance tradeoffs (Fig. 2).",
       "supported": "strong"
     },
     {
       "claim": "The amount of computation required to obtain reported results varies massively across papers, from hours to weeks.",
       "evidence": "Section 4.4: BiDAF on SQuAD requires ~18 GPU days (55 trials) to match reported performance in expectation (Fig. 4), while simpler models on SciTail require only 2 trials.",
       "supported": "strong"
     },
     {
       "claim": "None of 50 sampled EMNLP 2018 papers reported all checklist items; 10% or fewer reported hyperparameter search bounds, number of trials, or score distributions.",
       "evidence": "Table 1 in Appendix A provides the full breakdown across 50 randomly sampled EMNLP 2018 papers.",
       "supported": "strong"
     },
     {
       "claim": "The expected maximum method has strictly less error than the bootstrap for estimating expected best validation performance.",
       "evidence": "Section 3.2 argues the closed-form solution eliminates resampling error while retaining the same finite sample error. This is a mathematical argument, not an empirical comparison.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper demonstrates that reporting only test-set performance is insufficient because relative model rankings change depending on computational budget for hyperparameter search. Using a closed-form method for computing expected validation performance as a function of budget, the authors show that on SciTail and SST, the best model depends on how many hyperparameter configurations are tried. A survey of 50 EMNLP 2018 papers found that 10% or fewer reported hyperparameter search bounds, number of trials, or score distributions. The paper proposes a reporting checklist and releases the allentune tool for computing expected validation performance curves.",
   "red_flags": [],
   "cited_papers": [
     {
       "title": "Troubling Trends in Machine Learning Scholarship",
       "authors": ["Zachary C. Lipton", "Jacob Steinhardt"],
       "year": 2018,
       "relevance": "Directly addresses methodological problems in ML research including incorrectly attributing empirical gains to modeling choices."
     },
     {
       "title": "Green AI",
       "authors": ["Roy Schwartz", "Jesse Dodge", "Noah A. Smith", "Oren Etzioni"],
       "year": 2019,
       "relevance": "Addresses computational cost and efficiency in AI research, directly relevant to cost and practicality assessment."
     },
     {
       "title": "Winner's Curse? On Pace, Progress, and Empirical Rigor",
       "authors": ["D. Sculley", "Jasper Snoek", "Ali Rahimi", "Alex Wiltschko"],
       "year": 2018,
       "relevance": "Calls for stronger standards for empirical evaluation in ML, directly relevant to methodology quality assessment."
     },
     {
       "title": "Reporting Score Distributions Makes a Difference: Performance Study of LSTM-Networks for Sequence Tagging",
       "authors": ["Nils Reimers", "Iryna Gurevych"],
       "year": 2017,
       "relevance": "Demonstrates that reporting score distributions instead of single scores changes conclusions about model comparisons."
     },
     {
       "title": "Deep Reinforcement Learning That Matters",
       "authors": ["Peter Henderson", "Riashat Islam", "Philip Bachman", "Joelle Pineau", "Doina Precup", "David Meger"],
       "year": 2018,
       "relevance": "Addresses reproducibility and reporting issues in deep RL, a parallel concern to the NLP reporting issues in this paper."
     },
     {
       "title": "Are GANs Created Equal? A Large-Scale Study",
       "authors": ["Mario Lucic", "Karol Kurach", "Marcin Michalski", "Olivier Bousquet", "Sylvain Gelly"],
       "year": 2018,
       "relevance": "Found competing GAN methods to be comparable rather than clearly superior, exposing the need for better reporting standards."
     },
     {
       "title": "Datasheets for Datasets",
       "authors": ["Timnit Gebru", "Jamie H. Morgenstern", "Briana Vecchione", "Jennifer Wortman Vaughan", "Hanna M. Wallach", "Hal Daumé", "Kate Crawford"],
       "year": 2018,
       "relevance": "Proposes structured documentation for datasets, parallel to this paper's proposal for structured reporting of experimental results."
     },
     {
       "title": "Model Cards for Model Reporting",
       "authors": ["Margaret Mitchell", "Simone Wu", "Andrew Zaldivar", "Parker Barnes", "Lucy Vasserman", "Ben Hutchinson", "Elena Spitzer", "Inioluwa Deborah Raji", "Timnit Gebru"],
       "year": 2019,
       "relevance": "Proposes structured documentation for trained models, complementary to experimental results reporting."
     },
     {
       "title": "On the State of the Art of Evaluation in Neural Language Models",
       "authors": ["Gábor Melis", "Chris Dyer", "Phil Blunsom"],
       "year": 2018,
       "relevance": "Found that improvements attributed to novel architectures could be obtained through better hyperparameter tuning of existing models."
     },
     {
       "title": "Random Search and Reproducibility for Neural Architecture Search",
       "authors": ["Liam Li", "Ameet Talwalkar"],
       "year": 2019,
       "relevance": "Addresses reproducibility in neural architecture search, directly relevant to hyperparameter search methodology."
     }
   ]
 }