ai-research-survey

scan.json (25875B)

---

 {
   "paper": {
     "title": "Lessons from the Trenches on Reproducible Evaluation of Language Models",
     "authors": ["Stella Biderman", "Hailey Schoelkopf", "Lintang Sutawika", "Leo Gao", "Jonathan Tow", "Baber Abbasi", "Alham Fikri Aji", "Pawan Sasanka Ammanamanchi", "Sidney Black", "Jordan Clive", "Anthony DiPofi", "Julen Etxaniz", "Benjamin Fattori", "Jessica Zosa Forde", "Charles Foster", "Jeffrey Hsu", "Mimansa Jaiswal", "Wilson Y. Lee", "Haonan Li", "Charles Lovering", "Niklas Muennighoff", "Ellie Pavlick", "Jason Phang", "Aviya Skowron", "Samson Tan", "Xiangru Tang", "Kevin A. Wang", "Genta Indra Winata", "François Yvon", "Andy Zou"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2405.14782",
     "doi": "10.48550/arXiv.2405.14782"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "The paper identifies key challenges in LM evaluation including the 'Key Problem' of semantic equivalence, sensitivity to minor implementation details, and lack of reproducibility. Case studies demonstrate that prompt format changes can cause score swings of 10-20+ percentage points on ARC and MMLU. The lm-eval library addresses these via standardized task implementations, version tracking, standard error reporting, and configurable task definitions. Best practices include sharing exact prompts and code, avoiding copying results across papers, providing model outputs, and reporting uncertainty.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes lm-eval as an open source library and references the repository (Gao et al., 2021, with Zenodo DOI 10.5281/zenodo.5371628). The library is publicly available."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used are publicly available standard datasets (ARC, MMLU, etc.) accessed through the HuggingFace Datasets library. Task configurations are part of the open-source lm-eval codebase."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. The paper describes lm-eval's design but does not specify the environment needed to reproduce the case study results."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While YAML configuration files are shown in Appendix B.2 and the library is described, no step-by-step instructions for reproducing the specific case study results (Table 1, etc.) are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports 95% confidence intervals via standard error of the mean for all results (e.g., '38.0 ± 2.78%'). The paper also advocates for this practice (Section 3, Section 4.2)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Despite advocating for statistical significance testing (Section 3 'Measure and Report Uncertainty'), the paper does not perform significance tests on its own comparative results in Table 1."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides absolute scores with baseline context for different prompt styles, allowing readers to assess effect magnitudes (e.g., Mistral-7B: 58.6% MMLU-style vs 48.3% Hybrid, a 10.3pp difference)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 5 models in its case study (Table 1) without justifying why these specific models or this number were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run evaluations. The confidence intervals reported are standard errors of the metric (sampling variance), not variance across multiple experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The case studies compare multiple models (GPT-NeoX-20B, Llama-2-7B, Falcon-7B, Mistral-7B, Mixtral-8x7B) across prompt styles, serving as baselines for each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models used include Mistral-7B and Mixtral-8x7B (2023-2024), which were contemporary at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 and Section 5.2 serve as ablation studies on prompt format: comparing Cloze vs MMLU-style for ARC, and MMLU-style vs Hybrid for MMLU, isolating the effect of prompt formatting."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses and implements multiple metrics: accuracy, acc_norm (byte-length normalized), acc_mutual_info, perplexity, bits per byte, word-level perplexity (Section 4.1, Appendix A)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is an infrastructure/methodology paper about evaluation tools. Human evaluation of the tool's outputs is not relevant to its claims about standardization and reproducibility."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard test splits of ARC and MMLU benchmarks. The YAML configs show explicit test_split: test."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down results per model and per prompt style. Table 2 surveys architectures across multiple individual benchmarks. Results are not just aggregated."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 extensively discusses failure modes: sensitivity to prompts, reproducibility failures, the Key Problem, benchmark validity issues. Section 5.2 shows cases where prompt changes cause dramatic performance shifts."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively reports negative findings: difficulty reproducing results from other papers, cases where evaluation methodology breaks down, and problems with current practices (Section 2, Section 5.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about challenges in LM evaluation, best practices, and the lm-eval library are all substantiated in the paper's three main sections (2, 3, 4) with examples and case studies."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily about prompt format affecting scores. Table 1 demonstrates this through controlled experiments varying only the prompt style while holding models constant, which is adequate for this causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about scope: it focuses on autoregressive language models (Section 2.1 explicitly notes the Key Problem does not apply to all domains), and case studies are presented as illustrative examples rather than universal claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 extensively discusses alternative explanations for evaluation differences: tokenization effects, prompt sensitivity, normalization choices, benchmark validity. Section 5.2 notes that confidence intervals alone cannot resolve methodological differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly discusses benchmark validity — the distinction between benchmark scores (proxy) and real-world capabilities (outcome). The paper acknowledges benchmarks are proxies and discusses construct validity concerns."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 uses model names like 'Mistral-7B' and 'Mixtral-8x7B' without specifying exact versions, revision hashes, or snapshot dates. References point to papers but not specific model checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full YAML configuration files including complete prompt templates are provided in Appendix B.2 for both ARC and MMLU variants, with Jinja templates showing the exact text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other sampling/generation hyperparameters are reported for the evaluations in Table 1, despite the paper being about evaluation methodology."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates models directly through the lm-eval harness using loglikelihood-based scoring."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides detailed documentation of tokenization handling, whitespace shifting, loglikelihood computation, perplexity calculation methods, and sliding window approaches. YAML configs show exact data source and split configurations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section in the paper. The conclusion (Section 6) is brief and does not discuss limitations of the library or the paper's analysis."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity of the paper's own analyses are discussed. While Section 2.2 discusses validity of benchmarks in general, the paper does not reflect on limitations of its own case studies or recommendations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly notes the Key Problem does not impact domains with verifiers (coding, math, games). Section 2.2 states lm-eval focuses on measurement consistency rather than construct validity. Section 4 clarifies lm-eval does not prescribe benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release raw evaluation outputs, per-sample results, or logs from the case study experiments despite advocating for this practice in Section 3 ('Always provide model outputs')."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation procedure is described in detail: which benchmarks, which models, which prompt formats, and how scores are computed (Section 5.2, Appendix A, Appendix B.2)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates language models on standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides extensive documentation of the full data pipeline from tokenization through loglikelihood computation to final metric calculation, including normalization options and scoring approaches."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments or grant information is provided in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: EleutherAI, Stability AI, MBZUAI, Brown University, Amazon, Yale, CMU, etc. Multiple industry and academic affiliations are transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Several authors are from EleutherAI, which develops and maintains lm-eval — the tool being promoted in this paper."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors from Amazon, Stability AI, and other companies that develop LLMs are evaluating infrastructure related to LLM evaluation."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates several pretrained models (GPT-NeoX-20B, Llama-2, Falcon, Mistral, Mixtral) on ARC and MMLU without stating any training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Despite the paper discussing contamination as a general concern and even mentioning benchmark contamination issues (Section 2.4), it does not address potential train/test overlap for the specific models evaluated in its own case studies."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ARC (2018) and MMLU (2020) were published before all evaluated models were trained. The paper does not discuss whether these benchmarks may have been in the training data of the models used in Table 1."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or evaluation compute time are reported for the case study experiments despite the paper noting that evaluation is expensive (Section 2.3.3)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget (GPU hours, hardware used) is reported for running the case study evaluations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 1 appear to be single-run evaluations. No seed sensitivity analysis is performed despite the paper discussing the importance of multi-seed evaluation (Section 3)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs for Table 1 results is not stated. Standard errors reported are of the metric (sampling variance), not from multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter search was performed — the paper evaluates pretrained models with fixed configurations, not tuning any parameters."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection was needed — the paper compares predetermined prompt formats, not selecting among them for best performance."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors of lm-eval are evaluating lm-eval as a contribution to evaluation rigor. This self-evaluation bias is not acknowledged despite the paper being about evaluation methodology."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The paper compares prompt formats, not compute-intensive approaches. Compute differences between configurations are negligible."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly discusses benchmark validity and construct validity, referencing Subramonian et al. (2023), Raji et al. (2021), Saphra et al. (2023), and Davis (2023). The paper acknowledges that benchmarks may not measure what is claimed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated directly via loglikelihood scoring."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Despite Section 2.4 and Figure 1 discussing how benchmarks predate modern models, the paper does not address temporal leakage for its own experiments (e.g., whether Mistral-7B's training data included ARC/MMLU solutions)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information to models through the prompt format or few-shot examples."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples share structure or sources with model training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied to the paper's own evaluations."
       }
     }
   },
   "claims": [
     {
       "claim": "Minor implementation details like prompt format significantly impact LM evaluation scores",
       "evidence": "Table 1 shows Mistral-7B scores 72.4% vs 25.9% (Falcon-7B) on ARC with MMLU-style prompting but 50.1% vs 40.2% with cloze-style; MMLU scores for Mistral-7B shift from 58.6% to 48.3% between MMLU-style and Hybrid formats (Section 5.2)",
       "supported": "strong"
     },
     {
       "claim": "Copying results across papers with different evaluation setups leads to nonsensical comparisons",
       "evidence": "Section 5.2 demonstrates that different prompt styles produce substantially different scores, making cross-paper comparisons unreliable. Section 3 recommends against copying results (Section 3, citing Marie et al. 2021)",
       "supported": "strong"
     },
     {
       "claim": "lm-eval has been widely adopted by the community for standardized evaluation",
       "evidence": "Table 2 surveys 12 recent architecture releases and shows most use lm-eval for evaluation. Section 5.3 mentions adoption for the Open LLM Leaderboard (Beeching et al., 2023) and community benchmark contributions",
       "supported": "strong"
     },
     {
       "claim": "Standard error reporting and confidence intervals are important but insufficient for addressing evaluation methodology differences",
       "evidence": "Section 5.2 states: 'better statistical reporting such as the use of confidence intervals which we report, does not resolve these issues–while it gives a sense of how reliable a given measurement is, it cannot tell us how much model performance will vary across different measurement settings'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Authors evaluate their own tool",
       "detail": "EleutherAI authors present lm-eval, a tool they built and maintain, as the solution to evaluation challenges. The paper reads partly as advocacy for their tool without acknowledging self-evaluation bias."
     },
     {
       "flag": "Does not follow own recommendations",
       "detail": "The paper recommends sharing model outputs, reporting uncertainty across runs, and performing statistical significance testing, but does not do any of these for its own case study results. Table 1 shows single-run results with only sampling standard errors."
     },
     {
       "flag": "No contamination analysis despite discussing it",
       "detail": "The paper discusses benchmark contamination as a general concern (Section 2.4) and temporal misalignment (Figure 1), but does not analyze whether the models in its own experiments may have been contaminated on ARC/MMLU."
     }
   ],
   "cited_papers": [
     {
       "title": "The benchmark lottery",
       "authors": ["Mostafa Dehghani", "Yi Tay", "Alexey A. Gritsenko", "Zhe Zhao", "Neil Houlsby", "Fernando Diaz", "Donald Metzler", "Oriol Vinyals"],
       "year": 2021,
       "relevance": "Studies inconsistencies and biases in evaluation practices and their influence on research direction."
     },
     {
       "title": "Quantifying language models' sensitivity to spurious features in prompt design",
       "authors": ["Melanie Sclar", "Yejin Choi", "Yulia Tsvetkov", "Alane Suhr"],
       "year": 2023,
       "relevance": "Demonstrates sensitivity of LMs to minor prompt variations, core evidence for evaluation reproducibility concerns."
     },
     {
       "title": "When benchmarks are targets: Revealing the sensitivity of large language model leaderboards",
       "authors": ["Norah Alzahrani"],
       "year": 2024,
       "relevance": "Explores LLM sensitivity to evaluation setup including prompt format, directly relevant to benchmark contamination and evaluation rigor."
     },
     {
       "title": "Scientific credibility of machine translation research: A meta-evaluation of 769 papers",
       "authors": ["Benjamin Marie", "Atsushi Fujita", "Raphael Rubino"],
       "year": 2021,
       "relevance": "Meta-evaluation study finding widespread methodological issues in MT research, parallel to LLM evaluation concerns."
     },
     {
       "title": "Holistic evaluation of language models",
       "authors": ["Percy Liang"],
       "year": 2023,
       "relevance": "Major LLM benchmarking framework (HELM) that takes a complementary approach to lm-eval for standardized evaluation."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Introduces LLM-as-judge evaluation and Chatbot Arena, key alternative evaluation paradigm for LLMs."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt"],
       "year": 2020,
       "relevance": "Introduces MMLU benchmark, one of the most widely used LLM evaluations, central to the paper's case studies."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B Brown"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "GPT-3 paper that established in-context learning evaluation paradigm, foundational for current LM evaluation practices."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Questions emergent abilities claims, relevant to evaluation methodology and measurement choices affecting conclusions."
     },
     {
       "title": "Beyond the imitation game: Quantifying and extrapolating the capabilities of language models",
       "authors": ["Aarohi Srivastava"],
       "year": 2022,
       "arxiv_id": "2206.04615",
       "relevance": "BIG-bench benchmark suite, major evaluation infrastructure effort complementary to lm-eval."
     }
   ]
 }