ai-research-survey

scan-v5.json (20153B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Lessons from the Trenches on Reproducible Evaluation of Language Models",
     "authors": [
       "Stella Biderman",
       "Hailey Schoelkopf",
       "Lintang Sutawika",
       "Leo Gao",
       "Jonathan Tow",
       "Baber Abbasi",
       "Alham Fikri Aji",
       "Pawan Sasanka Ammanamanchi",
       "Sidney Black",
       "Jordan Clive",
       "Anthony DiPofi",
       "Julen Etxaniz",
       "Benjamin Fattori",
       "Jessica Zosa Forde",
       "Charles Foster",
       "Jeffrey Hsu",
       "Mimansa Jaiswal",
       "Wilson Y. Lee",
       "Haonan Li",
       "Charles Lovering",
       "Niklas Muennighoff",
       "Ellie Pavlick",
       "Jason Phang",
       "Aviya Skowron",
       "Samson Tan",
       "Xiangru Tang",
       "Kevin A. Wang",
       "Genta Indra Winata",
       "François Yvon",
       "Andy Zou"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2405.14782",
     "doi": "10.48550/arXiv.2405.14782"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All four abstract claims — challenges in LM evaluation, three years of experience informing guidance, best practices, and the lm-eval library — are substantiated by Sections 2–4 and the case studies.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim — that prompt format causes score differences — is demonstrated empirically in Table 1, where switching ARC from cloze to MMLU-style yields a 22pp swing for Mistral-7B (50.1% vs. 72.4%).",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Broad assertions such as 'most works on language modeling do not perform statistical significance testing' are stated without citation to systematic survey evidence, exceeding the demonstrated scope.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses tradeoffs between loglikelihood vs. generative evaluation, and between multiple normalization schemes, presenting pros and cons of each approach rather than a single interpretation.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly states 'we do not care about the actual numeric score of a model on a benchmark' and discusses validity as the correlation between benchmark scores and real-world phenomena.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the paper ends with a brief conclusion with no formal acknowledgment of what lm-eval does not address.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Construct validity is acknowledged as 'an ongoing problem' but explicitly deferred; no specific threats to the paper's own prescriptive claims are enumerated.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly states: 'While validity is an ongoing problem in language model evaluation, we focus on mitigating other concerns first' — a clear and specific scope boundary.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed anywhere in the provided paper text.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the title page (EleutherAI, Stability AI, MBZUAI, Brown University, NYU, CMU, etc.).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so this criterion is not applicable; however, the primary authors are from EleutherAI and are presenting their own lm-eval tool without disclosing this institutional self-interest.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'The Key Problem' is formally defined (semantic equivalence challenge), 'validity' is cited to Messick (1994), and loglikelihood, perplexity, and normalization approaches are formally defined in Appendix A.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists three contributions: (1) overview of common evaluation challenges, (2) best practices, (3) the lm-eval library for reproducible benchmarking.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper engages substantively with HELM, BIG-Bench, PromptSource, and individual benchmarks (MMLU, ARC, HellaSwag), situating lm-eval's 'orchestration-first' philosophy against prescriptive alternatives.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "The paper explicitly defers construct validity: 'While validity is an ongoing problem, we focus on mitigating other concerns first.' lm-eval is a framework wrapping existing benchmarks, not a new benchmark with a validity argument.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": false,
           "answer": false,
           "justification": "lm-eval is an evaluation framework, not a new benchmark; no new benchmark items are created, making difficulty distribution characterization not applicable.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": false,
           "answer": false,
           "justification": "No new benchmark items are introduced; the paper runs existing benchmarks through a standardized framework, so ceiling/floor checks on newly created items are not applicable.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": false,
           "answer": false,
           "justification": "The paper does not create a new benchmark requiring human performance comparison; all reported results are model-vs-model comparisons on existing benchmarks.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A provides extensive formal justification for all scoring approaches (token-length normalization, byte-length normalization, mutual information), explaining when each is appropriate and their tradeoffs.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "Data contamination is not discussed as a design consideration for lm-eval; no temporal splits, canary strings, or anti-gaming measures are described in the framework's design.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.4 and Figure 1 explicitly address how benchmarks become outdated as paradigms shift (fine-tuning → in-context learning → chat), noting that most benchmarks predate current evaluation paradigms.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 2 comprehensively covers failure modes of LM evaluation: the Key Problem, prompt sensitivity, implementation divergence, API deprecation rendering work irreproducible, and benchmark-paradigm mismatch.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "lm-eval is open source with YAML configurations and Python task implementations; Appendix B shows the exact configs used for ARC cloze, ARC MMLU-style, MMLU original, and MMLU hybrid variants.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "While YAML configs are shown, there is no data card, systematic collection methodology, or preprocessing documentation for the benchmarks the framework wraps.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The paper states lm-eval is open source but does not specify its license; no explicit licensing terms appear in the paper text.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 explicitly states lm-eval's purpose is orchestration and reproducibility, and Section 2.2 specifies what it does NOT do (prescribe which benchmarks or address construct validity).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLM benchmark scores are highly sensitive to prompt format, with the same model on the same benchmark varying by 22+ percentage points depending on prompting style.",
       "evidence": "Table 1 shows Mistral-7B on ARC scores 50.1% (cloze) vs. 72.4% (MMLU-style); for MMLU, MMLU-style vs. Hybrid yields 58.6% vs. 48.3%.",
       "supported": "strong"
     },
     {
       "claim": "Copying evaluation numbers across papers is misleading because differing implementation details make comparisons invalid.",
       "evidence": "Table 1 demonstrates this empirically; Marie et al. (2021) meta-evaluation of 769 MT papers is cited as systematic evidence for the field-wide problem.",
       "supported": "strong"
     },
     {
       "claim": "Most LM evaluation papers do not perform statistical significance testing, undermining confidence in reported results.",
       "evidence": "Asserted as a best practice motivation but no systematic survey data or citation supports the 'most works' quantification.",
       "supported": "weak"
     },
     {
       "claim": "lm-eval's standardized implementations give researchers confidence that results from prior work using the library are reproducible.",
       "evidence": "Table 2 shows lm-eval adoption across 12 novel architecture papers; the Open LLM Leaderboard uses lm-eval as its backend.",
       "supported": "moderate"
     },
     {
       "claim": "Most widely-used LM benchmarks were designed before current evaluation paradigms (in-context learning, chat) and are being used for purposes they were not designed for.",
       "evidence": "Figure 1 shows benchmark release dates all predate BERT/GPT-2/GPT-3/ChatGPT shifts; paper states 'common practice diverges from the method described in the paper for all listed tasks except MMLU and MATH.'",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study",
     "theoretical"
   ],
   "key_findings": "The paper's central empirical finding is that prompt format alone causes 20+ percentage point swings in benchmark scores (e.g., Mistral-7B on ARC: 50.1% cloze vs. 72.4% MMLU-style), rendering cross-paper comparisons unreliable when implementation details differ. The root cause is identified as the 'Key Problem': no reliable automatic method exists to judge semantic equivalence of LM outputs, forcing brittle workarounds (multiple choice, regex matching) that are hypersensitive to formatting. The lm-eval library addresses this through standardized open-source task implementations with version tracking, supporting three core request types (loglikelihood, perplexity, generation) and having been adopted by 12+ novel architecture papers and the HuggingFace Open LLM Leaderboard. Construct validity — whether benchmarks actually measure real-world capabilities — is explicitly out of scope.",
   "red_flags": [
     {
       "flag": "Self-evaluation bias undisclosed",
       "detail": "The primary authors are from EleutherAI and are presenting their own lm-eval tool as the solution to evaluation problems; this institutional self-interest is not disclosed as a potential conflict."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper makes broad prescriptive claims about evaluation best practices with no formal limitations section acknowledging where those practices may not apply."
     },
     {
       "flag": "Unsupported generalization",
       "detail": "'Most works on language modeling do not perform statistical significance testing' is asserted without systematic survey evidence."
     },
     {
       "flag": "Construct validity explicitly deferred",
       "detail": "The fundamental question of whether benchmarks measure meaningful capabilities is acknowledged as 'an ongoing problem' then set aside, leaving the paper's prescriptions incomplete."
     },
     {
       "flag": "Benchmark-creation misclassification",
       "detail": "This is primarily a position/tool paper about an evaluation framework; lm-eval wraps existing benchmarks rather than creating new ones, making several benchmark-creation criteria structurally inapplicable."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring Massive Multitask Language Understanding (MMLU)",
       "relevance": "Central case study demonstrating how scoring style (MMLU-style vs. hybrid) causes large score differences on the same benchmark."
     },
     {
       "title": "Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge",
       "relevance": "Primary case study showing cloze vs. MMLU-style prompting yields 22pp swings, demonstrating the reproducibility problem."
     },
     {
       "title": "Scientific credibility of machine translation research: A meta-evaluation of 769 papers (Marie et al., 2021)",
       "relevance": "Provides systematic field-wide evidence for harm from copying results and lack of significance testing, supporting the paper's prescriptions."
     },
     {
       "title": "Holistic Evaluation of Language Models (HELM)",
       "relevance": "Related unified evaluation library; lm-eval's orchestration-first, non-prescriptive approach is explicitly contrasted with HELM."
     },
     {
       "title": "Beyond the Imitation Game (BIG-Bench)",
       "relevance": "Related prescriptive benchmark suite; lm-eval is positioned as complementary by focusing on infrastructure rather than mandating specific benchmarks."
     },
     {
       "title": "Quantifying language models' sensitivity to spurious features in prompt design (Sclar et al., 2023)",
       "relevance": "Supporting evidence that prompt formatting changes significantly affect LM performance, corroborating the paper's central claim."
     },
     {
       "title": "The benchmark lottery (Dehghani et al., 2021)",
       "relevance": "Provides evidence that benchmark choice skews comparisons and influences research directions, motivating the need for lm-eval's standardized infrastructure."
     },
     {
       "title": "A framework for few-shot language model evaluation (Gao et al., 2021)",
       "relevance": "The original lm-eval v1 release that this paper formally introduces and describes the evolution of."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "lm-eval is a widely-used open-source tool directly applicable to anyone evaluating LLMs; best practices are immediately actionable for any evaluation project."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Demonstrating that identical models on identical benchmarks can show 22pp swings from prompt format alone challenges confidence in all published leaderboard numbers."
     },
     "fear_safety": {
       "score": 1,
       "justification": "The paper briefly notes that poor evaluation can lead to 'adverse effects from deploying suboptimal or harmful models' but safety implications are not the primary focus."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly criticizes industrial labs for not sharing evaluation code and for deprecated APIs making hundreds of studies irreproducible, but the tone is constructive rather than confrontational."
     },
     "demo_ability": {
       "score": 3,
       "justification": "lm-eval is installable via pip and runnable immediately; Appendix B provides YAML configs anyone can copy and reproduce the paper's ARC/MMLU comparisons."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "EleutherAI is well-known in the open LLM community; lm-eval backs the HuggingFace Open LLM Leaderboard, giving it high visibility."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "41536003",
         "title": "The Legend of Holy Sword: An Immersive Experience for Concentration Enhancement",
         "points": 139,
         "comments": 66,
         "url": "https://news.ycombinator.com/item?id=41536003",
         "created_at": "2024-09-13T23:07:24Z"
       },
       {
         "hn_id": "39144845",
         "title": "Tweets to Citations: The Impact of Social Media Influencers on AI Research",
         "points": 67,
         "comments": 47,
         "url": "https://news.ycombinator.com/item?id=39144845",
         "created_at": "2024-01-26T16:57:45Z"
       },
       {
         "hn_id": "40474294",
         "title": "Lessons from the trenches on reproducible evaluation of language models",
         "points": 42,
         "comments": 3,
         "url": "https://news.ycombinator.com/item?id=40474294",
         "created_at": "2024-05-25T11:42:23Z"
       },
       {
         "hn_id": "42590687",
         "title": "European Space Agency Benchmark for Anomaly Detection in Satellite Telemetry",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42590687",
         "created_at": "2025-01-03T23:37:19Z"
       },
       {
         "hn_id": "41049219",
         "title": "On the Design and Analysis of LLM-Based Algorithms",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41049219",
         "created_at": "2024-07-23T18:26:56Z"
       },
       {
         "hn_id": "40467826",
         "title": "Lessons from the Trenches on Reproducible Evaluation of Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40467826",
         "created_at": "2024-05-24T16:35:27Z"
       },
       {
         "hn_id": "41230495",
         "title": "Residual Quantization with Implicit Neural Codebooks",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41230495",
         "created_at": "2024-08-12T23:23:30Z"
       }
     ],
     "top_points": 139,
     "total_points": 255,
     "total_comments": 116
   }
 }