ai-research-survey

scan.json (22789B)

---

 {
   "paper": {
     "title": "The Benchmarking Epistemology: Construct Validity for Evaluating Machine Learning Models",
     "authors": ["Timo Freiesleben", "Sebastian Zezulka"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.23191"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or analysis scripts are mentioned or linked anywhere in the paper. The paper is theoretical but could have released supplementary materials."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or supplementary data is released. The paper analyzes existing benchmarks (ImageNet, WeatherBench, Fragile Families Challenge) through a conceptual framework but does not release any structured data from this analysis."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical/philosophical paper with no computational experiments, so environment specifications are structurally inapplicable."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical/philosophical paper with no experiments to reproduce. The contribution is a conceptual framework and case study analyses."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not report its own experimental results. It references confidence intervals from other studies (e.g., Russakovsky et al. 2015 bootstrap-based CIs) but does not produce original quantitative results."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative empirical claims are made by the authors. The paper is a theoretical analysis with case studies drawn from existing literature."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original empirical results are reported. Effect sizes referenced (e.g., ImageNet error rate drops, FFC scores) come from cited studies."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper that does not collect data or run experiments, so sample size justification is structurally inapplicable."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments are run, so variance reporting is structurally inapplicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper positions its framework relative to prior validity-based perspectives on benchmarks, including Yee (2024), Salaudeen et al. (2025), Schlangen (2021), and Raji et al. (2021). Section 3 explicitly discusses how existing concerns in the literature map onto their framework."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The related work is contemporary: Salaudeen et al. (2025), Ye et al. (2025), Suhr et al. (2025), Yee (2024), and the classical Messick (1995) framework. These represent the current state of thinking on benchmark validity."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical framework paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted, so evaluation metrics are structurally inapplicable."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper that does not produce system outputs requiring human evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted, so held-out test sets are structurally inapplicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The framework is demonstrated through three distinct case studies (ImageNet, WeatherBench, Fragile Families Challenge), each examining different types of validity conditions (content, consequential, auxiliary). Table 1 provides a structured breakdown of five validity types with their conditions and supported inferences."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Each case study extensively discusses where validity conditions fail. For ImageNet: adaptivity violations, distribution sensitivity, content validity limitations (Section 4.3). For WeatherBench: lack of consequential validity, inability to capture butterfly effect (Section 5.3). For FFC: auxiliary validity failures, unmeasured predictors (Section 6.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central contribution includes identifying when benchmark inferences are NOT valid. Each case study concludes with a 'Constraining the inference' section that explicitly states what cannot be inferred (Sections 4.3, 5.3, 6.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the paper develops construct validity conditions inspired by psychological measurement theory and examines them through three case studies. Sections 2-3 develop the framework, and Sections 4-6 present the three case studies as claimed. The abstract accurately reflects the paper's content."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It develops a conceptual framework for interpreting benchmark scores and analyzes case studies using logical argumentation rather than causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about scope. Section 7.1 acknowledges that the validity conditions 'remain incomplete' especially for LLM evaluation. Each case study concludes by explicitly constraining what can and cannot be inferred. The framework is presented as a structured approach, not as a universal solution."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is central to the paper's methodology. The FFC case study (Section 6) is entirely organized around ruling out alternative explanations for benchmark results. Section 6.2 discusses adaptivity vs. distribution shift for ImageNet, and Lundberg et al. (2024) qualitative interview findings about unmeasured predictors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No ML models are used or evaluated in this paper. It is a theoretical analysis."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting or LLM usage is involved in this paper's methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted, so hyperparameter reporting is structurally inapplicable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a theoretical paper."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data is collected or preprocessed. The paper analyzes existing benchmarks through a conceptual lens."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Towards a benchmarking epistemology') serves as both a discussion and limitations section. Section 7.1 explicitly states the framework's validity conditions 'remain incomplete' for LLM evaluation. Section 7.3 discusses the limits of benchmarking as an epistemic practice."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 identifies specific gaps: the framework does not fully address convergent and discriminant validity for complex constructs like LLM capabilities. Section 7.3 discusses construct underrepresentation and construct irrelevance as specific risks. These are specific to this framework, not boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 explicitly states that 'the proposed validity conditions remain incomplete' and that LLM capability assessment 'calls for additional conditions.' Section 7.2 acknowledges the paper focuses on epistemic roles of benchmarks and deliberately leaves aside their 'equally important social role.' Section 7.3 discusses what cannot be meaningfully measured by benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper that does not collect or analyze raw data. The case studies analyze existing published benchmarks."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No data is collected. The paper is a theoretical/philosophical contribution."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants are recruited. This is a theoretical paper."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline exists. This is a theoretical paper analyzing existing benchmarks."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section discloses funding from the Deutsche Forschungsgemeinschaft (DFG) under Germany's Excellence Strategy, the Carl Zeiss Stiftung, and the International Max Planck Research School for Intelligent Systems (IMPRS-IS)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Timo Freiesleben at LMU Munich (MCMP & MCML), Sebastian Zezulka at University of Tubingen. No conflicts with evaluated products are apparent since the paper does not evaluate any specific commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (DFG, Carl Zeiss Stiftung, IMPRS-IS) are academic research funding bodies with no financial stake in the paper's conclusions about benchmark validity."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on any benchmark. It is a theoretical analysis of benchmarking as an epistemic practice."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is performed. The paper discusses train/test overlap as a validity concern conceptually (Section 4.2, internal validity) but does not itself face this issue."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. Contamination is discussed conceptually as part of internal validity conditions but is not a concern for this paper's own methodology."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper with no computational experiments, so inference cost is structurally inapplicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper with no computational experiments, so compute budget is structurally inapplicable."
       }
     }
   },
   "claims": [
     {
       "claim": "Predictive benchmarks are best understood as measurement tools analogous to psychological and educational tests, and interpreting benchmark scores requires specifying and evaluating conditions of construct validity.",
       "evidence": "Section 2 develops the analogy between benchmark components (data instances, models, scores) and psychological test components (test items, test takers, scores). Section 3 adapts the four-step argument-based framework for construct validity from Messick (1995) and Kane (2013). Table 1 provides a structured summary of five validity types with conditions and supported inferences.",
       "supported": "strong"
     },
     {
       "claim": "ImageNet benchmark scores cannot be interpreted as indicating expected error rates on image classification in general, but can support the weaker inference that model rankings reflect relative progress.",
       "evidence": "Section 4 analyzes internal, external, and content validity for ImageNet. External validity evidence shows robust rank preservation across datasets (Kornblith et al. 2019, r=0.86 for large datasets) but 11-14% absolute performance drops on reproduced data (Recht et al. 2019). Content validity limitations include 120/1000 classes being dog breeds and 20% of images having multiple plausible labels (Shankar et al. 2020). Section 4.3 constrains the inference accordingly.",
       "supported": "strong"
     },
     {
       "claim": "WeatherBench scores cannot be directly used to reflect utility in downstream applications such as energy market planning, despite strong internal and content validity.",
       "evidence": "Section 5 identifies that default WeatherBench evaluation metrics do not capture context-specific utilities. GraphCast produces overly smoothed forecasts (Lam et al. 2023), rare events are under-penalized (Rasp et al. 2024), and models are evaluated on point predictions rather than uncertainty estimates needed in practice. Section 5.3 constrains the inference.",
       "supported": "strong"
     },
     {
       "claim": "The uniformly low scores in the Fragile Families Challenge cannot support the strong inference that life outcomes are inherently unpredictable, because auxiliary validity conditions (model diversity, data completeness) are not fully met.",
       "evidence": "Section 6 analyzes auxiliary validity. Lundberg et al. (2024) interviewed 40 families and identified three sources of prediction error: imperfectly measured features, unmeasured features, and unmeasurable features (events after survey). The small sample size (4,242 families, 12,942 features) limits model fitting. Section 6.3 constrains to the weaker inference that predictive performance is likely to be low, especially for young and marginalized groups.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["theoretical"],
   "key_findings": "The paper develops a construct validity framework for interpreting ML benchmark scores, adapted from psychological measurement theory. It identifies five types of validity (internal, external, content, consequential, auxiliary) with specific conditions for each. Through three case studies -- ImageNet, WeatherBench, and the Fragile Families Challenge -- it demonstrates that benchmarks support inferences only to the degree that corresponding validity conditions are met. The framework shows that stronger inferences (e.g., about model capabilities or real-world deployment) require progressively more validity conditions to be satisfied.",
   "red_flags": [],
   "cited_papers": [
     {
       "title": "Measurement to Meaning: A Validity-Centered Framework for AI Evaluation",
       "authors": ["Olawale Salaudeen", "Anka Reuel", "Ahmed Ahmed", "Suhana Bedi", "Zachary Robertson", "Sudharsan Sundar", "Ben Domingue", "Angelina Wang", "Sanmi Koyejo"],
       "year": 2025,
       "relevance": "Proposes a claim-centred framework for evaluating general-purpose AI systems using validity concepts, directly related to this survey's concern with methodological rigor in AI evaluation."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Demonstrates how metric choice can create the appearance of emergent abilities, highlighting construct validity issues in LLM evaluation."
     },
     {
       "title": "AI and the Everything in the Whole Wide World Benchmark",
       "authors": ["Deborah Raji", "Emily Denton", "Emily M. Bender", "Alex Hanna", "Amandalynne Paullada"],
       "year": 2021,
       "relevance": "Argues that only well-specified intensional tasks can be faithfully operationalized by benchmarks, raising fundamental questions about benchmark validity for general-purpose AI."
     },
     {
       "title": "Stop Evaluating AI with Human Tests, Develop Principled, AI-specific Tests instead",
       "authors": ["Tom Suhr", "Florian E. Dorner", "Olawale Salaudeen", "Augustin Kelava", "Samira Samadi"],
       "year": 2025,
       "relevance": "Challenges the practice of using human psychological tests for AI evaluation, proposing AI-specific testing methodologies."
     },
     {
       "title": "The Benchmark Lottery",
       "authors": ["Mostafa Dehghani", "Yi Tay", "Alexey A. Gritsenko", "Zhe Zhao", "Neil Houlsby", "Fernando Diaz", "Donald Metzler", "Oriol Vinyals"],
       "year": 2021,
       "relevance": "Documents how benchmark selection can favor methods already aligned with benchmark idiosyncrasies, a key concern for evaluation methodology."
     },
     {
       "title": "Reduced, Reused and Recycled: The Life of a Dataset in Machine Learning Research",
       "authors": ["Bernard Koch", "Emily Denton", "Alex Hanna", "Jacob G. Foster"],
       "year": 2021,
       "relevance": "Shows concentration of research on a small set of benchmark datasets produced by few institutions, raising concerns about bias and evaluation methodology."
     },
     {
       "title": "Do ImageNet Classifiers Generalize to ImageNet?",
       "authors": ["Benjamin Recht", "Rebecca Roelofs", "Ludwig Schmidt", "Vaishaal Shankar"],
       "year": 2019,
       "relevance": "Empirically tests external validity of ImageNet by recreating the dataset, finding 11-14% performance drops but stable rankings."
     },
     {
       "title": "Targeting the Benchmark: On Methodology in Current Natural Language Processing Research",
       "authors": ["David Schlangen"],
       "year": 2021,
       "relevance": "Introduces the key distinction between intensional task description and extensional benchmark instantiation, foundational for benchmark validity analysis."
     },
     {
       "title": "Are We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning",
       "authors": ["Thomas Liao", "Rohan Taori", "Inioluwa Deborah Raji", "Ludwig Schmidt"],
       "year": 2021,
       "relevance": "Meta-review documenting evaluation failures in ML research, directly relevant to methodological quality assessment."
     },
     {
       "title": "Position: Why We Must Rethink Empirical Research in Machine Learning",
       "authors": ["Moritz Herrmann", "F. Julian D. Lange", "Katharina Eggensperger", "Giuseppe Casalicchio", "Marcel Wever", "Matthias Feurer", "David Rugamer", "Eyke Hullermeier", "Anne-Laure Boulesteix", "Bernd Bischl"],
       "year": 2024,
       "relevance": "Position paper arguing for rethinking empirical research methodology in ML, showing task-specific performance variation concerns."
     },
     {
       "title": "Evaluating General-Purpose AI with Psychometrics",
       "authors": ["Xiting Wang", "Liming Jiang", "Jose Hernandez-Orallo", "David Stillwell", "Luning Sun", "Fang Luo", "Xing Xie"],
       "year": 2023,
       "relevance": "Proposes using psychometric methods to evaluate AI capabilities, closely related to construct validity framework for AI benchmarks."
     },
     {
       "title": "Large Language Model Psychometrics: A Systematic Review of Evaluation, Validation, and Enhancement",
       "authors": ["Haoran Ye", "Jing Jin", "Yuhang Xie", "Xin Zhang", "Guojie Song"],
       "year": 2025,
       "relevance": "Systematic review of psychometric evaluation approaches for LLMs, directly relevant to methodological quality in AI evaluation."
     }
   ]
 }