ai-research-survey

scan.json (22986B)

---

 {
   "paper": {
     "title": "Measuring what Matters: Construct Validity in Large Language Model Benchmarks",
     "authors": [
       "Andrew M. Bean",
       "Ryan Othniel Kearns",
       "Angelika Romanou",
       "Franziska Sofia Hafner",
       "Harry Mayne",
       "Jan Batzner",
       "Negar Foroutan",
       "Chris Schmitz",
       "Karolina Korgul",
       "Hunar Batra",
       "Oishi Deb",
       "Emma Beharry",
       "Cornelius Emde",
       "Thomas Foster",
       "Anna Gausen",
       "María Grandury",
       "Simeng Han",
       "Valentin Hofmann",
       "Lujain Ibrahim",
       "Hazel Kim",
       "Hannah Rose Kirk",
       "Fangru Lin",
       "Gabrielle Kaili-May Liu",
       "Lennart Luettgau",
       "Jabez Magomere",
       "Jonathan Rystrøm",
       "Anna Sotnikova",
       "Yushi Yang",
       "Yilun Zhao",
       "Adel Bibi",
       "Antoine Bosselut",
       "Ronald Clark",
       "Arman Cohan",
       "Jakob Foerster",
       "Yarin Gal",
       "Scott A. Hale",
       "Inioluwa Deborah Raji",
       "Christopher Summerfield",
       "Philip H.S. Torr",
       "Cozmin Ududec",
       "Luc Rocher",
       "Adam Mahdi"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025 Track on Datasets and Benchmarks",
     "arxiv_id": "2511.04703",
     "doi": "10.48550/arXiv.2511.04703"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states 'the code used to clean the dataset is available on GitHub' and the complete codebook dataset is available on Hugging Face."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states 'The complete codebook is available as a dataset on Hugging Face.' The 445-paper annotation dataset is released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are mentioned for the analysis code."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a high level but there are no runnable scripts or README with commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point percentages (e.g., '78.2% of reviewed benchmarks provide definitions') without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a descriptive systematic review. It does not make comparative claims between systems that would require significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental comparisons are made. The paper reports descriptive statistics about benchmark practices."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The corpus of 445 papers results from filtering 46,114 articles, but there is no justification for why this sample size is sufficient or discussion of statistical power for the descriptive claims."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs. The paper is a systematic review with descriptive coding."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper references prior work on LLM benchmark quality (Reuel et al., Bowman & Dahl, Raji et al.) but does not formally compare its findings against prior surveys or establish baselines for comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The related work discussed is contemporary: Reuel et al. (2024), Biderman et al. (2024), Miller (2024), Weidinger et al. (2025)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. This is a systematic review."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation. The paper codes benchmark papers against a codebook."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. The 29 expert reviewers are the researchers conducting the coding, not evaluators of a system."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No prediction task requiring train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by phenomenon type (Fig 2A), year (Fig 2B), and across multiple codebook dimensions (Fig 3, §4). Detailed per-field inter-rater agreement in Table 9."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses limitations of its own methodology: low inter-rater agreement on interpretive fields (task_ecology BPK=0.146, dataset_sampling_method BPK=0.122), potential false negatives from LLM filtering, and limited reviewers per paper (§6)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that inter-rater agreement is only moderate (mean BPK=0.524), with several fields showing poor reliability. It also reports that only 16% of benchmarks use statistical tests and nearly every paper has weaknesses."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about 'patterns related to measured phenomena, tasks, and scoring metrics which undermine validity' are supported by the codebook results in §4 and Fig 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims that identified patterns 'undermine the validity of the resulting claims' — this is a causal claim. However, the evidence is descriptive (counting practices) without empirically testing whether these practices actually reduce validity of conclusions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "§6 explicitly states: 'Our focus on leading conference proceedings... may systematically exclude certain types of impactful benchmarks' including industry benchmarks and domain-specific venues."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§6 discusses alternative explanations: distributional shift in language may bias toward recent papers, LLM screening may have introduced false negatives, and limited reviewers per paper reduces robustness."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines construct validity (§2), distinguishes between their codebook measurements and the underlying quality of benchmarks, and discusses the gap between coding annotations and actual validity (inter-rater agreement analysis in Appendix D)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o mini is named with a citation [30] but no specific API version or snapshot date is provided. Per schema criteria, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for the GPT-4o mini screening is provided in Table 7 (Appendix C), including system prompt and all three filtering step prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other hyperparameters are reported for the GPT-4o mini screening calls."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. GPT-4o mini is used for simple classification screening."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full filtering pipeline is documented with counts and criteria at each stage: 46,114 → 2,189 (keyword) → 938 → 846 → 522 (LLM filtering) → 445 (manual review). Table 7 details each step. Appendix C provides the flowchart."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§6 Discussion contains two substantial paragraphs describing limitations of the approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: LLM filtering may have introduced 'undetected false negative systematic errors,' distributional shift in language usage may bias toward recent papers, 'limiting the number of reviewers per paper, reducing the robustness of the reviews' (§6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§6 explicitly states: the review 'does not capture benchmarks developed and released by industry labs without formal peer review, or those published in specialised domain-specific venues.' Also: 'We primarily review benchmarks prevalent in mainstream academic AI research.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The complete annotation dataset is released on Hugging Face (Appendix B). This allows independent verification of the reported statistics."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§3 describes keyword search across 6 conferences, time range (2018-2024 for ML, 2020-2024 for NLP), keyword criteria ('benchmark' AND 'LLM' OR 'language model'), and LLM + manual filtering pipeline."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The 29 expert reviewers are mentioned as being 'matched on area of expertise' but no details are provided on how they were recruited, their expertise levels, or selection criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: 46,114 articles → 2,189 (keyword) → 522 (LLM filtering with counts at each stage in Table 7) → 445 (manual review). Appendix C provides detailed flowchart (Fig 4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Detailed Acknowledgments and Disclosure of Funding section lists grants and funding sources for individual authors (Clarendon, EPSRC, ESRC, NSF, Swiss NSF, etc.)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All 42 author affiliations are listed, spanning 14 institutions (Oxford, EPFL, Yale, UK AI Security Institute, etc.). No author evaluates their own company's product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are academic bodies (Clarendon Scholarship, EPSRC, ESRC, NSF, Swiss NSF, Royal Society) with no financial stake in the review outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. One author is affiliated with UK AI Security Institute which could have policy interests, but this is not discussed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic review of papers, not an evaluation of a pre-trained model on a benchmark. The GPT-4o mini screening is auxiliary and not the subject of the study's claims."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Systematic review; no pre-trained model evaluated on benchmark tasks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Systematic review; no benchmark evaluation of model capabilities."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The 29 reviewers are co-investigators conducting the coding, not human subjects. The unit of analysis is benchmark papers."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Reviewers are researchers, not research subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper. The GPT-4o mini screening cost is not the paper's method under evaluation."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper. No significant compute to report."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The review follows a structured systematic protocol: keyword search → LLM filtering → manual expert review, documented with a PRISMA-style flowchart (Fig 1, Fig 4). Inclusion/exclusion criteria are explicitly stated with counts at each stage."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper IS a quality assessment of its sources. The codebook with 21 question items across phenomena, tasks, metrics, and claims constitutes a structured quality evaluation of each of the 445 included benchmarks."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§6 acknowledges venue selection bias: 'Our focus on leading conference proceedings... may systematically exclude certain types of impactful benchmarks' including industry benchmarks and domain-specific venues."
       }
     }
   },
   "claims": [
     {
       "claim": "47.8% of benchmark definitions are contested, addressing phenomena with many possible definitions or no clear definition at all.",
       "evidence": "Codebook results in §4 Phenomenon section and Fig 3. 52.2% of definitions widely agreed upon, 47.8% contested among those that provide definitions.",
       "supported": "strong"
     },
     {
       "claim": "Only 16.0% of reviewed benchmarks used uncertainty estimates or statistical tests to compare results.",
       "evidence": "§4 Metric section: 'Once the responses were scored, 16.0% used uncertainty estimates or statistical tests to compare the results.' Also shown in Fig 3.",
       "supported": "strong"
     },
     {
       "claim": "27.0% of reviewed benchmarks incorporated convenience sampling as part of their sampling strategy, with 12.3% using it exclusively.",
       "evidence": "§4 Task section reports these figures directly from the codebook annotations.",
       "supported": "strong"
     },
     {
       "claim": "Nearly every paper had weaknesses in at least one area across phenomena, tasks, metrics, and claims.",
       "evidence": "§6 Discussion states this based on the full annotation dataset. Fig 3 shows that the intersection of all best practices (shaded area) is very small.",
       "supported": "strong"
     },
     {
       "claim": "Only 53.4% of articles presented evidence for the construct validity of their benchmark.",
       "evidence": "§4 Claims section and Fig 3.",
       "supported": "strong"
     },
     {
       "claim": "LLM-generated task items are used in 31.2% of benchmarks, with potential non-representative sampling risks.",
       "evidence": "§4 Task section. Sampling concerns discussed in §5.3.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "A systematic review of 445 LLM benchmark papers from top ML/NLP conferences reveals pervasive construct validity weaknesses. 47.8% of benchmarks have contested phenomenon definitions, only 16% use statistical tests, 27% rely on convenience sampling, and only 53.4% discuss their own construct validity. The authors provide eight recommendations and an operational checklist covering phenomenon definition, task design, contamination, statistical methods, error analysis, and validity justification. Inter-rater agreement among the 29 expert reviewers was moderate (mean Brennan-Prediger Kappa = 0.524).",
   "red_flags": [
     {
       "flag": "Moderate inter-rater reliability",
       "detail": "Mean BPK of 0.524 across 30 codebook fields indicates only moderate agreement. Several key fields have very low reliability: task_ecology (BPK=0.146), dataset_sampling_method (BPK=0.122), phenomenon_contested (BPK=0.317). This undermines confidence in the specific percentages reported."
     },
     {
       "flag": "Single reviewer per paper",
       "detail": "Each of the 445 papers was coded by a single primary reviewer, with a second reviewer only mapping responses. Only 46 papers (10%) were double-reviewed. Acknowledged as a limitation but still affects reliability of individual annotations."
     },
     {
       "flag": "LLM screening may introduce systematic bias",
       "detail": "GPT-4o mini was used to filter from 2,189 to 522 papers. Validated on only 50 papers (2.3% sample) achieving 80% precision and 89% recall. False negatives from this step are unrecoverable. The authors acknowledge this but the validation sample is small."
     }
   ],
   "cited_papers": [
     {
       "title": "AI and the Everything in the Whole Wide World Benchmark",
       "authors": ["I. D. Raji", "E. M. Bender", "A. Paullada", "E. Denton", "A. Hanna"],
       "year": 2021,
       "relevance": "Foundational work on benchmark definitions and limitations that this paper builds upon for its construct validity framework."
     },
     {
       "title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
       "authors": ["A. Reuel"],
       "year": 2024,
       "relevance": "Aggregates best practices for the benchmark lifecycle; this paper extends with construct validity focus."
     },
     {
       "title": "What Will It Take to Fix Benchmarking in Natural Language Understanding?",
       "authors": ["S. R. Bowman", "G. Dahl"],
       "year": 2021,
       "relevance": "Emphasizes the value of construct validity in NLP benchmarks and the need for shared evaluation standards."
     },
     {
       "title": "Lessons from the Trenches on Reproducible Evaluation of Language Models",
       "authors": ["S. Biderman"],
       "year": 2024,
       "relevance": "Addresses reproducibility and shared implementation standards for LLM evaluation."
     },
     {
       "title": "Inadequacies of Large Language Model Benchmarks in the Era of Generative Artificial Intelligence",
       "authors": ["T. R. McIntosh"],
       "year": 2024,
       "relevance": "Identifies inadequacies in LLM benchmarks, directly relevant to methodological quality assessment."
     },
     {
       "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
       "authors": ["E. Miller"],
       "year": 2024,
       "relevance": "Proposes statistical methods for LLM evaluation, addressing the paper's finding that only 16% of benchmarks use statistical tests."
     },
     {
       "title": "Toward an Evaluation Science for Generative AI Systems",
       "authors": ["L. Weidinger"],
       "year": 2025,
       "relevance": "Calls for shared standards and best practices in LLM evaluation science."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["X. Liu"],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLMs as agents, cited as example of composite phenomenon measurement."
     },
     {
       "title": "Datasheets for Datasets",
       "authors": ["T. Gebru"],
       "year": 2021,
       "relevance": "Proposes documentation standards for datasets relevant to benchmark transparency and social responsibility."
     },
     {
       "title": "Don't Make Your LLM an Evaluation Benchmark Cheater",
       "authors": ["K. Zhou"],
       "year": 2023,
       "relevance": "Addresses benchmark contamination, a key concern in the construct validity of LLM evaluations."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"],
       "year": 2023,
       "relevance": "Questions interpretation of emergent abilities, illustrating contested phenomena in LLM evaluation."
     },
     {
       "title": "A Careful Examination of Large Language Model Performance on Grade School Arithmetic",
       "authors": ["H. Zhang"],
       "year": 2024,
       "relevance": "Demonstrates contamination effects on GSM8K benchmark, used as example in the paper's recommendations."
     }
   ]
 }