ai-research-survey

scan.json (30892B)

---

 {
   "paper": {
     "title": "Rigor, Reliability, and Reproducibility Matter: A Decade-Scale Survey of 572 Code Benchmarks",
     "authors": [
       "Jialun Cao",
       "Yuk-Kit Chan",
       "Zixuan Ling",
       "Wenxuan Wang",
       "Shuqing Li",
       "Mingwei Liu",
       "Ruixi Qiao",
       "Yuting Han",
       "Chaozheng Wang",
       "Boxi Yu",
       "Pinjia He",
       "Shuai Wang",
       "Zibin Zheng",
       "Michael R. Lyu",
       "Shing-Chi Cheung"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2501.10711",
     "doi": "10.48550/arXiv.2501.10711"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "qualitative"],
   "key_findings": "A decade-scale survey of 572 code benchmarks (2014–2025) reveals that despite growing awareness of benchmark quality, the absolute number of flawed benchmarks continues to rise. Key deficiencies: 48.1% lack quality assurance checks, 82.5% ignore data contamination, 84.2% don't ensure test coverage, and 64% don't repeat experiments. A human study of 49 researchers confirms awareness gaps (16% unaware of data denoising needs, 40%+ unaware of environment impact on reproducibility). The paper introduces HOW2BENCH, a 55-criteria lifecycle checklist for code benchmark development.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or analysis scripts are provided in the paper. The paper provides the HOW2BENCH checklist in Appendix G and benchmark lists in Appendices E–F, but no downloadable code or analysis tools."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The full list of 572 benchmarks is provided in Appendix F and the 30 focused benchmarks in Appendix E, but these are inline in the paper only. No downloadable dataset, repository, or supplementary data file is referenced."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. As a survey, the paper could have specified the tools and environment used for data collection and analysis, but does not."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The survey methodology is described at a high level (Section 3.2, Figure 2), but there are no step-by-step instructions for reproducing the profiling or analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All statistics are reported as point estimates (e.g., '48.1%', '82.5%', '84.2%'). No confidence intervals or error bars are provided for any of the survey findings."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes trend claims (e.g., 'the number of benchmarks that ignore code coverage when providing test cases nearly matches the total count accumulated across the previous ten years') without any significance tests. No p-values, chi-squared tests, or trend tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports proportions with denominators providing context (e.g., '200/572 = 34.97%', '210/572 = 36.7%', '48/485 = 9.9%'). Trend comparisons give both raw counts and percentages across years (e.g., 'doubled from 60 in 2024 to 159 in 2025')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The survey covers 572 benchmarks and the human study includes 49 participants, but neither sample size is formally justified. No power analysis or representativeness argument is given for the 572 benchmarks or the 49 human participants."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All statistics are single-point percentages or counts without uncertainty quantification."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses related surveys (BetterBench, Koohestani et al., Chang et al.) in Section 2.2 and differentiates its scope, but does not systematically compare its findings against prior surveys' results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No systematic baseline comparison is conducted, so contemporaneity cannot be assessed. The related works discussed (BetterBench 2024, Koohestani et al. 2025) are recent but not used as empirical baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/position paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper assesses benchmarks across multiple dimensions organized by lifecycle phases: design (4 criteria), construction (19 criteria), evaluation (12 criteria), analysis (10 criteria), and release (10 criteria), with statistics on deduplication, contamination, code coverage, repeatability, open-sourcing, and many more."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Appendix D describe a human study with 49 participants who evaluated the importance and practicality of HOW2BENCH's 55 criteria via questionnaires."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey study — there is no training/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: by year (Figures 8, 10, 12, 16, 21, 24, 27, 29, 33, 36, 43, 47, 49), by coding task (Figure 9), by programming language (Figure 11), by granularity (Figure 15), and by individual quality criteria throughout Section 4 and Appendix C."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple specific failure cases are discussed: MBPP duplicated subjects (Figure 22), MBPP unexecutable code (Figure 25), MBPP incorrect tests (Figure 34), MBPP out-of-capability case (Figure 18), HumanEval incorrect ground truth (Figure 30), CruxEval unclear presentation (Figure 45), XSemPLR API key leakage (Figure 55), CrossVul name/email leakage (Figure 56)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The central finding is negative: despite growing awareness, flawed benchmarks continue to rise. The paper reports many negative findings: 48.1% no QA, 82.5% no contamination handling, 84.2% no test coverage, 64% no repeated experiments, 38.8% no prompts released, etc."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported by specific statistics in the body: '48.1% benchmarks did not go through quality assurance check' (Section 4.2, Figure 23), '82.5% did not consider data contamination' (Section 4.2, Figure 26), '84.2% did not ensure a reliable judgement' (Section 4.2, Figure 32), '64.0% benchmark evaluation was one-pass' (Section 4.3, Figure 42), '38.8% did not provide essential information for reproducibility' (Section 4.5, Figure 48)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'the current issues not only stem from the significant effort required, but also from a lack of awareness regarding their importance' (Section 1, Section 6.2). This causal claim is based on a cross-sectional questionnaire of 49 participants, which cannot establish causation. The study design (descriptive survey + questionnaire) is inadequate for causal inference about why benchmarks are flawed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'Most criteria listed in HOW2BENCH can be adopted or adapted to other benchmarks such as Question-answering, mathematical reasoning, and multi-modal benchmarks' (Section 1, Contributions) and the Impact Statement claims 'Broader Influence on Machine Learning Evaluation.' These generalizations beyond code benchmarks are not empirically tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses the trade-off between rigor and development efficiency as an alternative explanation. The 'Alternative View' subsection in Section 1 acknowledges that 'practical constraints such as time, budget, and human resources must also be considered.' Section 6.2 discusses multiple factors behind the awareness-action gap."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims are appropriately scoped to what it measures. It reports checklist compliance rates (e.g., '48.1% did not go through quality assurance check') and frames these as specific criterion violations rather than as holistic quality scores. The paper argues these criteria matter (citing prior work showing bugs in benchmarks, contamination effects, etc.) without conflating compliance with overall quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use any LLMs for its analysis."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no LLM experiments requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a 4-step study design (Section 3.2, Figure 2) including collection via snowballing and profiling, but does not document specific filtering criteria, stage-by-stage counts, or how many candidate benchmarks were screened before arriving at 572. The pipeline from initial search to final 572 benchmarks is not documented with intermediate counts or explicit inclusion/exclusion criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion' serves as a limitations section. Section 6.1 discusses 'Trade-offs Between Benchmark Rigor and Development Efficiency' and Section 6.2 discusses 'Awareness and Action.' The 'Alternative View' subsection in Section 1 also acknowledges practical constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The discussion sections address the phenomenon being studied (trade-offs in benchmark development) rather than threats to the study's own validity. There is no discussion of potential biases in their benchmark selection, inter-rater reliability of their profiling process, or representativeness of their 49-person human study sample."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper states its scope as 'code-related benchmarks' from '2014–2025,' it does not explicitly state what it does NOT cover or what claims it is NOT making. No specific exclusion criteria or boundary conditions are articulated (e.g., which types of code benchmarks were excluded and why)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The per-benchmark profiling data (whether each of 572 benchmarks meets each criterion) is not available for download. Only aggregated statistics are reported in the paper. The raw annotations cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 Step 2 describes the collection procedure: 'collecting related benchmarks according to their publication time, venue, and coding tasks, and then employing techniques like snowballing to ensure a comprehensive collection.' The profiling process is also described: 'profiling each selected benchmark through a thorough review of corresponding papers and examination of the released artifacts or homepages.'"
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 describes human study participant selection: 'we chose graduate students from SE or AI fields who have published at least one paper.' It also notes targeting individuals who have published on code benchmarks. Section D.3 describes distribution via 'online platforms, targeting academic and professional networks related to SE and AI' with start and end dates (October 27 – November 27, 2024)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from initial benchmark collection to the final 572 is not documented with stage counts. The paper states they used snowballing and scoped by time/venue/tasks but does not show how many candidate benchmarks were identified initially, how many were screened out, or at what stages filtering occurred."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the provided paper text. The paper does not disclose any funding information."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: HKUST, CUHK, Renmin University of China, Sun Yat-Sen University, Chinese Academy of Sciences, Beijing Language and Culture University, CUHK Shenzhen. All are academic institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. Without knowing who funded this work, this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark. (The paper discusses contamination as a quality criterion for benchmarks it surveys, but this question asks about contamination of the paper's own evaluation.)"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human study (questionnaire with 49 participants). No link to OSF, AsPredicted, or any registry is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human study involving 49 participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Participant demographics are reported: field distribution (AI 42.6%, SE 57.14%, Figure 61), geographic distribution across seven regions including US, UK, Germany, Australia, China (Figure 60), and experience (all published at least one paper, half had constructed code benchmarks)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 states inclusion criteria: 'graduate students from SE or AI fields who have published at least one paper.' Exclusion: 'responses from those selecting No to having published a paper were excluded. Also, incomplete surveys where not all questions were answered were also considered invalid and excluded.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional questionnaire study, not an experimental study with treatment/control conditions. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional questionnaire study, not an experimental study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attrition is reported in Appendix D.4: 'In total, we collected 50 responses... Only one survey was invalid due to the respondent selecting have not published a paper, leaving 49 valid surveys for analysis.'"
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/position paper with no computational method whose cost would need reporting."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/position paper with no significant computation requiring budget disclosure."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a structured 4-step workflow (Figure 2: guideline construction → literature profiling → focused case study → human study) but does not follow PRISMA or reference any established systematic review protocol. No reproducible search queries, database selections, or PRISMA flow diagram are provided."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "The paper introduces HOW2BENCH with 55 criteria and systematically profiles 572 benchmarks against quality dimensions (deduplication, contamination, test coverage, QA checks, etc.). The 30 focused case study benchmarks (Appendix E) are assessed against the full checklist. This constitutes structured quality assessment of source benchmarks."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias — whether published benchmarks are systematically different from unpublished ones, or whether benchmark papers with positive framing are more likely to be published. No funnel plots or publication bias tests are used."
       }
     }
   },
   "claims": [
     {
       "claim": "48.1% of benchmarks did not go through quality assurance checks; 82.5% did not consider data contamination; 62% (actually 65.7%) did not deduplicate data points.",
       "evidence": "Survey of 572 code benchmarks documented in Section 4.2 (Construction) with statistics in Figures 20, 23, 26 and Appendix C.3.",
       "supported": "strong"
     },
     {
       "claim": "Despite rising awareness of benchmark quality, the absolute number of flawed benchmarks has continued to grow.",
       "evidence": "Longitudinal trend data across 2014–2025 (Figures 21, 24, 27, 33, 43). For example, benchmarks ignoring code coverage: 19 (2023), 62 (2024), 102 (2025). Benchmarks without repeated experiments rose to 169 in 2025.",
       "supported": "strong"
     },
     {
       "claim": "84.2% of benchmarks did not ensure reliable judgment (e.g., code coverage when test suites are provided); 64.0% of evaluations were one-pass without repeating experiments.",
       "evidence": "Section 4.2 (Figure 32) shows 84.2% did not consider test coverage. Section 4.3 (Figure 42) shows 64.0% did not repeat experiments.",
       "supported": "strong"
     },
     {
       "claim": "38.8% of benchmarks did not provide essential information (e.g., prompts) for reproducibility; 12.4% are not open source.",
       "evidence": "Section 4.5 with Figures 46 and 48. 38.8% did not provide prompts, 12.4% not open-sourced, 2.6% only partially released.",
       "supported": "strong"
     },
     {
       "claim": "16% of participants were unaware of the necessity for data denoising; over 40% were not aware that experimental setup and environment could impact reproducibility.",
       "evidence": "Human study with 49 participants (Section 5, Appendix D). Questionnaire responses analyzed with results shown in Figures 62–63.",
       "supported": "moderate"
     },
     {
       "claim": "All participants agreed that having a checklist for benchmark construction would contribute to quality; 47/55 criteria deemed important by over 80% of participants.",
       "evidence": "Section 5 reports: 'all participants agreed that having a checklist for benchmark construction would contribute to the quality of the benchmark. 47/55 criteria in HOW2BENCH are deemed important by more 80% participants.'",
       "supported": "moderate"
     },
     {
       "claim": "Project-level benchmarks nearly tripled from 2024 to 2025 (30 to 87), indicating growing focus on real-world applicability.",
       "evidence": "Section 4.1 with Figure 16 showing the surge in project-level benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "HumanEval is the most significant source benchmark, benefiting at least 15 downstream benchmarks.",
       "evidence": "Appendix C.1 with Figure 59 showing benchmark inheritance relationships.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Small human study sample",
       "detail": "The human study has only 49 participants (50 collected, 1 invalid). Claims about awareness gaps in the research community are based on this small, potentially non-representative sample recruited through convenience sampling via online platforms. No power analysis or representativeness argument is provided."
     },
     {
       "flag": "No inter-rater reliability reported",
       "detail": "The profiling of 572 benchmarks against quality criteria appears to involve subjective judgment (e.g., whether a benchmark 'considered' contamination, whether quality assurance was adequate). The paper does not report how many raters assessed each benchmark, what the inter-rater agreement was, or how disagreements were resolved."
     },
     {
       "flag": "No systematic review protocol",
       "detail": "Despite surveying 572 benchmarks, the paper does not follow PRISMA or any established systematic review protocol. The collection methodology (snowballing, scoping by time/venue/tasks) is described at a high level without reproducible search queries, inclusion/exclusion criteria with stage counts, or database selections."
     },
     {
       "flag": "Raw profiling data not released",
       "detail": "The per-benchmark assessments for all 572 benchmarks are not available for independent verification. Only aggregated percentages are reported, making it impossible to check individual benchmark classifications or reproduce the analysis."
     },
     {
       "flag": "Self-referential validation",
       "detail": "HOW2BENCH is validated by asking 49 researchers whether a checklist for benchmark construction would be useful — which almost tautologically receives agreement. The validation does not test whether benchmarks scoring higher on HOW2BENCH criteria actually produce more reliable evaluation results."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "relevance": "Major code benchmark for evaluating LLMs on real-world software engineering tasks; used as motivating example throughout the paper."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval, the most influential code generation benchmark per the paper's analysis (benefits 15+ downstream benchmarks)."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "relevance": "Introduces MBPP benchmark; used as primary case study for demonstrating quality issues (duplicated subjects, incorrect tests, unexecutable code)."
     },
     {
       "title": "Is your code generated by chatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Found that HumanEval and MBPP contain bugs in implementation, documentation, and test cases; directly motivates the paper's concern about benchmark quality."
     },
     {
       "title": "Concerned with data contamination? Assessing countermeasures in code language model",
       "authors": ["Jialun Cao", "Wuqi Zhang", "Shing-Chi Cheung"],
       "year": 2024,
       "arxiv_id": "2403.16898",
       "relevance": "Addresses data contamination threats in code LLMs, one of the key quality dimensions assessed in HOW2BENCH."
     },
     {
       "title": "BetterBench: Assessing AI benchmarks, uncovering issues, and establishing best practices",
       "authors": ["Anka Reuel", "Amelia Hardy", "Colin Smith", "Max Lamparth", "Malcolm Hardy", "Mykel J. Kochenderfer"],
       "year": 2024,
       "arxiv_id": "2411.12990",
       "relevance": "Most closely related work: assesses 24 AI benchmarks against 46 criteria. HOW2BENCH differentiates by focusing on code benchmarks at much larger scale (572 vs 24)."
     },
     {
       "title": "Benchmarking AI models in software engineering: A review, search tool, and unified approach for elevating benchmark quality",
       "authors": ["Reza Koohestani", "Pieter de Bekker", "Burak Koç", "Maliheh Izadi"],
       "year": 2025,
       "relevance": "Concurrent work reviewing AI benchmarks in software engineering, representing the growing interest in benchmark quality assessment."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free benchmark design using temporal splits, directly relevant to the paper's analysis of contamination handling in benchmarks."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2024,
       "relevance": "Cited as part of the review methodology input and as an example of benchmark evaluation practices."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Major code generation benchmark used in the focused case study; cited as positive example for documentation quality (Figure 58)."
     },
     {
       "title": "The bitter lesson learned from 2,000+ multilingual benchmarks",
       "authors": ["Minghao Wu", "Weiwei Wang", "Shiyao Liu"],
       "year": 2025,
       "relevance": "Related work on benchmark quality at scale, providing guidelines for evaluation practices."
     },
     {
       "title": "Benchmark^2: Systematic evaluation of LLM benchmarks",
       "authors": ["Qianqi Qian", "Chenghua Huang", "Jiangzhuo Xu"],
       "year": 2026,
       "relevance": "Concurrent meta-evaluation work assessing LLM benchmarks, representing the growing field of benchmark quality assessment."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "HOW2BENCH is a directly usable 55-item checklist for anyone building or reviewing code benchmarks, with a printable version in the appendix."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that benchmark quality has gotten worse in absolute terms despite growing awareness challenges the assumption that the community is improving."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised; the paper focuses on research methodology quality."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Calls out widespread poor practices in benchmark construction with specific named examples (HumanEval bugs, MBPP duplicates), though framed diplomatically."
     },
     "demo_ability": {
       "score": 1,
       "justification": "The checklist is printed in the appendix but no tool, website, or downloadable resource is provided."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from established universities (HKUST, CUHK) but not from major AI labs; the paper does not evaluate any branded product."
     }
   }
 }