ai-research-survey

scan.json (24075B)

---

 {
   "paper": {
     "title": "Benchmarks for Automated Commonsense Reasoning: A Survey",
     "authors": ["Ernest Davis"],
     "year": 2023,
     "venue": "ACM Computing Surveys",
     "arxiv_id": "2302.04752",
     "doi": "10.1145/3615355"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or analysis scripts are released. The paper references a companion website (http://cs.nyu.edu/~davise/Benchmarks/) listing benchmarks, but no analysis code is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The companion website provides an annotated list of commonsense benchmarks with descriptions, examples, sizes, construction methods, and links. The survey's data (the catalog of 139 benchmarks) is publicly accessible via this site."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. This is a survey paper with no computational experiments requiring reproducible environments, but analysis scripts could still benefit from such specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The methodology for identifying and cataloging benchmarks is described in general terms but no step-by-step process for reproducing the survey is given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments or report quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not make statistical comparisons between systems or methods."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not report its own experimental effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper without experiments requiring sample size justification."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments with multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare its survey approach against prior surveys. While it cites prior work on AI benchmarks generally (e.g., references [128, 75, 108, 118]), it does not systematically compare its coverage, methodology, or conclusions to these prior surveys."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No systematic comparison to prior surveys of commonsense benchmarks is provided. The paper mentions prior surveys in passing but does not evaluate how its coverage or methodology compares."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "As a survey paper, there are no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate systems using metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not applicable to a survey paper cataloging benchmarks."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper without train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed breakdowns of benchmarks by category: text-based (102), image-based (18), video-based (12), and simulated physical environments (7), with further subcategorization within BIG-bench (48 tagged as common sense) and tables (2-10) organizing benchmarks by type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failures and flaws in existing benchmarks. Section 4.1.1 documents defective examples with a detailed taxonomy (incorrect answers, multiple correct answers, ill-formed questions, etc.). Table 1 provides concrete examples, and Appendix A analyzes 17 flawed items from the first 100 examples of OpenBookQA."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 ('Uncovered forms of commonsense reasoning') extensively discusses what existing benchmarks fail to test. The paper's central conclusion is negative: 'we do not currently have any reliable way of measuring to what extent existing AI systems have achieved common sense.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) more than 100 benchmarks exist (paper catalogs 139), (2) benchmarks are often flawed (supported by extensive examples in Section 4, Table 1, Appendix A), and (3) many aspects of common sense remain untested (supported by Section 10). All claims are well-supported by the paper's content."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It surveys, catalogs, and critically evaluates benchmarks without claiming causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is appropriately scoped to commonsense reasoning benchmarks specifically, and explicitly notes at the start that 'the primary focus in this paper is therefore on issues that are specific to commonsense reasoning, rather than on problems common to AI benchmarks generally.' The scope is well-bounded throughout."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses alternative explanations for AI performance on benchmarks. Section 2.1 discusses artifacts (e.g., SNLI/MultiNLI exploiting annotation artifacts). Section 5.2 discusses how adversarial filtering can create misleading improvement measurements. Section 5.3 discusses issues with interpreting 'human-level' performance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments with AI models. The ChatGPT examples in Section 9 and Table 18-19 are anecdotal illustrations, not systematic experiments."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use prompting for experiments."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no model experiments requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this survey paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document a systematic methodology for identifying and selecting the 139 benchmarks. The inclusion criterion is loosely stated: 'a benchmark is included in this collection if the authors describe it as a commonsense benchmarks' with 'a few exceptions.' No search strategy, databases queried, date ranges, or systematic filtering criteria are provided."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper includes recommendations (Section 11) but does not discuss the limitations of its own survey methodology."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to the validity of the survey itself are discussed. The paper does not acknowledge potential biases in its benchmark selection, the author's own involvement in several of the benchmarks reviewed (e.g., Winograd Schema Challenge), or the completeness of the catalog."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: it focuses on commonsense reasoning benchmarks specifically rather than AI benchmarks generally, and explicitly notes exclusions (e.g., 'datasets that the authors do not characterize as commonsensical and that are certainly only partially so have mostly been excluded'). Section 7.2 explains why mathematical word problem collections are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The annotated list of benchmarks is available on the companion website (http://cs.nyu.edu/~davise/Benchmarks/), allowing independent verification. Each benchmark entry includes name, description, examples, size, construction method, and links to the original paper and online site."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The data collection procedure for identifying the 139 benchmarks is not described in detail. There is no discussion of what databases were searched, what search terms were used, or how comprehensiveness was assessed. The inclusion criterion is stated only loosely."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this survey. The paper catalogs existing benchmarks rather than involving human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline from initial search to final catalog is documented. It is unclear how the author went from the space of all AI benchmarks to the final set of 139 commonsense benchmarks."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. The acknowledgements section thanks individuals for feedback but does not mention any funding sources or grants."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation (Dept. of Computer Science, New York University) is clearly stated at the beginning of the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The absence of a funding disclosure is itself a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. The author is a co-creator of the Winograd Schema Challenge (reference [82]) and has published extensively on commonsense reasoning, which represents a potential intellectual conflict of interest that is not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate pre-trained models on benchmarks. The ChatGPT anecdotes in Section 9 are illustrative, not systematic evaluations."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a pre-trained model on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. However, it notably discusses benchmark contamination as a concern for other researchers in Section 5.4, which is one of its strengths."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. Cost reporting is not applicable as no method is proposed or evaluated."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments requiring a compute budget."
       }
     }
   },
   "claims": [
     {
       "claim": "More than one hundred benchmarks have been developed to test commonsense knowledge and reasoning abilities of AI systems.",
       "evidence": "The paper catalogs 139 benchmarks across tables 2-10: 102 text-based, 18 image-based, 12 video-based, and 7 simulated physical environments (Section 7, abstract).",
       "supported": "strong"
     },
     {
       "claim": "Many commonsense benchmarks are significantly flawed, limiting their usefulness as measures.",
       "evidence": "Section 4.1.1 and Table 1 provide numerous concrete examples of flaws (incorrect answers, multiple correct answers, encyclopedic rather than commonsense knowledge, ill-formed questions). Appendix A shows 17 of the first 100 OpenBookQA test items are substantially flawed. Sections 8.1-8.3 document quality issues in specific benchmarks including CommonsenseQA 2.0, HellaSwag, and McTaco.",
       "supported": "strong"
     },
     {
       "claim": "Many significant aspects of commonsense reasoning remain untested by any existing benchmark.",
       "evidence": "Section 10 identifies untested areas including temporal interval reasoning, hypothetical timelines, commonsense psychological reasoning (beyond ATOMIC), spatial knowledge, physical knowledge, biological knowledge, and social knowledge. The paper argues these are 'not covered more than extremely partially and haphazardly.'",
       "supported": "strong"
     },
     {
       "claim": "For test sets, small and clean is better than large and noisy.",
       "evidence": "Section 5.1 presents a detailed argument that statistical precision from large noisy datasets is illusory because the underlying quantity being measured is not defined to high precision, and clean datasets avoid penalizing systems for knowing too much (e.g., recognizing multiple correct answers). The author estimates expert vetting of a 500-example set requires about 5.5 hours.",
       "supported": "moderate"
     },
     {
       "claim": "Adversarial filtering in benchmark construction can exaggerate measured improvement between AI systems.",
       "evidence": "Section 5.2 argues that removing questions system X answers correctly biases measurement in favor of any later system Y that answers different questions, even if Y is not overall better. The paper cites concrete evidence of retrogression between GPT-3 versions (Section 5.2, reference [34]).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["meta-analysis", "qualitative"],
   "key_findings": "This survey catalogs 139 commonsense reasoning benchmarks (102 text-based, 18 image-based, 12 video-based, 7 simulated environments) and finds that many are significantly flawed, with defective examples including incorrect answers, multiple correct answers, and reliance on encyclopedic rather than commonsense knowledge. The paper identifies substantial gaps in benchmark coverage, particularly in temporal interval reasoning, spatial and physical knowledge, psychological reasoning, and social commonsense. The author argues that small, carefully vetted test sets are preferable to large noisy ones, and that no existing benchmark reliably measures whether AI systems have achieved commonsense reasoning abilities.",
   "red_flags": [
     {
       "flag": "No systematic search methodology",
       "detail": "The paper does not describe a systematic literature search process (databases, search terms, date ranges, inclusion/exclusion criteria with counts) for identifying the 139 benchmarks. The inclusion criterion is loosely stated as self-description by benchmark authors, with 'a few exceptions' at the surveyor's discretion. This makes the survey's comprehensiveness unverifiable."
     },
     {
       "flag": "No quality assessment framework for reviewed benchmarks",
       "detail": "While the paper provides insightful qualitative analysis of individual benchmarks, there is no systematic quality assessment rubric applied consistently across all 139 benchmarks. Most benchmarks get only a brief table entry; 12 get detailed treatment. The selection of which 12 to examine in detail is not justified by explicit criteria."
     },
     {
       "flag": "Author conflict of interest not disclosed",
       "detail": "Ernest Davis is a co-creator of the Winograd Schema Challenge (reference [82]), one of the most prominent benchmarks surveyed. While the paper notes 'Having reviewed the Winograd Schema Challenge in detail elsewhere, we deliberately omit it here' (Section 8), the intellectual conflict is not formally disclosed, and the author's own benchmarks and test examples appear throughout the paper."
     },
     {
       "flag": "No limitations section",
       "detail": "The survey lacks any discussion of its own limitations, such as potential selection bias in benchmark inclusion, English-language bias in the survey itself, or the completeness of the catalog."
     }
   ],
   "cited_papers": [
     {
       "title": "Beyond the Imitation Game: Quantifying and Extrapolating the Capacities of Language Models",
       "authors": ["Aarohi Srivastava"],
       "year": 2022,
       "arxiv_id": "2206.04615",
       "relevance": "BIG-bench is a major multi-task benchmark collection (212 datasets) for evaluating LLM capabilities, directly relevant to evaluating LLM evaluation methodology."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2022,
       "arxiv_id": "2211.09110",
       "relevance": "HELM is a comprehensive LLM evaluation framework that selected commonsense benchmarks as part of its evaluation suite, directly relevant to LLM evaluation methodology."
     },
     {
       "title": "Large Language Models are Few-Shot Learners",
       "authors": ["Tom Brown"],
       "year": 2020,
       "relevance": "GPT-3 paper that established the foundation model paradigm affecting how benchmarks are used and the challenge of test set contamination."
     },
     {
       "title": "On the Opportunities and Risks of Foundation Models",
       "authors": ["Rishi Bommasani"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Foundational paper on foundation models that discusses risks and evaluation challenges relevant to understanding LLM capabilities."
     },
     {
       "title": "Annotation Artifacts in Natural Language Inference Data",
       "authors": ["Suchin Gururangan", "Swabha Swayamdipta", "Omer Levy"],
       "year": 2018,
       "arxiv_id": "1803.02324",
       "relevance": "Demonstrates how annotation artifacts in SNLI/MultiNLI allow solving problems without seeing premises, directly relevant to benchmark quality and evaluation methodology."
     },
     {
       "title": "Reduced, Reused and Recycled: The Life of a Dataset in Machine Learning Research",
       "authors": ["Bernard Koch", "Emily Denton", "Alex Hanna"],
       "year": 2021,
       "arxiv_id": "2112.01716",
       "relevance": "Studies dataset reuse patterns in ML research, relevant to understanding benchmark methodology and dataset quality issues."
     },
     {
       "title": "Targeting the Benchmark: On Methodology in Current Natural Language Processing Research",
       "authors": ["David Schlangen"],
       "year": 2020,
       "arxiv_id": "2007.04792",
       "relevance": "Critical analysis of benchmark-driven NLP research methodology, directly relevant to questions about whether benchmarks measure what they claim to measure."
     },
     {
       "title": "Data and Its (Dis)contents: A Survey of Dataset Development and Use in Machine Learning Research",
       "authors": ["Amandalynne Paullada", "Inioluwa Deborah Raji", "Emily M. Bender"],
       "year": 2021,
       "relevance": "Survey of dataset development practices in ML research, relevant to understanding data quality and documentation issues in AI evaluation."
     },
     {
       "title": "Theory of Mind may have Spontaneously Emerged in Large Language Models",
       "authors": ["Michael Kosinski"],
       "year": 2023,
       "arxiv_id": "2302.02083",
       "relevance": "Controversial claim about emergent LLM capabilities that the survey critiques as an example of overclaiming based on test contamination, relevant to LLM evaluation methodology."
     },
     {
       "title": "How Not to Test GPT-3",
       "authors": ["Gary Marcus", "Ernest Davis"],
       "year": 2023,
       "relevance": "Critique of methodology for testing LLM capabilities showing that GPT-3's apparent theory of mind reflects memorization rather than understanding."
     },
     {
       "title": "Winogrande: An Adversarial Winograd Schema Challenge at Scale",
       "authors": ["Keisuke Sakaguchi", "Ronan Le Bras", "Chandra Bhagavatula"],
       "year": 2021,
       "relevance": "Major adversarially-constructed commonsense benchmark demonstrating adversarial filtering methodology for LLM evaluation."
     },
     {
       "title": "HellaSwag: Can a Machine Really Finish Your Sentence?",
       "authors": ["Rowan Zellers", "Ari Holtzman", "Yonatan Bisk"],
       "year": 2019,
       "arxiv_id": "1905.07830",
       "relevance": "Widely-used commonsense benchmark for LLM evaluation, selected as part of HELM, demonstrating adversarial generation methodology."
     }
   ]
 }