ai-research-survey

scan.json (18449B)

---

 {
   "paper": {
     "title": "On the Measure of Intelligence",
     "authors": ["François Chollet"],
     "year": 2019,
     "venue": "arXiv",
     "arxiv_id": "1911.01547"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The ARC dataset is released at github.com/fchollet/ARC, as stated in Section III.1.1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The ARC dataset (400 training tasks, 400 public evaluation tasks, 200 private evaluation tasks) is publicly available at github.com/fchollet/ARC."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper with a dataset release. There is no computational environment to specify — the dataset consists of JSON task files."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are run that would require reproduction instructions. The paper is theoretical with a benchmark proposal."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not report quantitative experimental results. It is a theoretical paper proposing definitions and a benchmark."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative empirical claims are made that would require significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No empirical experiments are conducted."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical experiments with samples are conducted."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No empirical experiments are conducted."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any system's performance. It proposes a benchmark and theoretical framework."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation is performed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system is proposed that could be ablated."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation is conducted."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ARC is 'fully solvable by humans' and that 'each task included in ARC has been successfully solved by at least one member of a group of three high-IQ humans,' but no formal human evaluation with reported metrics or systematic study is presented. The paper acknowledges this as a weakness: 'we hope to be able to further investigate human performance on ARC by gathering a statistically significant amount of human testing data.'"
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is conducted. The benchmark itself has a private evaluation set, but no system is tested on it."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No empirical results are reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No system is evaluated, so no failure cases are discussed. The paper does discuss weaknesses of the ARC benchmark itself (Section III.2) but not failure cases of any evaluated system."
       },
       "negative_results_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted to produce positive or negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims are primarily theoretical (proposing a definition, proposing a benchmark) and argumentative (skill measurement is insufficient). The paper delivers on these: it provides the formal definition in Section II.2, critiques skill-based evaluation in II.1, and presents ARC in Section III."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes theoretical arguments and proposals, not causal empirical claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit about the scope of its claims. Section II.1.2 argues intelligence is tied to a specific scope, and Section III.2 lists known weaknesses. The paper states 'Our claims are highly speculative and may well prove fully incorrect' (III.1.4) and 'ARC is a work in progress, not a definitive solution' (III.2)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses alternative views of intelligence (task-specific skills vs. general learning ability in I.2), critiques existing approaches (I.3, II.1), and acknowledges competing formalisms (AIXI, Universal Intelligence, C-Test) in II.2.1. Section III.3 discusses alternative benchmark approaches."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used or evaluated in this paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models are trained or evaluated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data preprocessing pipeline is involved. ARC tasks are manually created."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section III.2 'Weaknesses and future refinements' provides a detailed discussion of five specific weaknesses of the ARC benchmark."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section III.2 lists specific threats: generalization is not quantified, test validity is not established, dataset size and diversity may be limited, evaluation format is overly close-ended and binary, and Core Knowledge priors may not be well captured. These are specific to this work, not generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries throughout. Section II.1.2 argues intelligence is tied to a scope of application. The paper states 'Our claims are highly speculative and may well prove fully incorrect' (III.1.4). Section II.2.1 notes the definition is not 'the one true definition' and aims to be 'actionable' rather than exhaustive."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The ARC dataset is publicly available at github.com/fchollet/ARC."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.1.1-III.1.2 describes how ARC tasks are designed: manually created based on Core Knowledge priors, with specific constraints on grid sizes (1x1 to 30x30), 10 symbols, and the four prior categories described in III.1.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are formally studied. The mention of 'three high-IQ humans' checking task feasibility is informal and does not constitute a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the design principles behind ARC tasks but does not document a systematic pipeline for task creation, selection criteria, or quality control beyond the Core Knowledge framework."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The author is affiliated with Google, but no funding acknowledgment section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation with Google, Inc. is clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. The author works at Google, which has a vested interest in AI benchmarks and intelligence measurement, but no explicit funding statement is made and no conflict analysis is possible."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper with a benchmark proposal. No computational method is run."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computation is performed."
       }
     }
   },
   "claims": [
     {
       "claim": "Measuring skill at specific tasks is insufficient for measuring intelligence, because skill is heavily modulated by prior knowledge and experience that can mask generalization power.",
       "evidence": "Extensive theoretical argument in Sections II.1.1 and II.2, with examples including DeepBlue, OpenAI Five (beaten by non-champion humans after public release despite 45,000 years of training), and a hashtable thought experiment showing local-generalization systems can solve any task with unlimited data.",
       "supported": "strong"
     },
     {
       "claim": "Intelligence should be formally defined as skill-acquisition efficiency over a scope of tasks, with respect to priors, experience, and generalization difficulty.",
       "evidence": "Formal definition provided in Section II.2.1 using Algorithmic Information Theory, with definitions of generalization difficulty, priors, experience, and a mathematical formula for intelligence.",
       "supported": "moderate"
     },
     {
       "claim": "ARC can be used to measure a human-like form of general fluid intelligence and enables fair general intelligence comparisons between AI systems and humans.",
       "evidence": "Section III describes the benchmark design. However, the paper acknowledges this claim is 'highly speculative' (III.1.4), test validity is not established (III.2), and only informal human testing with three individuals was conducted.",
       "supported": "weak"
     },
     {
       "claim": "ARC is fully solvable by humans but cannot be meaningfully approached by any existing machine learning technique.",
       "evidence": "Section III.1.4 states each task was solved by at least one of three high-IQ humans. The claim about ML is stated without systematic evidence — no ML baselines are reported.",
       "supported": "weak"
     },
     {
       "claim": "Human general intelligence is not universal but is highly specialized, tied to the scope of human-relevant tasks.",
       "evidence": "Section II.1.2 provides theoretical arguments with examples: humans fail at 4D+ navigation, perform poorly on non-Euclidean TSP variants, and cannot do long-term planning or large working memory tasks. Cites cognitive psychology literature on dimensional bias and Core Knowledge.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical"],
   "key_findings": "The paper proposes a formal definition of intelligence as skill-acquisition efficiency, grounded in Algorithmic Information Theory, arguing that measuring task-specific skill alone is insufficient because unlimited priors or training data can artificially inflate performance without generalization. It introduces the Abstraction and Reasoning Corpus (ARC), a benchmark of 1,000 visual reasoning tasks built on Core Knowledge priors, designed to measure developer-aware broad generalization rather than narrow skill. The paper explicitly acknowledges ARC's limitations including unestablished test validity and unquantified generalization difficulty.",
   "red_flags": [
     {
       "flag": "No empirical validation",
       "detail": "The core claims about ARC measuring general intelligence and being unsolvable by ML are not empirically tested. No ML baselines are reported. Human solvability is tested only informally with three 'high-IQ' individuals with no systematic data collection."
     },
     {
       "flag": "Author conflict not analyzed",
       "detail": "The author works at Google, a major AI company with interests in AI benchmarking and intelligence measurement. No conflict of interest statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Building machines that learn and think like people",
       "authors": ["Brenden M. Lake", "Tomer D. Ullman", "Joshua B. Tenenbaum", "Samuel J. Gershman"],
       "year": 2016,
       "relevance": "Foundational work on cognitive priors for AI that directly informs the ARC benchmark design and the paper's argument about Core Knowledge."
     },
     {
       "title": "Universal intelligence: A definition of machine intelligence",
       "authors": ["Shane Legg", "Marcus Hutter"],
       "year": 2007,
       "relevance": "Alternative AIT-based formal definition of intelligence that the paper critiques and contrasts with its own definition."
     },
     {
       "title": "Evaluation in artificial intelligence: from task-oriented to ability-oriented measurement",
       "authors": ["José Hernández-Orallo"],
       "year": 2017,
       "relevance": "Comprehensive survey of AI evaluation methods that motivates the paper's argument for ability-oriented rather than skill-oriented evaluation."
     },
     {
       "title": "Program Synthesis",
       "authors": ["Sumit Gulwani", "Alex Polozov", "Rishabh Singh"],
       "year": 2017,
       "relevance": "The paper frames ARC as a program synthesis benchmark, connecting intelligence measurement to automated program generation."
     },
     {
       "title": "Quantifying generalization in reinforcement learning",
       "authors": ["Karl Cobbe", "Oleg Klimov", "Christopher Hesse", "Taehoon Kim", "John Schulman"],
       "year": 2018,
       "relevance": "Discusses generalization measurement in RL, which the paper critiques as measuring only local generalization rather than broad abilities."
     },
     {
       "title": "Deep learning: A critical appraisal",
       "authors": ["Gary Marcus"],
       "year": 2018,
       "arxiv_id": "1801.00631",
       "relevance": "Critique of deep learning limitations that supports the paper's argument about local generalization being insufficient."
     },
     {
       "title": "A collection of definitions of intelligence",
       "authors": ["Shane Legg", "Marcus Hutter"],
       "year": 2007,
       "relevance": "Survey of 70+ intelligence definitions that provides context for the paper's own formal definition."
     },
     {
       "title": "The Measure of All Minds: Evaluating Natural and Artificial Intelligence",
       "authors": ["José Hernández-Orallo"],
       "year": 2017,
       "relevance": "Comprehensive book on AI evaluation that the paper builds upon for its intelligence measurement framework."
     }
   ]
 }