ai-research-survey

scan-v5.json (21420B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "GAIA: a benchmark for General AI Assistants",
     "authors": [
       "Grégoire Mialon",
       "Clémentine Fourrier",
       "Craig Swift",
       "Thomas Wolf",
       "Yann LeCun",
       "Thomas Scialom"
     ],
     "year": 2023,
     "venue": "arXiv.org",
     "arxiv_id": "2311.12983",
     "doi": "10.48550/arXiv.2311.12983"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The 92% human vs ~15% GPT-4+plugins claim is confirmed by Table 4 (aggregate: ~91.7% human, ~14.6% GPT-4+plugins across 466 questions). The performance disparity and difficulty levels are all supported by reported results.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'augmenting LLMs via tool APIs or access to the web improves answer accuracy,' but the GPT-4+plugins condition involves manually human-selected plugins, which the paper itself acknowledges as an 'oracle estimate' that 'cannot be reproduced exactly.' This confound makes causal attribution to tool augmentation unreliable.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'GAIA is only a first step to estimate the potential of AI assistants, but should not be seen as an absolute general proof of their success,' and Section 6 bounds the benchmark to English-language questions with compensated US-based annotators.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not systematically consider alternative explanations for the human-AI performance gap, such as whether question design inherently favors human reasoning styles, or whether the specific selection of factoid questions introduces its own biases unrelated to general AI capability.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper treats exact-match accuracy on 466 factoid questions as a measure of 'General AI Assistant' capability without formally arguing the validity of this proxy. The connection between answering specific real-world questions and broader assistant capability is assumed rather than demonstrated.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitations' section covering missing trace evaluation, the cost of unambiguous question design, and lack of linguistic/cultural diversity.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific threats: (1) GPT-4+plugins scores are not reproducible due to manual plugin selection, (2) questions may decay as web sources change, (3) only English is supported excluding ~80% of the global population, and (4) 32% of initially crafted questions required repair or removal.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states GAIA 'should not be seen as an absolute general proof' of AI assistant success, notes English-only coverage, and acknowledges that the benchmark does not evaluate reasoning traces or tool-call logs.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no funding disclosure in the paper. Surge AI annotators are mentioned as 'compensated' in a footnote, but no funding source for the research itself is disclosed.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in the header: FAIR Meta, HuggingFace, AutoGPT, and GenAI Meta.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears anywhere in the paper. Meta authors evaluating OpenAI's GPT-4 creates a potential indirect conflict that is undisclosed.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "'General AI Assistant' — the central concept — is never formally defined; it is treated as self-evident. 't-AGI' is defined only via a footnote link. The capability categories (web browsing, multi-modality, etc.) are defined in Appendix C but the core benchmark concept is not formally grounded.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it contributes a benchmark of 466 questions, a design methodology for creating further questions, a leaderboard, and an analysis of current LLM performance on real-world assistant tasks.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides detailed comparison with prior benchmarks (MMLU, GLUE, AgentBench, ToolQA, OpenAGI, Gorilla) explaining specific differences in design philosophy, evaluation environment, and contamination risk.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that real-world multi-step factoid questions requiring tool use measure 'fundamental abilities' of general AI assistants, explicitly invoking Chollet (2019) and the Proof of Work analogy to justify that successful completion reflects genuine capability rather than pattern matching.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Three difficulty levels are defined by number of steps and tools (Figure 3), with distributions shown graphically and validated by correlation with model performance degradation (Table 4: Level 1 30.3% → Level 3 0% for best LLM).",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "All evaluated LLMs score 0% on Level 3 questions, which is a floor effect that limits discriminability among models at the hardest tier. The paper presents this data but does not address it as a floor effect or discuss implications for inter-model ranking at Level 3.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": true,
           "justification": "Human annotator performance is reported as 92% aggregate and broken down by level (94%/92%/87% for Levels 1/2/3 in Table 4), derived from the validation annotation process.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Quasi-exact-match scoring is justified by the design requirement for unambiguous factoid answers; the justification is that factoid answers enable 'simple, fast and factual evaluation.' The scoring function is released with the leaderboard, though detailed normalization rules are not fully described in the paper.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Non-gameability is a stated design principle: answers are absent from pre-training data by design, question diversity prevents brute-forcing, and reasoning traces can be checked. Multiple-choice formats are explicitly avoided because they make contamination harder to detect.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5 addresses static vs. dynamic benchmark decay, identifying risks of web evidence disappearing or contamination entering future pre-training data, and proposes annual question refresh as a mitigation strategy.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Failure modes discussed include: web sources changing over time, closed-source API behavior changing (Chen et al. 2023), inability to evaluate reasoning traces, robots.txt compliance issues, and the benchmark's English-only coverage limiting generalization.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The scoring function is released alongside the leaderboard at huggingface.co/gaia-benchmark, 166 questions with answers are released as a developer set, and AutoGPT's git hash is provided (ed172dec). GPT-4+plugins baseline is explicitly noted as non-reproducible.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B provides a data card following Bender & Friedman (2018) covering language variety, annotator demographics (age, gender, academic background), curation rationale, and text characteristics. The question creation and validation process is described in detail in Section 3.4.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The paper states questions are released on HuggingFace with 300 held for the leaderboard, but no license (e.g., CC-BY, MIT) is stated in the paper for use, modification, or redistribution of the dataset.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The paper specifies GAIA is for evaluating general AI assistants in zero-shot settings, explicitly states it 'should not be seen as an absolute general proof' of success, and discusses what the benchmark does NOT evaluate (reasoning traces, fine-grained tool attribution).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Human respondents achieve 92% accuracy on GAIA while GPT-4 with oracle-selected plugins achieves only ~15%.",
       "evidence": "Table 4 shows human annotators at 93.9%/91.8%/87.3% across levels; GPT-4+plugins at 30.3%/9.7%/0%, aggregating to ~14.6%.",
       "supported": "strong"
     },
     {
       "claim": "GAIA difficulty levels (1/2/3) are validated by their correlation with model performance.",
       "evidence": "Table 4 shows monotonic performance degradation from Level 1 to Level 3 for all systems, supporting the difficulty ordering.",
       "supported": "strong"
     },
     {
       "claim": "AutoGPT with GPT-4 backend underperforms plain GPT-4 on GAIA despite having tool access.",
       "evidence": "Table 4: AutoGPT scores 14.4%/0.4%/0% vs. GPT-4 (no tools) at 9.1%/2.6%/0%; AutoGPT is worse at Level 2 and similarly poor at Level 3.",
       "supported": "strong"
     },
     {
       "claim": "Tool augmentation (web browsing, code execution) significantly improves LLM performance on GAIA.",
       "evidence": "GPT-4 + plugins scores 30.3% at Level 1 vs. GPT-4 without tools at 9.1%; however, plugin selection was manual/oracle, acknowledged as non-reproducible.",
       "supported": "moderate"
     },
     {
       "claim": "GAIA questions are resistant to contamination because answers are absent from pre-training data.",
       "evidence": "The paper asserts this by design principle but does not empirically verify that answers are absent from training corpora of evaluated models.",
       "supported": "weak"
     },
     {
       "claim": "68% of crafted GAIA questions were valid without modification, with Level 3 questions having the lowest initial validity (47%).",
       "evidence": "Table 3 reports validation statistics: 68% valid overall, 75%/68%/47% for Levels 1/2/3.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "GAIA reveals a stark capability gap: human annotators achieve 92% accuracy while the best-equipped LLM (GPT-4 with oracle-selected plugins) reaches only ~15%, with all models scoring 0% on Level 3 questions. The benchmark's three-level difficulty structure is empirically validated by monotonic performance degradation across all evaluated systems. Surprisingly, AutoGPT with automatic tool selection underperforms plain GPT-4 at Level 2, suggesting that autonomous agent orchestration remains immature. The 466-question benchmark is designed around four principles — real-world grounding, interpretability, contamination resistance, and ease of use — and is hosted as a live leaderboard with the developer set publicly released.",
   "red_flags": [
     {
       "flag": "Oracle plugin selection",
       "detail": "GPT-4+plugins scores were obtained by a human manually selecting the best plugins per question. The paper acknowledges this is an 'oracle estimate' that 'cannot be reproduced exactly.' This is the strongest-performing condition yet is not a realistic or reproducible experimental setup."
     },
     {
       "flag": "Level 3 floor effect unaddressed",
       "detail": "All LLMs score 0% on Level 3 (75 questions), creating a floor that prevents any ranking among models on the hardest tier. The paper presents this data but does not analyze whether Level 3 questions are too difficult to be useful discriminators."
     },
     {
       "flag": "No competing interests statement",
       "detail": "Meta FAIR authors evaluate OpenAI's GPT-4 critically, with a potential indirect interest in demonstrating GPT-4's weaknesses. No competing interests or independence statement is provided."
     },
     {
       "flag": "32% question failure rate",
       "detail": "Table 3 shows only 68% of crafted questions were valid; the remainder required correction or removal. At Level 3, only 47% passed initial validation. This suggests significant annotation effort and possible quality variation in surviving questions."
     },
     {
       "flag": "No dataset license stated",
       "detail": "The paper does not specify a license for the released questions, creating uncertainty about permissible downstream use for research or commercial applications."
     },
     {
       "flag": "Contamination claim unverified",
       "detail": "The claim that GAIA answers are absent from pre-training data is a design principle, not an empirically verified property. No membership inference or data provenance analysis is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "On the Measure of Intelligence",
       "relevance": "Core theoretical framework for evaluating AI capability; GAIA explicitly builds on Chollet's argument for benchmarks requiring flexible reasoning rather than specialized skills."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding (MMLU)",
       "relevance": "Primary benchmark GAIA positions itself against; MMLU saturation motivates GAIA's different design philosophy targeting real-world tasks rather than multiple-choice knowledge."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "relevance": "Most similar prior work; GAIA distinguishes itself by using the open real world rather than closed environments, avoiding evaluation of API-specific knowledge."
     },
     {
       "title": "Holistic Evaluation of Language Models (HELM)",
       "relevance": "Comprehensive benchmark compilation that GAIA critiques for difficulty of meaningful aggregation and sensitivity to experimental setup."
     },
     {
       "title": "ToolQA: A Dataset for LLM Question Answering with External Tools",
       "relevance": "Related tool-augmented QA benchmark; GAIA argues ToolQA risks contamination by reusing existing datasets."
     },
     {
       "title": "Augmented Language Models: A Survey",
       "relevance": "Survey of tool-augmented LLMs authored by overlapping GAIA authors; provides the context for why evaluating augmented assistants requires a new benchmark."
     },
     {
       "title": "Levels of AGI: Operationalizing Progress on the Path to AGI",
       "relevance": "AGI taxonomy used to situate GAIA's purpose; paper argues a GAIA-solving system would qualify as a 'competent General AI' under Morris et al.'s framework."
     },
     {
       "title": "WebGPT: Browser-Assisted Question-Answering with Human Feedback",
       "relevance": "Key prior system for web-browsing LLMs that GAIA is designed to evaluate and surpass."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Direct benchmark for practitioners building AI assistants; live leaderboard and released developer set enable immediate use."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Dramatically reverses the 'LLMs surpass humans' narrative — humans dominate 92% vs 15% — while also finding AutoGPT underperforms plain GPT-4 despite having tools."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Briefly mentions full automation reshaping socio-economic landscape and favoring open-source, but this is not a primary focus."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly critiques OpenAI (GPT-4 scores poorly, official search plugin was removed and brought back) but the paper avoids explicit conflict framing."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Live leaderboard at HuggingFace with 166 public questions allows anyone to test their system immediately."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Meta FAIR and HuggingFace are high-profile institutions; Yann LeCun's name alone drives significant attention."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "38388990",
         "title": "Meta: Gaia - A Benchmark for General AI Assistants",
         "points": 36,
         "comments": 8,
         "url": "https://news.ycombinator.com/item?id=38388990",
         "created_at": "2023-11-23T03:43:15Z"
       },
       {
         "hn_id": "39143540",
         "title": "The Optimal Choice of Hypothesis Is the Weakest, Not the Shortest",
         "points": 5,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=39143540",
         "created_at": "2024-01-26T15:14:52Z"
       },
       {
         "hn_id": "42413236",
         "title": "Fast and Efficient Memory Reclamation for Serverless MicroVMs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42413236",
         "created_at": "2024-12-13T23:18:15Z"
       },
       {
         "hn_id": "37985842",
         "title": "Eureka: Human-Level Reward Design via Coding Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37985842",
         "created_at": "2023-10-23T14:06:23Z"
       },
       {
         "hn_id": "37968009",
         "title": "Eureka: Human-Level Reward Design via Coding Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37968009",
         "created_at": "2023-10-21T16:09:03Z"
       },
       {
         "hn_id": "29504080",
         "title": "DeepMind's PolyViT: A multi-modal AI model",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=29504080",
         "created_at": "2021-12-09T22:45:22Z"
       },
       {
         "hn_id": "38401986",
         "title": "Gaia: A Benchmark for General AI Assistants",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38401986",
         "created_at": "2023-11-24T09:17:33Z"
       },
       {
         "hn_id": "38011924",
         "title": "Eureka: Human level reward design via coding large language models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38011924",
         "created_at": "2023-10-25T12:09:56Z"
       }
     ],
     "top_points": 36,
     "total_points": 51,
     "total_comments": 9
   }
 }