ai-research-survey

scan.json (28505B)

---

 {
   "paper": {
     "title": "GAIA: A Benchmark for General AI Assistants",
     "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom"],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2311.12983",
     "doi": "10.48550/arXiv.2311.12983"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GAIA introduces 466 questions for evaluating general AI assistants, where human respondents achieve 92% accuracy versus 15% for GPT-4 with plugins and 0% for Level 3 questions. The benchmark targets conceptually simple but execution-heavy tasks requiring web browsing, multi-modality, and tool use. The difficulty levels (defined by number of steps and tools needed) correlate with model performance, validating the level design. Augmenting LLMs with tools significantly improves performance but still leaves a large gap to human performance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a HuggingFace link (https://huggingface.co/gaia-benchmark) for the benchmark data and scoring function. The leaderboard and dev set of 166 questions are released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "166 annotated dev questions are released, and 300 questions are available via a leaderboard (answers withheld). The paper states 'We release our questions while retaining answers to 300 of them to power a leader-board.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are provided for running the scoring function or reproducing the evaluation."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The evaluation procedure is described at a high level (zero-shot prompting with a system prompt) but no runnable scripts or detailed reproduction guide is given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 shows ± notation for GPT-4 and GPT-4 Turbo (from 3 runs), but GPT-4+plugins and AutoGPT have no uncertainty measures. The human baseline has no uncertainty quantification. Partial reporting across only some conditions is insufficient."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite claims that GPT-4+plugins outperforms other models and that difficulty levels correlate with performance. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute accuracy scores with baselines (e.g., '92% vs. 15%' for humans vs GPT-4, '30.3% vs 9.1%' for plugins vs no plugins at Level 1), providing enough context to assess magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 466 questions but no justification for why this number is adequate. The paper acknowledges 'we preferred to favour quality over quantity' but does not justify whether 466 questions provides sufficient statistical power for the comparisons made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only GPT-4 and GPT-4 Turbo report variance from 3 runs (Table 4). GPT-4+plugins was run once manually. AutoGPT shows no variance. The human baseline has no variance measure. Inconsistent reporting across conditions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares GPT-4, GPT-4 Turbo, AutoGPT, GPT-4+plugins, human annotators, and web search as baselines (Section 4, Table 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 and GPT-4 Turbo were the most capable models available at time of writing (Nov 2023). The paper acknowledges API access limitations for evaluating other models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not test how removing specific question types, capabilities, or difficulty levels affects the benchmark's discriminative power."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (exact match) is used as an evaluation metric. No other metrics such as partial credit, reasoning trace quality, or time-adjusted scores are computed, despite time data being collected."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human annotators independently answered all questions as part of the validation phase (Section 3.4, Table 3). The 92% human accuracy serves as a key comparison point."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "300 questions have answers withheld for the leaderboard, and 166 are released as a dev set. The paper explicitly separates these: 'We release a developer set of 166 annotated questions and release the remaining 300 questions without annotations.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty level (1, 2, 3) in Table 4 and by capability type (web browsing, coding, multi-modality, filetype reading) in Figure 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figures 9-11 show detailed examples of GPT-4 failures, including a puzzle failure (Figure 11) and a case where lack of web access prevents answering (Figure 9). Section 4 discusses AutoGPT's disappointing performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "AutoGPT's poor performance relative to even base GPT-4 is reported and discussed: 'AutoGPT4, which allows GPT4 to automatically use tools, offer disappointing results.' GPT-4 gets 0% on Level 3."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 92% human vs 15% GPT-4 performance, which is supported by Table 4. The claim that questions are 'conceptually simple for humans yet challenging for most advanced AIs' is supported by the 92% human baseline."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'augmenting LLMs via tool APIs or access to the web improves answer accuracy' based on comparing GPT-4 with and without plugins, but this is confounded by plugin selection being done manually per question (an oracle setting the authors acknowledge)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'General AI Assistants' but Section 6 acknowledges the benchmark is English-only and web-centric. However, the abstract and introduction frame GAIA as a milestone for AGI ('the advent of Artificial General Intelligence hinges on a system's capability to exhibit similar robustness') which significantly overstates the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the human-AI gap. For example, the gap could be partly due to prompt engineering, plugin selection, or the specific types of questions chosen rather than fundamental capability differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames exact-match accuracy on 466 questions as measuring 'general AI assistant' capability and links it to AGI milestones, without discussing what 'general' means beyond the specific question types tested. The proxy gap between 'answers factoid questions correctly' and 'is a general AI assistant' is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4' and 'GPT-4 Turbo' without specific version identifiers or API snapshot dates. AutoGPT's git hash is provided (ed172dec) but the underlying GPT-4 version is not specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full system prompt used for evaluation is provided in Figure 2, including exact formatting instructions for answers."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API parameters are reported for any of the model evaluations."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "AutoGPT is used but its scaffolding is described only as 'able to do this selection automatically' without detail on retry logic, memory management, or tool selection strategy. GPT-4+plugins setup is described loosely ('we often rely on (i) a tool for reading various types of links, (ii) a web browsing tool, and (iii) a tool for computation')."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The question creation and validation pipeline is well documented in Section 3.4 and Appendix D, with statistics on validation outcomes (Table 3): 68% valid as-is, the rest corrected or removed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' is a dedicated section covering missing evaluations, cost of unambiguous question design, and lack of linguistic/cultural diversity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: evidence changing over time on the web, reliance on English-only questions (80% of world population excluded), inability to reproduce GPT-4+plugins results due to plugin instability, and the cost and difficulty of designing unambiguous questions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'GAIA is only a first step to estimate the potential of AI assistants, but should not be seen as an absolute general proof of their success.' It also notes the benchmark does not test actions beyond clicks, and does not evaluate reasoning traces."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dev set of 166 questions with annotations is released via HuggingFace, allowing independent verification of the question design and human baseline."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 and Appendix D describe the question creation process in detail: authors created initial questions, gave examples and instructions to Surge AI annotators, who created more. Validation required two independent annotators per question."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Annotators were from Surge AI (footnote 5) and demographics are reported in Appendix B, but how they were recruited or selected from Surge AI's pool is not described. No discussion of potential selection bias in annotator recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from question creation → validation by 2 annotators → repair/removal is documented with statistics: 623 questions created, 68% valid as-is, rest repaired or removed, yielding 466 final questions (Table 3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. No acknowledgment of grants, corporate funding for annotators, or research support beyond the author affiliations."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: FAIR/Meta, HuggingFace, AutoGPT, GenAI/Meta. These are prominent in the header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are from Meta and HuggingFace, both with commercial interests in AI assistants. Meta competes with OpenAI; the benchmark primarily evaluates OpenAI's GPT-4. The funder independence is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors from Meta and HuggingFace have clear commercial interests in the AI assistant space that are not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4's training cutoff is not stated. The paper discusses contamination risk conceptually but does not specify when any model's training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 extensively discusses contamination mitigation: answers are 'absent by design in plain text from current pre-training data,' questions require multi-step execution rather than memorization, and reasoning traces can be checked. Section 5 further discusses contamination risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is explicitly designed to resist contamination: 'the accuracy required in the answers, their absence from pre-training data, and the possibility to check the reasoning trace mitigate this risk' (Section 3.1). The benchmark is new, so questions were not in training data at time of release."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The study uses paid human annotators (Surge AI) to create and validate questions. No pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite using paid human annotators for question creation and validation."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports annotator demographics: age distribution (17% 18-25, 39% 26-35, etc.), gender (57% male, 43% female), and academic background (61% bachelor's, 26% master's, 17% PhD)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for annotator selection are described. The paper notes annotators were 'based in the US' but does not explain selection criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control conditions requiring randomization. Annotators all performed the same task."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not applicable to this study design. Validation annotators answered questions independently, which is a form of blinding to the original answer, but this is inherent to the design rather than a blinding protocol."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on annotator attrition or dropout is provided. The paper does not state how many annotators started vs. finished the task."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or token consumption are reported for any model evaluation. Time-to-answer is reported (Table 4) but monetary cost is not."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated for the evaluation. The paper mentions running GPT-4 API three times for variance but does not quantify the total compute or cost."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only GPT-4 and GPT-4 Turbo are run 3 times. No seed sensitivity analysis is performed. The ± values in Table 4 show some variance but this is not framed as seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Whenever an API is available, we run the model three times and report the average results' (Section 4). GPT-4+plugins was manually tested once."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates models in zero-shot with a fixed prompt; no hyperparameter search is performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "For GPT-4+plugins, the authors manually selected plugins per question ('according to our best guess of the most important capabilities'), creating an oracle setting. This is acknowledged but not justified as a fair comparison methodology."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluated the models. No discussion of author-evaluation bias, such as whether the question design inadvertently favors or disfavors certain systems."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "AutoGPT uses significantly more time (7.6-11.7 min) than GPT-4 API (0.12-0.24 min) but performance comparison does not account for this compute difference."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses at length what GAIA measures and why, and Section 5 discusses what a system solving GAIA would represent (t-AGI). The paper positions GAIA's design philosophy against existing benchmarks' shortcomings. The difficulty levels are validated against model performance (Section 4)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4+plugins uses manually selected plugins per question (oracle setting), AutoGPT uses automatic selection. The scaffold difference is acknowledged but not controlled: 'our score for GPT4 with plugins is an \"oracle\" estimate' — the confound between model capability and scaffold quality is not resolved."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is newly created (Nov 2023) with answers designed to be absent from pre-training data. The paper explicitly addresses this: 'the resulting answer is absent by design in plain text from current pre-training data.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., the system prompt, question phrasing) could leak information about the answer format or expected approach."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether questions share structural similarities, common sources, or patterns that could be exploited. Questions may share information sources (e.g., Wikipedia, arXiv) that could create dependencies."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The paper discusses contamination resistance by design but does not use canary strings, membership inference, or overlap analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "Human respondents achieve 92% accuracy on GAIA versus 15% for GPT-4 equipped with plugins",
       "evidence": "Table 4 shows human accuracy of 93.9%/91.8%/87.3% across levels. GPT-4+plugins achieves 30.3%/9.7%/0%. The 15% figure appears to be an approximate overall average. (Section 4, Table 4)",
       "supported": "strong"
     },
     {
       "claim": "GAIA's difficulty levels correlate with model performance",
       "evidence": "Table 4 shows monotonic decrease in model performance from Level 1 to Level 3 for all models. All models achieve 0% on Level 3. (Section 4, Figure 4)",
       "supported": "strong"
     },
     {
       "claim": "Augmenting LLMs with tools improves answer accuracy on GAIA",
       "evidence": "GPT-4 alone: 9.1% Level 1. GPT-4+plugins: 30.3% Level 1. However, plugin selection was manual/oracle, making this an upper bound rather than a fair comparison. (Section 4, Table 4)",
       "supported": "moderate"
     },
     {
       "claim": "GAIA is robust against memorization and data contamination",
       "evidence": "The paper argues questions require multi-step execution with answers absent from plain text on the internet, and reasoning traces can be checked. This is argued by design rather than empirically verified. (Section 3.1)",
       "supported": "moderate"
     },
     {
       "claim": "A system solving GAIA would represent a milestone toward AGI",
       "evidence": "The paper links GAIA to t-AGI and Morris et al. (2023) levels framework, but this is a framing argument rather than an empirical claim. No evidence that GAIA comprehensively covers AGI capabilities. (Section 1, Section 5)",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Oracle plugin selection",
       "detail": "GPT-4+plugins results used human-selected plugins per question, making this an oracle upper bound rather than a reproducible evaluation. The paper acknowledges this but still uses it as a primary comparison point."
     },
     {
       "flag": "Conflict of interest: Meta evaluating OpenAI",
       "detail": "Authors from Meta (a direct competitor) primarily evaluate OpenAI's GPT-4. While the benchmark itself is model-agnostic, the initial evaluation results only cover a competitor's product."
     },
     {
       "flag": "AGI framing exceeds evidence",
       "detail": "The paper repeatedly frames GAIA as an AGI benchmark, claiming 'the advent of Artificial General Intelligence hinges on a system's capability to exhibit similar robustness as the average human does on such questions.' This is a very strong claim for a 466-question English-only benchmark."
     },
     {
       "flag": "Non-reproducible evaluation",
       "detail": "The paper explicitly states that GPT-4+plugins scores 'cannot be reproduced exactly' due to changing plugins. The evaluation relies on closed-source APIs that may change behavior over time (acknowledged in Section 5)."
     },
     {
       "flag": "No statistical tests for any comparison",
       "detail": "All claims of performance differences between models are based solely on comparing raw numbers without any statistical testing, despite having multiple runs for some conditions."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "relevance": "Benchmark for evaluating LLM agents in closed-box environments, a direct comparison point for GAIA's open-world evaluation approach."
     },
     {
       "title": "On the Measure of Intelligence",
       "authors": ["François Chollet"],
       "year": 2019,
       "relevance": "Foundational work on measuring AI intelligence that influenced GAIA's design philosophy of testing fundamental abilities over specialized skills."
     },
     {
       "title": "Augmented Language Models: A Survey",
       "authors": ["Grégoire Mialon", "Roberto Dessì", "Maria Lomeli"],
       "year": 2023,
       "relevance": "Survey of tool-augmented LLMs directly relevant to understanding the agentic capabilities GAIA evaluates."
     },
     {
       "title": "Levels of AGI: Operationalizing Progress on the Path to AGI",
       "authors": ["Meredith Ringel Morris", "Jascha Sohl-dickstein", "Noah Fiedel"],
       "year": 2023,
       "relevance": "Framework for measuring AGI progress that GAIA positions itself within."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2022,
       "relevance": "HELM provides comprehensive LLM evaluation; GAIA positions itself as addressing HELM's limitations around tool use and real-world grounding."
     },
     {
       "title": "Beyond the Imitation Game: Quantifying and Extrapolating the Capabilities of Language Models",
       "authors": ["Aarohi Srivastava", "Abhinav Rastogi", "Abhishek Rao"],
       "year": 2023,
       "relevance": "BIG-Bench aggregated benchmark that GAIA contrasts with by focusing on fewer, higher-quality questions."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Model-based evaluation approach that GAIA avoids by using factual exact-match scoring."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "relevance": "Key work on LLM tool use, a core capability that GAIA benchmarks."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2021,
       "relevance": "MMLU benchmark that GAIA positions against as being near-saturated and susceptible to contamination."
     },
     {
       "title": "Efficient Benchmarking (of Language Models)",
       "authors": ["Yotam Perlitz", "Elron Bandel", "Ariel Gera"],
       "year": 2023,
       "relevance": "Discusses efficiency and reliability of LLM benchmarks, a concern GAIA addresses through curated high-quality questions."
     },
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "authors": ["Shishir G. Patil", "Tianjun Zhang", "Xin Wang"],
       "year": 2023,
       "relevance": "API-calling benchmark for LLMs; GAIA contrasts by not specifying possible APIs."
     }
   ]
 }