ai-research-survey

scan.json (32532B)

---

 {
   "paper": {
     "title": "GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks",
     "authors": [
       "Tejal Patwardhan",
       "Rachel Dias",
       "Elizabeth Proehl",
       "Grace Kim",
       "Michele Wang",
       "Olivia Watkins",
       "Simón Posada Fishman",
       "Marwan Aljubeh",
       "Phoebe Thacker",
       "Laurance Fauconnet",
       "Natalie S. Kim",
       "Patrick Chao",
       "Samuel Miserendino",
       "Gildas Chabot",
       "David Li",
       "Michael Sharman",
       "Alexandra Barr",
       "Amelia Glaese",
       "Jerry Tworek"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.04374",
     "doi": "10.48550/arXiv.2510.04374"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided. The paper open-sources task prompts and reference files, and provides an automated grading service at evals.openai.com, but no code repository (e.g., GitHub) for the evaluation pipeline, grading code, or analysis scripts is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'We open-source the prompts and reference files in our 220-task gold subset.' The 220-task gold subset including prompts and reference files is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.6.4 provides an extensive list of pre-installed Python packages with specific version numbers in the Docker image used for grading and model sampling."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The automated grader at evals.openai.com is available, but the paper does not include instructions for reproducing the full evaluation pipeline (sampling, grading, analysis)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section A.6.2 states: 'Plots below show 95% confidence intervals obtained by bootstrapping.' Figure 16 shows confidence intervals for both human-automated grader agreement and human inter-rater agreement."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares model win rates against human baselines and across models but does not apply any statistical significance tests (p-values, t-tests, or other hypothesis tests). Claims like 'Claude Opus 4.1 was the best performing model' are based on comparing raw percentages without formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Win rates are reported with baseline context (e.g., '47.6% of deliverables by Claude Opus 4.1 were graded as better than or as good as the human deliverable'). Speed ratios (90x-327x), cost ratios, and prompting improvements ('5 percentage points') are reported with sufficient context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 220 gold subset tasks and 1,320 full set tasks with at least 30 tasks per occupation, but provides no power analysis or explicit justification for why these sample sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrapped 95% confidence intervals are reported in Section A.6.2 and Figure 16. The paper states 9 comparisons per prompt per model (3 samples × 3 graders), and bootstrapping provides variance estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The primary baseline is human expert deliverables. Models are compared against human expert completions in blinded pairwise comparisons (Section 2.5, fig. 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates GPT-4o, o4-mini, o3, GPT-5, Claude Opus 4.1, Gemini 2.5 Pro, and Grok 4 — all current frontier models as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 tests reasoning effort (low/medium/high), prompt tuning, and scaffolding improvements (GET requests, best-of-N sampling). Section A.2.7 tests under-contextualized prompts. These function as ablations of key experimental variables."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports win rates, speed ratios, cost ratios, human-automated grader agreement, inter-rater agreement, failure severity categorization, and breakdowns by sector, occupation, deliverable type, and task duration."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human expert pairwise comparison is the primary evaluation metric. Section 2.5: 'blinded expert pairwise comparisons, where experts in the relevant occupation were presented with a request and reference files and asked to rank two or more unlabeled work deliverables.' Grading averaged over an hour per comparison."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The gold subset of 220 tasks is used for all main experiments and is also open-sourced. The prompt-tuning experiment (Section A.3) and scaffolding improvements were developed and evaluated on the same gold subset, with no clear separation of development and test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Breakdowns are provided by sector (fig. 10), occupation (fig. 11), deliverable file type (fig. 12), time to complete (fig. 13), and failure severity (fig. 14)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 and fig. 8 analyze why experts preferred or rejected model deliverables (instruction-following failures, formatting errors, hallucinated data, missing deliverables). Section A.2.6 categorizes GPT-5 failures as catastrophic (~3%), bad (~26%), or acceptable but subpar (~48%)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The under-contextualized experiment (Section A.2.7) shows degraded performance. GPT-4o achieves only 12.5% win rate. Formatting errors persist even after prompt tuning (86% → 64% for PowerPoint). Longer tasks show steadily declining win rates (fig. 13)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about linear improvement (fig. 6), approaching expert parity (fig. 5), speed/cost savings (Table 2, fig. 7), reasoning effort improvements (fig. 9a), and open-sourcing (Section 4) are all supported by data presented in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about reasoning effort improving performance are supported by controlled single-variable experiments (fig. 9a, varying reasoning level while holding model constant). The prompt-tuning claim is supported by a before/after intervention design (fig. 9b). These are adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Limitations) explicitly bounds the results: 'only 44 occupations and 30 total tasks per occupation,' 'oriented around knowledge work that can be performed on a computer,' 'Manual labor and physical tasks are not included,' and tasks are 'precisely-specified and one-shot, not interactive.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the core findings. For example, the linear improvement claim (fig. 6) does not consider whether the trend could be driven by improved prompting/tooling rather than raw model capability. The win rate differences across models are not analyzed for confounds beyond stylistic identification."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The introduction explicitly distinguishes between measuring AI capabilities and predicting economic impact: 'While informative when available, these methods are lagging indicators of AI impacts. We consider an alternate method for understanding the potential economic impacts of AI: directly measuring AI model capabilities.' The limitations section further notes the gap between one-shot task completion and real-world work."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names only: 'GPT-4o, o4-mini, o3, GPT-5, Claude Opus 4.1, Gemini 2.5 Pro, and Grok 4.' No API versions, snapshot dates, or model IDs are provided. Model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The 220 task prompts and reference files are open-sourced (Section 4). The scaffolding/quality-check prompt used for the prompt-tuning experiment is provided verbatim in Section A.3."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or max tokens settings are reported for any model. Reasoning effort levels (low/medium/high) are noted but these are not standard hyperparameters. Claude was 'sampled via the UI' with no parameter specification."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes: web search and code interpreter tools enabled for OpenAI models, Claude sampled via UI with 'Upgraded file creation and analysis' feature, background sampling, pre-installed packages (Section A.6.4), best-of-N sampling with N=4 and GPT-5 judge, and GET requests enabled in the container."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 2.1-2.4 detail the full pipeline: sector selection by GDP contribution, occupation selection by total wages and digital classification, expert recruitment with criteria, task creation against O*NET categories, and multi-stage quality control (model screening + 5+ human reviews). Appendix A.7 provides further methodological detail."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is a dedicated 'Limitations' section covering dataset size, focus on knowledge work, task specification, grader performance, and cost."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'only 44 occupations and 30 total tasks per occupation,' 'Manual labor and physical tasks are not included,' 'tasks that involve extensive tacit knowledge, access to personally identifiable information, use of proprietary software tools, or communication between individuals are out of scope.' Expert time self-reporting bias is acknowledged in footnote 6."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states what's excluded: manual labor, physical tasks, tasks requiring tacit knowledge, PII access, proprietary tools, or interpersonal communication. Tasks are described as 'precisely-specified and one-shot, not interactive,' and the benchmark covers only 'a limited, initial cut of knowledge work tasks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the 220 gold subset task prompts and reference files are released, the raw grading data (expert judgments, justifications, scores), model outputs, and human expert deliverables are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 2.1-2.5 describe the full data collection procedure: sector identification from GDP data, occupation selection via BLS wage data and digital classification, expert recruitment criteria, task creation against O*NET categories, and iterative quality control pipeline."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes expert recruitment: minimum 4 years experience, strong resume with professional recognition, video interview, background check, training and quiz. Less than 10% of applicants were selected (Section A.4.4). Each occupation had minimum 5 qualified professionals."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: sector selection (9 sectors >5% GDP) → occupation selection (5 per sector by wage contribution and digital classification) → expert recruitment (criteria, <10% acceptance) → task creation (with O*NET mapping) → quality control (model screening + 3-stage human review, average 5 reviews per task). Figure 3 illustrates the review pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is present. All authors are affiliated with OpenAI, which implicitly funds the research, but there is no explicit funding statement or acknowledgment of funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed under the OpenAI affiliation prominently at the top of the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI has a direct financial interest in demonstrating that its GPT models perform well on economically valuable tasks. The benchmark was designed and evaluated by OpenAI employees testing OpenAI products (GPT-4o, o4-mini, o3, GPT-5) alongside competitors."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. All authors are OpenAI employees whose livelihoods depend on the commercial success of the models being evaluated."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are provided for any of the evaluated models (GPT-4o, o3, GPT-5, Claude Opus 4.1, Gemini 2.5 Pro, Grok 4)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether any of the evaluated models could have been trained on content similar to the expert-created tasks, reference files, or related professional materials."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Although GDPval tasks are newly created from expert work (reducing contamination risk), the paper does not explicitly discuss contamination risk or the advantage of novel tasks for contamination avoidance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a benchmark evaluation of AI models. The human experts serve as task creators and evaluators, not as subjects of a human-subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation, not a human-subjects study. Experts are compensated professional evaluators, not research subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a benchmark evaluation. Expert demographics (years of experience, prior employers) are reported for credibility context, but the study does not investigate human behavior."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a benchmark evaluation. Expert selection criteria are described for methodological transparency, but this is not a human-subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human-subjects experimental study. No treatment/control assignment of human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human-subjects study. Blinding of graders to model identity is described for evaluation quality purposes but is not a human-subjects blinding protocol."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human-subjects study. Expert participation rates are not relevant as human-subjects attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports cost improvement ratios for OpenAI models. Section A.2.1 defines model completion cost MC and states 'we recorded the average invoiced cost per task.' Average human completion cost HC = $361 is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated for the evaluation (total API spend across all models, total expert hours, total project cost). Per-task costs and ratios are reported but not the aggregate compute budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While 3 samples per model per prompt are collected, no explicit seed sensitivity analysis is performed or reported. The paper does not discuss how results vary across random seeds or sampling runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 2 states: 'we sampled each model 3 times for each prompt, and then had 3 different human graders grade each sample (yielding 9 comparisons per prompt, per model, across 220 tasks).'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The prompt-tuning experiment (Section A.3) and scaffolding choices appear to have been iteratively developed but no search budget or number of configurations tried is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The prompt-tuning and scaffolding configurations (Section A.3, best-of-N with N=4) are presented without justification for how they were selected from alternatives or whether they were tuned on the same data used for evaluation."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 7 models across multiple sectors, occupations, and deliverable types without applying any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper acknowledges automated grader self-bias ('models often favor their own responses,' Section A.6.2), it does not address the broader institutional bias of OpenAI designing a benchmark, controlling the experimental setup, selecting which models to test, and evaluating its own commercial products."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models are compared at different reasoning efforts for the same model family (fig. 9a), but cross-model comparisons do not account for compute budget differences. The naive speed ratios (Table 2) vary from 90x to 327x across models, suggesting very different compute usage that is not factored into performance comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section A.7.1 validates the digital-task classification against the Acemoglu & Autor (2011) task-content framework, showing alignment with established economic measures. Expert representativeness ratings average 4.50/5.00 (Table 3). O*NET coverage analysis (Table 6) shows 71.4% skills coverage and 63.4% work activities coverage."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different models received different scaffolding: OpenAI models got web search and code interpreter tools; Claude was sampled via UI with 'Upgraded file creation and analysis'; packages were preinstalled. Footnote 2 notes these differences but does not address how they confound model comparisons. Performance differences may reflect scaffolding quality rather than model capability."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Tasks are newly created by experts, which inherently reduces temporal leakage risk, but the paper does not discuss temporal leakage or explicitly note the advantage of using novel tasks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether reference files or task contexts could inadvertently leak information about the expected deliverable quality or structure."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between tasks. Multiple tasks per occupation may share structural similarities, and the paper does not analyze whether this affects results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are described or applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Frontier model performance on GDPval is improving roughly linearly over time.",
       "evidence": "Figure 6 shows performance of OpenAI frontier models (GPT-4o through GPT-5) increasing roughly linearly on the gold subset.",
       "supported": "weak"
     },
     {
       "claim": "The current best frontier models are approaching industry experts in deliverable quality, with 47.6% of Claude Opus 4.1 deliverables graded as better than or equal to human expert deliverables.",
       "evidence": "Figure 5 shows win rates for 7 models. Claude Opus 4.1 achieves the highest combined win+tie rate of 47.6%. GPT-5 achieves ~39% win rate.",
       "supported": "strong"
     },
     {
       "claim": "Models can save time and money relative to unaided experts when paired with human oversight.",
       "evidence": "Table 2 and Section A.2.1 show speed and cost improvement ratios under 'try n times, then fix it' setups. GPT-5 shows 1.39x speed improvement and 1.63x cost improvement under try-nx.",
       "supported": "moderate"
     },
     {
       "claim": "Increased reasoning effort improves model performance on GDPval.",
       "evidence": "Figure 9a shows win rates increasing from low to medium to high reasoning effort for both o3 and GPT-5.",
       "supported": "strong"
     },
     {
       "claim": "Increased task context improves model performance.",
       "evidence": "Section A.2.7 shows GPT-5 performance degrades on under-contextualized prompts (42% of original token length), demonstrating context sensitivity.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt-tuning and scaffolding improvements increase GPT-5 performance by 5 percentage points.",
       "evidence": "Figure 9b shows improvement from prompt-tuning. Prompting 'fully eliminated black-square artifacts' and 'reduced egregious formatting errors in PowerPoint files from 86% to 64%.'",
       "supported": "moderate"
     },
     {
       "claim": "Claude Opus 4.1 excels on aesthetics while GPT-5 excels on accuracy.",
       "evidence": "Figure 8 and Section 3.1 analyze failure modes. Section A.2.4 shows GPT-5 leads for pure text while Claude leads for .pdf, .xlsx, .pptx deliverables.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GDPval evaluates frontier AI models on 1,320 real-world economically valuable tasks across 44 occupations and 9 GDP sectors. Claude Opus 4.1 achieves the highest win+tie rate (47.6%) against human expert deliverables, with GPT-5 close behind (39% wins). OpenAI model performance shows roughly linear improvement over time. Increased reasoning effort, context, and scaffolding all predictably improve performance, with prompt-tuning alone adding 5 percentage points to GPT-5 win rates.",
   "red_flags": [
     {
       "flag": "Company evaluating its own products",
       "detail": "All authors are OpenAI employees. OpenAI designed the benchmark, controlled the experimental setup, selected models, and evaluated its own commercial products (GPT-4o, o4-mini, o3, GPT-5) alongside competitors. The paper does not acknowledge this institutional conflict of interest despite it being a major potential source of bias."
     },
     {
       "flag": "Non-equivalent scaffolding across models",
       "detail": "OpenAI models received web search tools, code interpreter, and pre-installed packages. Claude was sampled via the UI. Different models had access to different capabilities, making win rate comparisons reflect scaffolding quality as much as model capability. The scaffold confound is noted but not addressed."
     },
     {
       "flag": "Self-favoring automated grader",
       "detail": "The automated grader is based on GPT-5-high and shows lower agreement with human experts for OpenAI models. The authors acknowledge this bias (Section A.6.2) but the automated grader is still released as the primary grading service."
     },
     {
       "flag": "Linear trend from only 4 data points",
       "detail": "The claim of 'roughly linear' improvement (fig. 6) is based on only 4 OpenAI models. This is too few points to establish a trend, and the claim does not extend to non-OpenAI models."
     },
     {
       "flag": "Incomplete cost comparison",
       "detail": "Cost analyses (Table 2) cover only OpenAI models. Footnote 4 states: 'We were not able to obtain cost estimates for Claude, Gemini, and Grok.' This makes the cost-effectiveness claims incomplete and potentially misleading."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Win rate differences across models (e.g., Claude 47.6% vs GPT-5 39%) are compared without statistical tests. With 220 tasks and 9 comparisons each, sampling variability could account for some differences, but no formal testing is performed."
     }
   ],
   "cited_papers": [
     {
       "title": "Generative ai at work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey Raymond"],
       "year": 2025,
       "relevance": "Empirical study of generative AI productivity effects in the workplace, directly relevant to AI-augmented productivity claims."
     },
     {
       "title": "How people use chatgpt",
       "authors": ["Aaron Chatterji", "Thomas Cunningham", "David J Deming"],
       "year": 2025,
       "relevance": "Large-scale study of real-world AI usage patterns, relevant to understanding AI adoption and economic impact."
     },
     {
       "title": "Clio: Privacy-preserving insights into real-world ai use",
       "authors": ["Alex Tamkin", "Miles McCain", "Kunal Handa"],
       "year": 2024,
       "arxiv_id": "2412.13678",
       "doi": "10.48550/arXiv.2412.13678",
       "relevance": "Privacy-preserving analysis of real-world AI usage data, relevant to understanding AI economic adoption patterns."
     },
     {
       "title": "GPTs are GPTs: An early look at the labor market impact potential of large language models",
       "authors": ["Tyna Eloundou", "Sam Manning", "Pamela Mishkin", "Daniel Rock"],
       "year": 2023,
       "arxiv_id": "2303.10130",
       "relevance": "Foundational paper on LLM labor market exposure using O*NET task framework, directly comparable methodology to GDPval."
     },
     {
       "title": "SWE-Lancer: Can frontier LLMs earn $1 million from real-world freelance software engineering?",
       "authors": ["Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke"],
       "year": 2025,
       "arxiv_id": "2502.12115",
       "relevance": "Benchmark evaluating AI on real-world software engineering freelance tasks with economic value framing."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2020,
       "doi": "10.48550/arXiv.2009.03300",
       "arxiv_id": "2009.03300",
       "relevance": "Major LLM benchmark (MMLU) for evaluating model capabilities across academic domains."
     },
     {
       "title": "GPQA: A graduate-level google-proof q&a benchmark",
       "authors": ["David Rein", "Betty Li Hou", "Asa Cooper Stickland"],
       "year": 2023,
       "doi": "10.48550/arXiv.2311.12022",
       "arxiv_id": "2311.12022",
       "relevance": "Expert-level benchmark requiring graduate knowledge, relevant to AI capability evaluation methodology."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "doi": "10.48550/arXiv.2308.03688",
       "relevance": "Benchmark for evaluating LLMs as agents on diverse tasks, relevant to agentic AI evaluation."
     },
     {
       "title": "LLM evaluators recognize and favor their own generations",
       "authors": ["Arjun Panickssery", "Samuel R. Bowman", "Shi Feng"],
       "year": 2024,
       "arxiv_id": "2404.13076",
       "relevance": "Documents self-preference bias in LLM evaluators, directly relevant to GDPval's automated grader limitations."
     },
     {
       "title": "Humanity's last exam",
       "authors": ["Long Phan"],
       "year": 2025,
       "arxiv_id": "2501.14249",
       "relevance": "Frontier AI capability benchmark testing expert-level reasoning, comparable benchmark evaluation methodology."
     },
     {
       "title": "The simple macroeconomics of AI",
       "authors": ["Daron Acemoglu"],
       "year": 2025,
       "relevance": "Economic analysis of AI's macroeconomic impact, provides theoretical context for GDPval's economic framing."
     },
     {
       "title": "Displacement or complementarity?: The labor market impact of generative AI",
       "authors": ["Wilbur Xinyuan Chen", "Suraj Srinivasan", "Saleh Zakerinia"],
       "year": 2025,
       "relevance": "Studies whether AI displaces or complements human labor, directly relevant to GDPval's economic impact framing."
     }
   ]
 }