ai-research-survey

scan.json (26726B)

---

 {
   "paper": {
     "title": "The AI Productivity Index: APEX-v1-extended",
     "authors": [
       "Bertie Vidgen",
       "Abby Fennelly",
       "Evan Pinnix",
       "Julien Benchek",
       "Daniyal Khan",
       "Zach Richards",
       "Austin Bridges",
       "Calix Huang",
       "Kanishka Sahu",
       "Abhishek Kottamasu",
       "Bo Ma",
       "Ben Hunsberger",
       "Isaac Robinson",
       "Akul Datta",
       "Chirag Mahapatra",
       "Dominic Barton",
       "Cass R. Sunstein",
       "Eric Topol",
       "Brendan Foody",
       "Osvald Nitski"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.25721",
     "doi": "10.48550/arXiv.2509.25721"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our eval harness is available on Github' (Section 1) with a reference to 'Mercor on Github'. A GitHub repository is cited."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases 'n = 25 non-benchmark example cases per job (n = 100 total)' as 'apex-v1-devset' on Hugging Face with a CC-BY license (Section 1). The main n=400 heldout set is intentionally kept closed for benchmark integrity."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency versions are mentioned. The paper does not describe how to set up the evaluation harness environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub repository and devset are referenced, no step-by-step reproduction instructions are provided in the paper. There is no 'Reproducing Results' section or commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for model scores. Table 3 shows z-scores with 95% CIs (e.g., 'GPT 5 (High) 0.50, 0.42–0.58'), and Figure 1 displays confidence intervals visually."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Friedman omnibus test is used (p < 0.000001) for overall model differences, and paired t-tests with Bonferroni correction are used for pairwise comparisons (Section 4.2): '35 of the 45 pairwise comparisons (78%) remain statistically significant at the adjusted threshold (p = 0.001).'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Z-scores are reported as a measure of effect size (Table 3), showing how many standard deviations each model's performance is above/below the mean. Raw percentage differences are also provided with baseline context (e.g., GPT 5 at 67.0% vs. second-best at 64.3%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark was increased from n=200 to n=400 and runs increased from 3 to 8, but no power analysis or formal justification for these specific numbers is provided. No discussion of whether n=100 per job is sufficient for the granularity of claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation across the 8 runs is reported: 'the mean standard deviation over the eight runs is 9.05 percentage points, ranging from 8.25 (Claude Opus 4.5) to 9.5 (Grok 4)' (Section 4.1)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "10 frontier models are compared against each other across four job domains (Table 2). This is primarily a leaderboard paper, and the inter-model comparison serves as the baseline structure."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All 10 models tested are frontier models from late 2025: GPT 5, GPT 5.1, Gemini 3 Pro, Gemini 2.5 Pro/Flash, Claude Opus 4.5/4.1, Sonnet 4.5, o3, and Grok 4. These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not investigate which components of the benchmark design (rubric structure, source document inclusion, prompt format) contribute to the observed performance differences."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both raw mean percentage scores and z-scores (Table 2, Table 3), which capture different aspects of performance — raw scores for absolute capability and z-scores for relative standing adjusting for task difficulty."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All grading is done by a single LM judge (Gemini 2.5 Flash). No human evaluation of model outputs is reported. The paper acknowledges switching from a panel of LM judges to a single LM judge but does not validate against human judgments for the extended benchmark."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly maintains a held-out evaluation set of n=400 cases: 'The n = 400 cases will remain a closed heldout dataset for rigorous evaluation of frontier models' (Section 1). The devset (n=100) is separate and open-sourced."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-job breakdowns (Investment banking, Law, Management consulting, Medicine) for all 10 models, showing meaningful performance variation across domains."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis is provided. The paper reports aggregate scores but does not examine specific cases where models failed, what types of tasks are hardest, or why certain rubric criteria are more challenging."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that 'APEX-v1-extended shows that frontier models still have substantial limitations when performing typical professional tasks' (Abstract). It also highlights that some models (GPT 5.1, Gemini 2.5 Flash, Sonnet 4.5) score 10+ percentage points lower on the devset, and that Investment banking remains particularly difficult."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT 5 (Thinking = High) is the top model at 67.0%, which matches Table 2. The claim that 'frontier models still have substantial limitations' is supported by the observation that no model exceeds 67% overall."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims about thinking mode: models are tested with 'Thinking = High' as if this setting causes better performance, and the z-score analysis implies Opus 4.5 is 'particularly strong on the hardest tasks' (Section 4.2). No controlled analysis isolates the effect of thinking mode vs. model architecture."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'The AI Productivity Index' and framing suggest this measures general AI productivity for professional work, but the benchmark covers only 4 specific US-centric professional roles with n=100 cases each. The paper does not bound its claims to these specific jobs and settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example: could differences be driven by the LM judge's biases rather than genuine model capability differences? Could source document parsing affect some models more than others? The paper does not address these possibilities."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT 5 (Thinking = High)', 'Gemini 3 Pro (Thinking = High)', 'Opus 4.5 (On)', etc. No API version identifiers, snapshot dates, or specific model IDs are provided. The paper states 'Responses were collected at the end of November 2025' but gives no model version strings."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The LM judge prompt is provided in full in Appendix A. The instruction appended to every prompt is provided in full in Appendix B. An example prompt and rubric are shown in Figure 3. The actual task prompts are in the released devset."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only temperature is mentioned: 'If temperature can be configured, we set it to 0.7' (Section 3.1). No other hyperparameters (top-p, max tokens, etc.) are reported. The conditional 'if temperature can be configured' leaves ambiguity about which models used what settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive single-turn prompts with attached source documents and produce single-turn responses."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Source document processing is described: 'Sources were parsed using Reducto and appended to the prompt in the context window' (Section 2.3). The production process is shown in Figure 2. The quality control pipeline (expert creation, review, approval/rejection) is described in Section 2.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no Limitations, Threats to Validity, or Discussion section in the paper. The paper goes from Results (Section 4) directly to Acknowledgments (Section 5) and References."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. Issues such as LM judge reliability, rubric quality variation, source document parsing fidelity, or benchmark representativeness are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what professional tasks are excluded, or what limitations exist in generalizing from these 4 jobs to broader AI productivity claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The main benchmark data (n=400 heldout set) is not released. Only the devset (n=100) is open-sourced. Raw model responses, judge outputs, and per-criterion scores are not publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data collection procedure in detail: expert selection from the Mercor platform, interview process, assessment, prompt creation by experts based on their day-to-day work, rubric creation, and multiple rounds of review (Sections 2.1-2.4)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Expert recruitment is described in Section 2.1: 'From the Mercor platform we sourced experts with extensive professional experience, and prioritized experts with data labeling experience.' The interview process (30-45 min interview, 1-2 hour assessment) and ongoing quality management are described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The production process is shown in Figure 2 and described across Sections 2.1-2.4, covering expert selection, prompt creation, review, rubric creation, and source document processing. The pipeline from APEX-v1.0 to APEX-v1-extended is documented (recalibration of original 50 + 50 new cases per job)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. The Acknowledgments section (Section 5) thanks annotators and advisors but does not mention funding sources. Mercor is a for-profit company, and the funding for this work is not stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: most authors are affiliated with Mercor (superscript 1), with Cass R. Sunstein at Harvard Law School (superscript 2) and Eric Topol at The Scripps Research Institute (superscript 3)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Mercor is an AI hiring/talent platform that benefits commercially from AI productivity measurement. The benchmark is branded with Mercor's name and hosted on their website (mercor.com/apex). The funder (Mercor) has a direct commercial interest in demonstrating that AI productivity can be measured and benchmarked, making them non-independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Given that Mercor is a commercial company that could benefit from the benchmark's adoption and visibility, the absence of any disclosure is a gap."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 10 models tested. The paper only mentions that responses were collected 'at the end of November 2025' but does not discuss when models' training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. Given that APEX-v1.0 was published in September 2025 and the devset is open-sourced, models trained after that date could have seen the benchmark design or similar tasks. This is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination risk through design: 'The n = 400 cases will remain a closed heldout dataset for rigorous evaluation of frontier models' (Section 1). Keeping the evaluation set private is a direct contamination mitigation, though the paper does not explicitly discuss contamination as a concept."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper, not a human subjects study. The experts who created the benchmark are contributors/annotators, not research participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. Expert annotators are paid contractors creating benchmark content, not research subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. Some expert characterization is provided (137 experts, 7+ years mean experience) but this describes benchmark creators, not research participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied as research subjects. Expert selection criteria are described but in the context of benchmark construction quality, not as a human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects experimental study. No participant randomization to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects experimental study. No blinding procedures are applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. While the paper mentions offboarding underperformers among experts, this is workforce management, not participant attrition in a study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported. Each model is run 8 times on 400 cases (3,200 runs per model, 32,000 total), but the cost of these API calls is not mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The total API spend, compute hours, or evaluation time is not reported despite running 10 models x 400 cases x 8 runs = 32,000 model inferences plus LM judge evaluations."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT 5 (Thinking = High) is the top performing model on APEX-v1-extended with a mean score of 67.0%.",
       "evidence": "Table 2 shows GPT 5 (Thinking = High) at 67.0% overall, followed by Gemini 3 Pro (High) at 64.3% and Grok 4 at 63.5%. Z-scores in Table 3 confirm GPT 5 leads at 0.50 vs. second-place Opus 4.5 at 0.28.",
       "supported": "strong"
     },
     {
       "claim": "Differences between models' scores are statistically significant overall.",
       "evidence": "Friedman omnibus test (p < 0.000001) and paired t-tests with Bonferroni correction showing 35/45 pairwise comparisons significant at p = 0.001 (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Frontier models still have substantial limitations when performing typical professional tasks.",
       "evidence": "The best model achieves only 67.0% overall, with scores as low as 51.4% for the weakest model. Performance varies across jobs with Investment banking being hardest (top score 63.0%). However, 'substantial limitations' is somewhat vague without a defined threshold.",
       "supported": "moderate"
     },
     {
       "claim": "Model ranks are consistent between the heldout benchmark and the open-source devset.",
       "evidence": "Table 4 shows ranks are largely preserved between the two sets, with GPT 5 ranked #1 on both and only minor swaps among adjacent models. However, absolute score differences range from 1.2 to 11.9 percentage points.",
       "supported": "moderate"
     },
     {
       "claim": "Claude Opus 4.5 (Thinking = On) is particularly strong on the hardest tasks.",
       "evidence": "Section 4.2 states Opus 4.5 jumps from 5th to 2nd in z-score ranking, suggesting strong performance relative to task difficulty. However, no formal analysis of performance conditional on task difficulty is presented beyond the z-score reranking.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "APEX-v1-extended benchmarks 10 frontier AI models on 400 realistic professional tasks across investment banking, management consulting, law, and medicine. GPT 5 (Thinking = High) leads the leaderboard at 67.0%, with statistically significant differences between most model pairs after Bonferroni correction. Models perform worst on investment banking tasks and best on law tasks. The benchmark reveals that even top frontier models fail to complete roughly one-third of professional task criteria, suggesting meaningful gaps between AI capability and professional work requirements.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product category",
       "detail": "Mercor is an AI-powered talent/hiring platform that commercially benefits from the existence and adoption of AI productivity benchmarks. The benchmark is branded and hosted on Mercor's website (mercor.com/apex). This creates a structural conflict of interest where positive benchmark attention benefits the company regardless of which model wins."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or discussion section whatsoever. It goes directly from Results to Acknowledgments. This is a significant methodological gap for a paper making claims about AI productivity across professional domains."
     },
     {
       "flag": "Single LM judge without human validation",
       "detail": "All grading is performed by a single model (Gemini 2.5 Flash). The paper acknowledges switching from a panel of judges to a single judge for efficiency but provides no human-agreement validation for the extended benchmark. LM judge biases could systematically affect the leaderboard rankings."
     },
     {
       "flag": "Overbroad framing vs narrow evidence",
       "detail": "The paper is titled 'The AI Productivity Index' suggesting comprehensive measurement of AI economic productivity, but it tests only 4 US-centric professional roles with 100 constructed cases each. The gap between the framing (general AI productivity) and the evidence (specific professional task completion in 4 jobs) is significant."
     },
     {
       "flag": "No model version identifiers",
       "detail": "Models are identified only by marketing names without API versions or snapshot dates. Model behavior changes across versions, making these results difficult to reproduce or verify independently."
     },
     {
       "flag": "Closed heldout set prevents independent verification",
       "detail": "The primary evaluation data (n=400) is kept private. While this protects against contamination, it also means the leaderboard results cannot be independently verified. Only Mercor can run the benchmark, creating a gatekeeping dynamic."
     }
   ],
   "cited_papers": [
     {
       "title": "Dynabench: Rethinking Benchmarking in NLP",
       "authors": ["Douwe Kiela", "Max Bartolo", "Yixin Nie"],
       "year": 2021,
       "relevance": "Foundational work on dynamic benchmarking methodology in NLP, relevant to understanding benchmark design principles."
     },
     {
       "title": "GDPVal: Evaluating AI's Economic Value",
       "authors": ["OpenAI"],
       "year": 2025,
       "relevance": "OpenAI's benchmark for evaluating AI's economic value, directly comparable to APEX in measuring real-world AI productivity."
     },
     {
       "title": "Introducing BigLaw Bench: A Public-Facing Dataset for Evaluating LLMs on Legal Tasks",
       "authors": ["Harvey Team"],
       "year": 2024,
       "relevance": "Domain-specific legal benchmark from Harvey AI, relevant as a comparison benchmark for professional AI evaluation."
     },
     {
       "title": "The AI Productivity Index (APEX)",
       "authors": ["Bertie Vidgen", "Abby Fennelly", "Evan Pinnix"],
       "year": 2025,
       "arxiv_id": "2509.25721",
       "relevance": "Original APEX-v1.0 paper; this extended version builds on it with expanded data and improved grading."
     },
     {
       "title": "Estimating AI Productivity Gains from Claude Conversations",
       "authors": ["Alex Tamkin", "Peter McCrory"],
       "year": 2025,
       "relevance": "Anthropic's study on AI productivity gains, directly relevant to AI productivity measurement methodology."
     },
     {
       "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health",
       "authors": ["Rahul K. Arora", "Jason Wei"],
       "year": 2025,
       "relevance": "Domain-specific health/medical LLM benchmark, relevant as a comparable professional-domain evaluation."
     },
     {
       "title": "Reality Check: A New Evaluation Ecosystem is Necessary to Understand AI's Real World Effects",
       "authors": ["Reva Schwartz", "Rumman Chowdhury"],
       "year": 2025,
       "relevance": "Argues for new evaluation approaches to measure real-world AI effects, directly relevant to the AI benchmarking methodology discussion."
     },
     {
       "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
       "authors": ["Giulio Starace", "Oliver Jaffe"],
       "year": 2025,
       "relevance": "Benchmark evaluating AI's ability to replicate research, relevant to AI capability evaluation methodology."
     },
     {
       "title": "A Survey on LLM-as-a-Judge",
       "authors": ["Jiawei Gu", "Xuhui Jiang"],
       "year": 2025,
       "relevance": "Survey on LLM-as-judge methodology, directly relevant since APEX uses an LM judge for grading."
     },
     {
       "title": "Miracle or Myth? Assessing the Macroeconomic Productivity Gains from Artificial Intelligence",
       "authors": ["Francesco Filippucci", "Peter Gal", "Matthias Schief"],
       "year": 2024,
       "relevance": "OECD analysis of macroeconomic AI productivity gains, relevant to the broader AI productivity measurement context."
     },
     {
       "title": "Toward an Evaluation Science for Generative AI Systems",
       "authors": ["Laura Weidinger", "Inioluwa Deborah Raji"],
       "year": 2025,
       "relevance": "Proposes evaluation science for generative AI, relevant to benchmark design methodology and evaluation rigor."
     },
     {
       "title": "LMUnit: Fine-grained Evaluation with Natural Language Unit Tests",
       "authors": ["Jon Saad-Falcon", "Rajan Vivek"],
       "year": 2024,
       "relevance": "Proposes fine-grained evaluation using natural language unit tests, directly relevant to APEX's rubric-based criterion approach."
     }
   ]
 }