ai-research-survey

scan.json (27171B)

---

 {
   "paper": {
     "title": "SKATE, a Scalable Tournament Eval: Weaker LLMs differentiate between stronger ones using verifiable challenges",
     "authors": ["Dewi S. W. Gould", "Bruno Mlodozeniec", "Samuel F. Brown"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.06111",
     "doi": "10.48550/arXiv.2508.06111"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper mentions 'Supplementary Material' containing the codebase and prompts (Section J, Appendix). The prompts are included in the paper, and code is referenced as supplementary."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or repository URL is provided for the generated questions or game results. The paper references supplementary material but does not provide a URL to a data release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or dependency specifications are mentioned. The paper does not describe the environment setup beyond mentioning Python 3 and standard built-ins for COP tasks."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the framework conceptually but does not give commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "TrueSkill provides uncertainty estimates (σ) shown in figures. The p(correct) scoring algorithm (Algorithm 1) explicitly computes standard deviation and terminates when σ ≤ 0.05. Figures show (μ, σ) values."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical significance tests are used to compare model rankings. Differences are assessed via TrueSkill μ values and visual inspection of figures, but no p-values or hypothesis tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "TrueSkill μ scores with specific values are reported (e.g., Sonnet 4: 30.7, Sonnet 3.5: 25.7). p(correct) differences between models are quantified. Table 2 provides correlation coefficients (Pearson, Spearman)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 50 rounds is stated but not justified with any formal analysis. The choice of 6 models is not justified. The stability threshold σ* = 0.05 is described as a 'balance' but no formal justification is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "TrueSkill σ values are reported throughout. The p(correct) scoring includes explicit standard deviation computation (Algorithm 1). Figure 11 shows variance of p(correct) across models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares SKATE rankings against established benchmarks (MMLU-Pro, GPQA Diamond, HLE, BigBench Hard) in Table 2 and reports correlation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparison benchmarks include recent evaluations: MMLU-Pro, GPQA Diamond, HLE, BigBench Hard. Models tested include frontier models (Sonnet 4, GPT-4o, Gemini 2.0 Flash)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Five augmentation strategies are tested (Section 5, Table 1, Figure 2), serving as an ablation of the information available to task-setters. The self-preferencing filter (Figure 4b) is also an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two TrueSkill scoring variants (Relative Pairwise and Absolute Pairwise, Section 4.3), p(correct) values, asking skill vs answering skill (Figure 12), and correlation with external benchmarks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All scoring is automated via code execution sandboxes and TrueSkill. No human assessment of question quality, difficulty, or ranking validity is reported."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The framework generates questions dynamically during gameplay — there is no static dataset to split into train/test. The evaluation is inherently generative, not dataset-based."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (Figures 1, 3, 4, 5), per augmentation strategy (Figure 2), and per question variance level (Figure 11). Asking vs answering skill is separated (Figure 12)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses models failing to generate valid questions (Haiku 3.5 fails to write questions it can answer, Figure 4b). Question examples where models score 0.0 are shown (Figure 11). Section 7 discusses limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Augmentation strategies had 'minimal effect on the final rankings' (Section 5). Models 'may not yet possess the advanced strategic reasoning capabilities to fully exploit' game state (Section 7). Haiku 3.5 fails the self-preferencing filter entirely."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (weaker models score stronger ones, self-preferencing behavior, fine-grained capability differences) are all supported by specific figures and results in the paper (Figures 4, 5, 11)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models 'exhibit a capacity to write self-favoring questions' but the mechanism is observational — correlation between setter identity and performance advantage. No controlled experiment isolates the causal mechanism from, e.g., question-style familiarity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims SKATE is a 'general, scalable evaluation framework' but experiments are limited to 6 models on COP tasks only. The paper acknowledges COP 'does not capture the full range of LLM capabilities' (Section 7) but the title and abstract are broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses that negative results could 'reflect prompt design rather than true model failure', that models with code-execution tools trivially solve COP tasks, and that question diversity may be limited by shared blind spots."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that COP is a proxy testbed and that SKATE is general-purpose. Section 7 and Appendix C acknowledge COP's limitations and discuss the gap between COP performance and general capability evaluation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified as 'GPT-4o', 'Sonnet 3.5', 'Sonnet 4', 'Gemini 2.0 Flash', 'Haiku 3', 'Haiku 3.5'. Only GPT-4o has a date in Table 2 ('2024-11-20'). Claude and Gemini versions lack snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full task-setting prompt for the historical performance strategy is provided in Appendix J. The paper states 'All other prompts used in this work are included in the codebase in the Supplementary Material.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature = 0.7 is stated (Section 5). σ* = 0.05 stability threshold, dthresh = 0.336 similarity threshold, Nrounds = 50, Nattempts = 3, pthresh = 0.55 are all specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The game scaffolding is described in detail: round structure, question validation pipeline (verifiable, distractor-rich, unique), code-execution sandbox, augmentation strategies, retry logic (3 attempts with feedback). Section 4 and appendices cover the full scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Question filtering pipeline is documented: verification via code sandbox, distractor generation, uniqueness check via embedding similarity (threshold 0.336), and the adaptive scoring algorithm (Algorithm 1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) contains substantive limitations discussion spanning multiple paragraphs, covering prompt sensitivity, tool-augmented models, narrow model set, and COP scope."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats: 'prompting strategy may not reliably elicit the intended game-playing behavior' (prompt validity), models with code execution trivially solve COP (construct validity), narrow model set may miss patterns, shared blind spots in COP questions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: COP 'does not capture the full range of LLM capabilities', experiments cover 'a narrow set of models', restricting to MCQ 'inherently restricts the evaluation space, for example by excluding open-ended responses' (Section 3)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (question sets, model responses, p(correct) scores per question) is released. Supplementary material is mentioned but no URL is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation process is thoroughly described: LLMs generate COP questions, questions are validated via sandbox execution, scored via Algorithm 1, and clustered via embedding similarity. The full pipeline is documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Models are the subjects and their selection is simply described (6 frontier LLMs). Standard benchmark — NA."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from question generation → validation (verifiable, distractor-rich) → uniqueness filtering → scoring (Algorithm 1) → TrueSkill ranking is fully documented across Sections 3-4 and appendices."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'We are grateful to the Supervised Program for Alignment Research for facilitating collaboration and providing financial support.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Alan Turing Institute, University of Cambridge / Max Planck Institute, and Independent. None are affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Supervised Program for Alignment Research) is an alignment research program, not a model provider. They have no direct financial stake in which model ranks highest."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 6 models used. This matters because COP questions could overlap with training data patterns."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues that dynamically generated questions reduce contamination risk compared to static benchmarks: 'Game-based evaluations provide advantages including... reduced data contamination' (Section 2). However, no formal overlap analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core design addresses contamination by generating novel questions each game rather than using static benchmarks. Section 2 explicitly positions this as an advantage over contamination-prone static evaluations."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix K provides token counts (~130,000 input tokens per asking model, ~150 tokens per question) and estimates scaling with N players."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix K states: 'All experiments in this paper cost less than $500.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. The game is run once per configuration with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of game iterations is not explicitly stated for the main experiment. The augmentation strategy experiment uses 'five iterations of SKATE' but the main 6-player game appears to be a single run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Key parameters (σ* = 0.05, dthresh = 0.336, Nrounds = 50, temperature = 0.7) are presented without systematic search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of historical performance as the augmentation strategy is justified qualitatively ('balance between informativeness and computational cost') but not through systematic comparison on a validation set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The framework inherently addresses this: models evaluate each other rather than the authors evaluating their own system. The authors are not affiliated with any model provider."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of very different sizes and costs are compared (Haiku 3 vs Sonnet 4) without discussing whether performance differences reflect compute differences. No matched-compute comparison is attempted."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C extensively discusses what COP actually tests and its relationship to broader capabilities. The paper acknowledges COP 'does not capture the full range of LLM capabilities' and discusses construct validity at length."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models use the same scaffolding (same prompt structure, same game rules, same scoring). The augmentation strategy experiments use identical strategies across models. No scaffold confound exists in the comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's design inherently addresses temporal leakage — questions are generated fresh by LLMs during the game, not drawn from existing datasets. This is positioned as a key advantage over static benchmarks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MCQ distractor options could leak information about the correct answer, or whether the question format itself provides hints not present in real-world code understanding tasks."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Questions generated by the same model may share systematic patterns (style, difficulty, topic). The uniqueness filter uses embedding similarity but does not address deeper structural non-independence. Not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No formal leakage detection method is applied. The dynamic question generation is argued to reduce contamination conceptually, but no canary strings, membership inference, or n-gram overlap analysis is used."
       }
     }
   },
   "claims": [
     {
       "claim": "Weaker models can reliably score and differentiate between stronger models using SKATE-generated questions.",
       "evidence": "Figure 5 shows four weaker models generating questions that differentiate Sonnet 3.5 and Sonnet 4 when added. Rankings remain stable across panels (a)-(d).",
       "supported": "moderate"
     },
     {
       "claim": "LLMs exhibit self-preferencing behavior, generating questions that favor their own capabilities.",
       "evidence": "Figure 4b shows that after filtering to questions with p(correct) > 0.55, most models perform best on their own questions (diagonal maxima in the heatmap).",
       "supported": "moderate"
     },
     {
       "claim": "SKATE automatically surfaces fine-grained capability differences between models.",
       "evidence": "Figure 11 shows questions with high p(correct) variance, revealing model-specific strengths (e.g., Gemini 2.0 Flash scoring 1.0 where Sonnet 3.5 scores 0.0 on a specific question).",
       "supported": "moderate"
     },
     {
       "claim": "Rankings are stable to the sequential addition of new models.",
       "evidence": "Figure 10 (Appendix F) shows that adding Sonnet 3.5 then Sonnet 4 (and vice versa) preserves relative TrueSkill ordering.",
       "supported": "moderate"
     },
     {
       "claim": "SKATE correlates well with established benchmarks like MMLU-Pro and GPQA Diamond.",
       "evidence": "Table 2 shows Spearman rank correlations of 0.92 (MMLU-Pro) and 0.90 (GPQA Diamond), but weaker correlations with HLE (0.58) and BigBench Hard (0.30).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SKATE introduces a peer-challenge evaluation framework where LLMs compete by generating and solving verifiable code-output-prediction tasks, using TrueSkill ranking. The framework demonstrates that weaker models can differentiate stronger ones, that LLMs exhibit self-preferencing behavior in question generation, and that dynamically generated questions can surface fine-grained capability differences. Rankings show moderate-to-strong correlation with established benchmarks (Spearman 0.92 with MMLU-Pro, 0.90 with GPQA Diamond) across 6 frontier models.",
   "red_flags": [
     {
       "flag": "Very narrow model set",
       "detail": "Only 6 models tested, all from 3 providers (Anthropic, OpenAI, Google). No open-source models. Claims of 'scalable evaluation' rest on a very small sample."
     },
     {
       "flag": "Single-run main experiment",
       "detail": "The main 6-player game appears to be run once. Game outcomes depend on stochastic question generation, but no repeated runs are reported to assess stability of the main result."
     },
     {
       "flag": "COP-only proof of concept generalized broadly",
       "detail": "All experiments use only code-output-prediction tasks, but claims extend to 'general, scalable evaluation.' Appendices B and C discuss other task types but none are empirically tested."
     },
     {
       "flag": "Weak correlation with some benchmarks",
       "detail": "Spearman correlation with BigBench Hard is only 0.30, yet this is not treated as concerning. The paper selectively emphasizes the strong correlations (MMLU-Pro, GPQA Diamond)."
     }
   ],
   "cited_papers": [
     {
       "title": "ZeroSumEval: An Extensible Framework For Scaling LLM Evaluation with Inter-Model Competition",
       "authors": ["Hisham A. Alyahya", "Haidar Khan", "Yazeed Alnumay", "M. Saiful Bari", "Bülent Yener"],
       "year": 2025,
       "arxiv_id": "2503.10673",
       "relevance": "Directly related LLM competition-based evaluation framework with 7 static game types."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT measuring AI impact on developer productivity, cited as criticism of unrealistic benchmarks."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex/HumanEval — foundational code generation benchmark referenced as prior work."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major agentic coding benchmark used as comparison point for LLM evaluation approaches."
     },
     {
       "title": "AI Agents That Matter",
       "authors": ["Sayash Kapoor", "Benedikt Stroebl", "Zachary S. Siegel", "Nitya Nadgir", "Arvind Narayanan"],
       "year": 2024,
       "relevance": "Key meta-research paper on benchmark validity gaps and evaluation methodology for AI agents."
     },
     {
       "title": "Red Teaming Language Models with Language Models",
       "authors": ["Ethan Perez", "Saffron Huang"],
       "year": 2022,
       "arxiv_id": "2202.03286",
       "relevance": "LLM-based automated red-teaming, foundational work on models evaluating models."
     },
     {
       "title": "Discovering Language Model Behaviors with Model-Written Evaluations",
       "authors": ["Ethan Perez", "Sam Ringer"],
       "year": 2022,
       "arxiv_id": "2212.09251",
       "relevance": "LLM-generated evaluation datasets for probing model behaviors, direct predecessor approach."
     },
     {
       "title": "Weak-to-Strong Generalization: Eliciting Strong Capabilities With Weak Supervision",
       "authors": ["Collin Burns", "Pavel Izmailov"],
       "year": 2023,
       "arxiv_id": "2312.09390",
       "relevance": "Scalable oversight work on weaker models supervising stronger ones, directly related to SKATE's weak-scores-strong claim."
     },
     {
       "title": "JuStRank: Benchmarking LLM Judges for System Ranking",
       "authors": ["Ariel Gera", "Odellia Boni"],
       "year": 2025,
       "arxiv_id": "2412.09569",
       "relevance": "Benchmarks LLM judge biases, directly motivating SKATE's verifiable-task design to avoid judge bias."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2025,
       "arxiv_id": "2406.15877",
       "relevance": "Code generation benchmark used in Table 2 comparison (BigBench Hard)."
     },
     {
       "title": "Automated Capability Discovery via Foundation Model Self-Exploration",
       "authors": ["Cong Lu", "Shengran Hu", "Jeff Clune"],
       "year": 2025,
       "arxiv_id": "2502.07577",
       "relevance": "LLM self-exploration for capability discovery, related approach using LLMs to generate evaluation tasks."
     },
     {
       "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark",
       "authors": ["Yubo Wang"],
       "year": 2024,
       "arxiv_id": "2406.01574",
       "relevance": "Benchmark used for correlation analysis with SKATE rankings (Spearman 0.92)."
     }
   ]
 }