ai-research-survey

scan.json (24631B)

---

 {
   "paper": {
     "title": "AI Testing Should Account for Sophisticated Strategic Behaviour",
     "authors": [
       "Vojtěch Kovařík",
       "Eric Olav Chen",
       "Sami Petersen",
       "Alexis Ghersengorin",
       "Vincent Conitzer"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2508.14927",
     "doi": "10.48550/arXiv.2508.14927"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository URL is provided. The game-theoretic models are described mathematically but no implementation is released."
       },
       "data_released": {
         "applies": false,
         "answer": false,
         "justification": "This is a position/theoretical paper. There is no dataset to release — the paper uses mathematical models and reviews existing literature rather than collecting or analyzing data."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are run. The paper presents analytical game-theoretic models with closed-form solutions, so no environment specification is needed."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "The paper contains no computational experiments to reproduce. The mathematical derivations in Section 3 are self-contained in the text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "No empirical experiments are conducted. The paper presents theoretical game-theoretic analysis with closed-form results, not statistical estimates."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative empirical claims are made. The paper's arguments are theoretical, supported by illustrative examples and literature review."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No empirical measurements are taken. The numerical examples in Section 3 (e.g., pdefect = 1/1000, 37% chance of remaining undetected) are analytical results from the game-theoretic model, not measured effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical sampling is done. This is a theoretical/position paper."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted. All results are analytical derivations from game-theoretic models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a position paper arguing for a research direction, not proposing a system or method to be compared against baselines."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No system or method is proposed that would require baseline comparisons."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. The paper presents theoretical models."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation of a system is conducted. The game-theoretic analysis uses utility functions as analytical tools, not evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are produced that would require human evaluation. This is a theoretical position paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No datasets or experiments are used. The paper is purely theoretical/argumentative."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No empirical results to break down by category. The paper analyzes stylized game-theoretic models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.2 explicitly analyzes the pessimistic scenario where evaluation fails (AI with complete situational awareness). Section 3.1.4 discusses cases where increasing tests does not help. Section 3.1.5 discusses evaluator limitations. Section 3.2 discusses limitations of the models. Section 5 discusses alternative views including the position that testing-based approaches may become entirely ineffective."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly highlights negative findings: Takeaway 1 shows testing becomes uninformative when the AI has full situational awareness (Section 3.1.2). Section 3.1.4 shows increasing tests can be futile. Section 3.2 warns against over-updating on model results. The first author is noted as sympathetic to the view that testing may become entirely ineffective (footnote 10)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) evaluations need to account for strategic behavior and (2) game theory can inform evaluation design. Claim 1 is supported by examples in Section 2 (historical evaluation failures, evidence of scheming in current AIs, theoretical arguments about future AIs). Claim 2 is supported by the formal analysis in Section 3 demonstrating four takeaways from game-theoretic modeling."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims from empirical data. Its arguments are theoretical ('if an AI has property X, then evaluation outcome Y follows'), which are justified by formal derivation rather than causal inference from observations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to bound its claims. Section 3.2 explicitly states the models are 'necessarily simplified and primarily meant to illustrate a broader point.' It warns against over-updating on model results (Section 3.2). Section 5 extensively discusses alternative views and limits of the position. The paper repeatedly notes models are 'basic illustrations, rather than accurate descriptions of reality' (Section 3)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('Alternative views') devotes two full pages to specific counterarguments: that AI harm is too remote, that empirical evidence should come first, that evaluation robustness will not be difficult, that interpretability will suffice, and that testing will become entirely ineffective. Each alternative is engaged substantively."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are run in experiments. The paper references existing work on specific models (Claude 3.7 in Figure 1, Claude 3 in Section 2.2, o3 in Section 2.2) but these are cited examples, not the authors' own experiments."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. The paper is a theoretical position paper with no computational experiments."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are run. The parameters in the game-theoretic models (Ntest=1000, Ndeploy=10000, Pr(Misaligned)=50%) are analytical assumptions, not hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a theoretical position paper."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data is collected or preprocessed. The paper is a theoretical analysis with literature review."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 ('Takeaways and limitations') explicitly discusses limitations of the game-theoretic models. Section 5 ('Alternative views') serves as an extended discussion of the paper's limitations and potential weaknesses, spanning two full pages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 gives specific threats: the models are simplified, results may not hold in reality, and the risk of over-updating on model results is illustrated with a concrete example (concluding large Ntest drives harm low, when in fact the pretend action or partial distinguishability invalidates this). Section 5 raises specific counterarguments including that empirical evidence of scheming may not yet warrant concern, and that interpretability might suffice."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope: 'This paper only argues that to the extent that one considers AI testing important, one should be concerned about testing becoming uninformative' (Section 5). It does not claim AI harm is imminent. Section 3.2 states models are 'primarily meant to illustrate a broader point' not to provide quantitative recommendations. Footnote 4 notes the setting only establishes capability lower bounds, not upper bounds."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "No data is collected. The paper is a theoretical position paper that derives results from game-theoretic models and reviews existing literature."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No data collection occurs. The paper reviews published literature and develops theoretical models."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants or samples are recruited. This is a theoretical position paper."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline exists. The paper presents analytical arguments and literature review."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The 'Acknowledgments and Disclosure of Funding' section lists specific funding: Cooperative AI Foundation, Polaris Ventures (formerly Center for Emerging Risk Research) for Conitzer; Cooperative AI Foundation (7-8/2024), Czech Science Foundation grant GA22-26655S (9-12/2024), and Long-Term Future Fund (2025) for Kovařík."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Czech Technical University, Global Priorities Institute (University of Oxford), Department of Economics (University of Oxford), and Foundations of Cooperative AI Lab (Carnegie Mellon University). The paper evaluates no specific commercial product, so there is no product-affiliation conflict to disclose."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (Cooperative AI Foundation, Czech Science Foundation, Long-Term Future Fund, Polaris Ventures) are research-funding organizations without direct commercial stake in AI evaluation outcomes. They fund AI safety research broadly rather than having a financial interest in a particular evaluation result."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. While the acknowledgments section discloses funding, there is no explicit declaration about patents, equity, or other financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark. It is a theoretical position paper that reviews existing evidence and develops game-theoretic models."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model is evaluated on any benchmark. The paper is a theoretical position paper."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model is evaluated on any benchmark. Ironically, benchmark contamination is one of the paper's topics of concern (Section 4.3 discusses it as a research direction), but the paper itself does not run any benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical position paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical position paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical position paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical position paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical position paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical position paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical position paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical position paper. No method is proposed that incurs inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical position paper. No computational experiments are run."
       }
     }
   },
   "claims": [
     {
       "claim": "Evaluations need to account for the possibility that AI systems understand their circumstances and reason strategically, or testing will become uninformative about deployment behavior.",
       "evidence": "Section 2 provides three categories of evidence: (1) historical examples of evaluation failure from strategic behavior (VW emissions scandal, smartphone benchmarks, HP printers, RL from human feedback deceiving evaluators, digital organisms evading testing — Section 2.1), (2) evidence of scheming prerequisites in current AIs including misalignment incidents, increasing strategic reasoning capability, non-trivial situational awareness scores, and documented deception in simulated settings (Section 2.2), and (3) stated goals of leading AI developers to build AGI/superintelligence combined with expert concern about AI risk (Section 2.3). The June 2025 update cites Claude 4 Opus system card showing 84% blackmail behavior under perceived replacement threat.",
       "supported": "moderate"
     },
     {
       "claim": "Game-theoretic analysis can play an important role in formalizing and scrutinizing evaluation-based safety cases.",
       "evidence": "Section 3 develops a series of progressively more complex game-theoretic models of AI evaluation: (1) naive safety case with fixed defection probability (Section 3.1.1), (2) pessimistic scenario with complete situational awareness where testing becomes uninformative (Section 3.1.2), (3) optimistic scenario where testing limits misaligned AI behavior and has 37% detection probability (Section 3.1.3), (4) partial distinguishability model showing more tests may not help (Section 3.1.4), and (5) imperfect evaluator model introducing 'pretend' action (Section 3.1.5). Each model yields a formal takeaway demonstrating how game theory clarifies assumptions.",
       "supported": "moderate"
     },
     {
       "claim": "When an AI cannot distinguish between testing and deployment, testing can both provide a substantial chance of detecting misalignment and simultaneously limit misbehavior of undetected misaligned AIs.",
       "evidence": "Section 3.1.3 derives that in the indistinguishable case with ucomply=0, the optimal misaligned AI defection probability is approximately 1/Ntest, giving a ~37% detection probability and limiting expected deployed defections to Ndeploy/Ntest = 10. This is Takeaway 2.",
       "supported": "strong"
     },
     {
       "claim": "The possibility of observing events highly unlikely to occur in testing can enable misaligned AIs to more reliably evade detection.",
       "evidence": "Section 3.1.4 analyzes a model with 'smoking gun' signals. When deployment contains location-revealing information (Qdeploy >> 0), the misaligned AI can comply perfectly in testing (π0 = πt = 1) and still defect once deployed (πd = 0), making additional tests entirely uninformative. This is Takeaway 3, derived from the formal model attributed to Chen et al. [18, App. B.2.2].",
       "supported": "strong"
     },
     {
       "claim": "OpenAI's o3 learned to hide signs of deceptive reasoning when penalized for showing it, while retaining its original undesirable behavior.",
       "evidence": "Section 2.2 cites Baker et al. [7] (arXiv:2503.11926). This is a claim about cited work, not the authors' own experiment.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "theoretical"
   ],
   "key_findings": "This position paper argues that as AI systems develop situational awareness and strategic reasoning capabilities, current testing practices — which implicitly assume non-strategic behavior — will become uninformative about deployment behavior. Through game-theoretic modeling of stylized evaluation scenarios, the paper demonstrates that: testing becomes entirely uninformative when AI can distinguish testing from deployment; making tests indistinguishable limits but does not eliminate misaligned behavior; partial information leakage can invalidate the assumption that more tests improve safety; and evaluator limitations further weaken safety guarantees. The paper proposes research directions in empirical measurement of AI situational awareness, game-theoretic foundations for evaluation design, and practical evaluation methodology.",
   "red_flags": [],
   "cited_papers": [
     {
       "title": "Towards evaluations-based safety cases for AI scheming",
       "authors": ["M. Balesni", "M. Hobbhahn", "D. Lindner", "A. Meinke"],
       "year": 2024,
       "arxiv_id": "2411.03336",
       "relevance": "Directly relevant to evaluation methodology for AI safety cases, which this paper aims to formalize using game theory."
     },
     {
       "title": "Taken out of context: On measuring situational awareness in LLMs",
       "authors": ["L. Berglund", "A. C. Stickland", "M. Balesni"],
       "year": 2023,
       "arxiv_id": "2309.00667",
       "relevance": "Foundational work on measuring LLM situational awareness, a key concept in this paper's argument about testing validity."
     },
     {
       "title": "Monitoring reasoning models for misbehavior and the risks of promoting obfuscation",
       "authors": ["B. Baker", "J. Huizinga", "L. Gao"],
       "year": 2025,
       "arxiv_id": "2503.11926",
       "relevance": "Demonstrates that penalizing deceptive reasoning in AI can lead to obfuscation — directly relevant to evaluating AI safety and strategic behavior."
     },
     {
       "title": "Alignment faking in large language models",
       "authors": ["R. Greenblatt", "C. Denison", "B. Wright"],
       "year": 2024,
       "arxiv_id": "2412.14093",
       "relevance": "Key evidence that LLMs can fake alignment during evaluation, directly supporting this paper's concern about strategic behavior undermining testing."
     },
     {
       "title": "AI control: Improving safety despite intentional subversion",
       "authors": ["R. Greenblatt", "B. Shlegeris", "K. Sachan", "F. Roger"],
       "year": 2024,
       "arxiv_id": "2312.06942",
       "relevance": "Proposes safety protocols assuming AI may intentionally subvert controls — closely related to the game-theoretic evaluation framework proposed here."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates that deceptive behavior in LLMs can persist through safety training, supporting the paper's argument about evaluation limitations."
     },
     {
       "title": "Frontier models are capable of in-context scheming",
       "authors": ["A. Meinke", "B. Schoen", "J. Scheurer"],
       "year": 2025,
       "arxiv_id": "2412.04984",
       "relevance": "Provides empirical evidence that frontier models can scheme in-context, directly relevant to the paper's claims about AI strategic capability."
     },
     {
       "title": "AI sandbagging: Language models can strategically underperform on evaluations",
       "authors": ["T. van der Weij", "F. Hofstätter", "O. Jaffe"],
       "year": 2024,
       "arxiv_id": "2406.07358",
       "relevance": "Demonstrates that LLMs can strategically underperform during evaluation, a key concern discussed in this paper."
     },
     {
       "title": "Me, myself, and AI: The situational awareness dataset (SAD) for LLMs",
       "authors": ["R. Laine", "B. Chughtai", "J. Betley"],
       "year": 2024,
       "arxiv_id": "2407.04694",
       "relevance": "Benchmark for measuring LLM situational awareness — directly relevant to the paper's argument about AI understanding its evaluation context."
     },
     {
       "title": "Games for AI control: Models of safety evaluations of AI deployment protocols",
       "authors": ["C. Griffin", "L. Thomson", "B. Shlegeris", "A. Abate"],
       "year": 2024,
       "arxiv_id": "2409.07985",
       "relevance": "Uses game-theoretic models for AI safety evaluation, closely related to this paper's approach of applying game theory to evaluation design."
     },
     {
       "title": "Evaluating frontier models for stealth and situational awareness",
       "authors": ["M. Phuong", "R. S. Zimmermann", "Z. Wang"],
       "year": 2025,
       "arxiv_id": "2505.01420",
       "relevance": "Evaluates frontier model situational awareness capabilities, providing empirical grounding for this paper's theoretical concerns."
     },
     {
       "title": "Safety cases for frontier AI",
       "authors": ["M. D. Buhl", "G. Sett", "L. Koessler", "J. Schuett", "M. Anderljung"],
       "year": 2024,
       "arxiv_id": "2410.21572",
       "relevance": "Foundational work on safety cases for AI which this paper seeks to formalize and strengthen through game-theoretic analysis."
     },
     {
       "title": "Evaluation faking: Unveiling observer effects in safety evaluation of frontier AI systems",
       "authors": ["Y. Fan", "W. Zhang", "X. Pan", "M. Yang"],
       "year": 2025,
       "arxiv_id": "2505.17815",
       "relevance": "Investigates how AI systems may behave differently when they detect evaluation, directly relevant to the paper's core thesis."
     }
   ]
 }