ai-research-survey

scan.json (26121B)

---

 {
   "paper": {
     "title": "Prover-Verifier Games improve legibility of LLM outputs",
     "authors": ["Jan Hendrik Kirchner", "Yining Chen", "Harri Edwards", "Jan Leike", "Nat McAleese", "Yuri Burda"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2407.13692",
     "doi": "10.48550/arXiv.2407.13692"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "Training LLM provers for checkability by smaller LLM verifiers using a Prover-Verifier Game (PVG) improves the legibility of solutions to humans, demonstrating a surprising transfer from model-checkability to human-checkability. Optimizing solely for correctness via RL produces solutions that become unintelligible to time-constrained humans despite maintaining high accuracy. The Signed Relative Convincingness (SRC) reward yields better training dynamics than alternatives (CGC, goodharting). Results reveal a 'legibility tax' — a tension between solution correctness and human understandability.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "They use the public GSM dataset but augmented it with 100k synthetic examples. The augmented dataset, human evaluation data, and model outputs are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details are provided. Models are described only as 'from the GPT4 family.'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The algorithmic description is detailed but insufficient for reproduction without model access and implementation details."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Figures show point estimates (mean accuracy) without confidence intervals or error bars. No uncertainty quantification on main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'human accuracy increases') based on comparing numbers across conditions without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context — e.g., human accuracy from ~0.75 at initialization to ~0.85 after round 5 (Fig 8b), model accuracy from ~0.4 to ~0.65 (Fig 8a). Percentage points of change are interpretable from the figures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "75 human trainers and 15,983 datapoints are stated but no justification for why these sample sizes are adequate. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or spread measures are reported for training runs or human evaluation results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against a correctness-only RL baseline (Section 4.1) and the pretrained initialization. Alternative reward formulations (CGC, goodharting) are also compared in Section 5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The task (legibility training) is novel, so no prior methods exist. The baselines (RL for correctness, initialization, alternative reward variants) are appropriate internal comparisons."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ablates reward variants (SRC vs CGC vs goodharting). Additional ablations on verifier size (Figs S4, S5, 5c) and initialization (Fig S9) are provided."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Model accuracy, human judge accuracy, time-to-solve, verifier robustness (steps to exploit), balanced best-of-n accuracy, GPT-4 validity scores, and ROC AUC for prover separability are all reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "75 human contractors evaluated solution correctness under a 45-second time limit (Appendix I). Results in Figs 4, 7b, 8."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The GSM test set is used for evaluation. Training data is partitioned into separate prover and verifier splits (Section 3.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by training round (Figs 5-8), verifier size (Figs 6a, 10), reward variant (Section 5), and prover role (helpful/sneaky)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases with very small verifiers (Fig S4) and very large verifiers (Fig S5) are discussed. Appendix B shows degenerate solutions. CGC reward failure dynamics are analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CGC reward leads to worse training dynamics (Section 5.1). Goodharting training does not improve human legibility (Section 5.2, Fig S6). Very small and very large verifiers fail (Appendix D, E)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about accuracy-legibility tradeoff, verifier robustness increasing, helpful prover legibility increasing, and sneaky prover becoming more subtle are all supported by experimental results in Sections 4.1-4.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('checkability training improves legibility') are supported by controlled comparisons: same initialization with different training objectives (correctness-only vs. checkability), ablations across reward variants, and round-by-round progression."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'legibility training against small verifiers as a practical avenue for increasing legibility of large LLMs to humans' and 'could help with alignment of superhuman models' — generalizing well beyond the tested setting of grade-school math. While limitations (Section 6.1) acknowledge single-dataset scope, the framing is broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses weight sharing confounds (Section 4.2), capability gap effects (Section 4.2), separation as alternative explanation for verifier improvement (Appendix C), and initialization effects (Appendix F)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines legibility formally, acknowledges that human accuracy under time constraints is 'only one of many ways to measure human legibility' (Section 2), and discusses the gap between model-checkability and true human legibility."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are described only as 'from the GPT4 family' (Section 3.2) with relative compute sizes (10^-4 to 10^0). No specific model version, checkpoint, or snapshot date is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "System prompts for helpful and sneaky provers are provided in Appendix G (Fig S10) and few-shot prompts in Appendix H (Fig S11)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1 states 'We used the default ChatGPT post-training hyperparameters such as the optimizer, learning rate, or KL regularization coefficient' without providing specific values. V0=-2 is mentioned for CGC reward."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The setup involves standard RL training of language models."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes dataset augmentation with 100k synthetic examples, random partitioning into equal Dπ and DV splits, and balanced sampling of correct/incorrect solutions for verifier training."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 is a dedicated Limitations subsection with 7 specific limitation categories."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific: single dataset (GSM), conflating CoT with legible justifications, reliance on ground truth, heterogeneous stopping criteria across experiments, non-optimal hyperparameter tuning, and initialization with math-finetuned models."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly states what the paper does not show: no out-of-domain generalization, no unsupervised setup, only one dataset, only math domain, only one way to measure legibility."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (human evaluations, model outputs, training logs) is made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I describes human data collection: 75 trainers, 45-second time limit, 15,983 datapoints, balanced assignment, up to 4 hours per trainer, instructions shown in Fig S12."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Contractors are referred to as 'human contractors' and 'human trainers' with no description of how they were recruited, from what platform, or what qualifications they had."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I documents: sampling solutions from each checkpoint → randomizing assignment to 75 trainers with balanced conditions → collecting judgments → excluding 4 trainers at chance level → 15,983 final datapoints."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure. All authors are from OpenAI but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with OpenAI on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI has a direct commercial interest in demonstrating that alignment techniques work for their models. The paper evaluates OpenAI models using OpenAI infrastructure."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The GPT-4 family model's training data cutoff is not stated. The system prompt mentions 'Knowledge cutoff: 2023-04' but this is the prompt setting, not a discussion of contamination risk."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM test set problems appeared in GPT-4's pretraining data, despite GSM being published in 2021 and GPT-4 being trained on data through at least 2023."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K was published in 2021. GPT-4 was trained on data well past that date. No contamination analysis is performed or discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned for the human evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite the study involving human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographic information about the 75 human contractors is provided — no experience level, background, or other characteristics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion criteria for contractor selection are stated. The only exclusion criterion mentioned is post-hoc removal of 4 trainers performing at chance."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I states: 'We randomized assignment of those samples to 75 trainers (making sure each person see balanced correct vs. incorrect samples from each checkpoint).'"
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I states trainers 'are not told whether they judged correctly' and samples were randomized across conditions, so trainers didn't know which training round produced each solution."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I reports starting with 75 trainers and excluding 4 whose performance was statistically indistinguishable from chance, leaving 71."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times are reported for any component of the system."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. Verifier sizes are described only as relative fractions of prover compute (10^-4 to 10^0), with no absolute numbers."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results appear to be from single training runs per configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of independent training runs is not stated. It appears each configuration was run once."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1 acknowledges 'We did not tune all hyperparameters thoroughly' and describes tuning sample ratios for one verifier size, but no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Section 5 justify why SRC reward is preferred, with explicit comparisons to CGC and goodharting alternatives showing worse dynamics."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI authors evaluate their own training method on their own models. No discussion of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance is shown as a function of RL steps (Figs 4a, 5) and across verifier sizes spanning 4 orders of magnitude (Fig 10, 6a), providing compute-performance relationships."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 acknowledges GSM is 'relatively easy and the solutions are quite standardized' and discusses the desire to extend to 'harder math datasets and coding, as well as fuzzier domains.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this work."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4's pretraining data includes GSM solutions or similar grade-school math content."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage in the evaluation setup."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes partitioning training data into disjoint Dπ and DV subsets so 'the verifier and the prover are never optimized on the same prompt.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, etc.) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Direct RL optimization for correctness produces solutions that are difficult for time-constrained humans to evaluate.",
       "evidence": "Section 4.1, Fig 4b: Human accuracy drops from ~0.82 (initialization) to ~0.70 (7k RL steps). Fig 4c shows evaluation time increases. Fig S1 shows degenerate solutions.",
       "supported": "strong"
     },
     {
       "claim": "Checkability training achieves decent performance while retaining legibility to humans.",
       "evidence": "Section 4.3.3, Fig 8: Human accuracy increases from ~0.75 to ~0.87 across rounds while model accuracy increases from ~0.42 to ~0.65. Fig 1 shows the performance-legibility tradeoff.",
       "supported": "strong"
     },
     {
       "claim": "Verifiers become increasingly robust to adversarial attacks over training rounds.",
       "evidence": "Section 4.3.1, Fig 6a: Steps to exploit increase across rounds. Fig 6b: Balanced best-of-n accuracy improves. Cross-attack generalization (SRC, CGC, goodharting) confirmed.",
       "supported": "strong"
     },
     {
       "claim": "The sneaky prover produces increasingly subtle flaws that are harder for humans to detect.",
       "evidence": "Section 4.3.2, Fig 7a: GPT-4 average validity score increases. Fig 7b: Human accuracy decreases on sneaky solutions. However, Fig 7b is from a pilot study with acknowledged design flaws.",
       "supported": "moderate"
     },
     {
       "claim": "Legibility to smaller LLM verifiers transfers to legibility to humans.",
       "evidence": "Section 4.3.3: Both human accuracy and model verifier robustness improve across rounds. This is shown for the helpful prover (Fig 8) and sneaky prover (Fig 7b, with caveats).",
       "supported": "moderate"
     },
     {
       "claim": "The existence of a verifier-leading Stackelberg equilibrium is necessary and sufficient for completeness and soundness.",
       "evidence": "Theorem 1 with full proof in Appendix A. The proof assumes H is the set of all deterministic mappings, which does not hold for neural networks in practice.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are from OpenAI evaluating an alignment technique on OpenAI's GPT-4 family models. No independent verification or external evaluation."
     },
     {
       "flag": "Pilot study with acknowledged design flaws",
       "detail": "Some human evaluation results (Fig 7b, S5c, S6) come from a pilot study where solutions were shown in round order across a month, introducing learning/fatigue effects. The authors acknowledge this but still present the results."
     },
     {
       "flag": "No model versions or compute details",
       "detail": "Models are described only as 'GPT-4 family' with relative compute sizes. Absolute model sizes, versions, and total compute budget are withheld, making reproduction impossible."
     },
     {
       "flag": "Single dataset evaluation",
       "detail": "All results are on GSM8K (grade-school math), a relatively easy benchmark. Claims about alignment of superhuman models extrapolate far beyond this setting."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "No error bars, confidence intervals, statistical tests, or multi-seed results are reported despite being a training-dynamics study where stochasticity could significantly affect conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "Learning to give checkable answers with prover-verifier games",
       "authors": ["C. Anil", "G. Zhang", "Y. Wu", "R. Grosse"],
       "year": 2021,
       "arxiv_id": "2108.12099",
       "relevance": "Original prover-verifier game framework that this paper extends to LLMs at scale."
     },
     {
       "title": "Weak-to-strong generalization: Eliciting strong capabilities with weak supervision",
       "authors": ["C. Burns", "P. Izmailov", "J. H. Kirchner"],
       "year": 2023,
       "arxiv_id": "2312.09390",
       "relevance": "Closely related alignment paradigm of using weak models to supervise strong ones."
     },
     {
       "title": "Let's Verify Step by Step",
       "authors": ["H. Lightman", "V. Kosaraju", "Y. Burda"],
       "year": 2024,
       "relevance": "Process reward models for math reasoning — alternative approach to improving reasoning verification."
     },
     {
       "title": "Self-critiquing models for assisting human evaluators",
       "authors": ["W. Saunders", "C. Yeh", "J. Wu"],
       "year": 2022,
       "arxiv_id": "2206.05802",
       "relevance": "Scalable oversight via LLM self-critique, a complementary approach to prover-verifier games."
     },
     {
       "title": "AI safety via debate",
       "authors": ["G. Irving", "P. Christiano", "D. Amodei"],
       "year": 2018,
       "arxiv_id": "1805.00899",
       "relevance": "Foundational scalable oversight proposal; PVG is a single-turn variant of debate."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2022,
       "relevance": "CoT prompting that this paper aims to make more legible through adversarial training."
     },
     {
       "title": "Open problems and fundamental limitations of reinforcement learning from human feedback",
       "authors": ["S. Casper", "X. Davies", "C. Shi"],
       "year": 2023,
       "relevance": "Systematic analysis of RLHF limitations motivating complementary approaches like PVG."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistent deceptive behavior in LLMs, motivating legibility-based oversight."
     },
     {
       "title": "Sycophancy to Subterfuge: Investigating Reward-Tampering in Large Language Models",
       "authors": ["C. Denison", "M. MacDiarmid", "F. Barez"],
       "year": 2024,
       "arxiv_id": "2406.10162",
       "relevance": "Studies reward hacking and deceptive alignment in LLMs, directly relevant to safety."
     },
     {
       "title": "Scaling laws for reward model overoptimization",
       "authors": ["L. Gao", "J. Schulman", "J. Hilton"],
       "year": 2023,
       "relevance": "Characterizes reward hacking/goodharting which the PVG approach aims to mitigate."
     },
     {
       "title": "LLM Critics Help Catch LLM Bugs",
       "authors": ["N. McAleese", "Rai", "J. F. C. Uribe"],
       "year": 2024,
       "relevance": "Complementary scalable oversight method using LLM critics for bug detection."
     },
     {
       "title": "Debating with More Persuasive LLMs Leads to More Truthful Answers",
       "authors": ["A. Khan", "J. Hughes", "D. Valentine"],
       "year": 2024,
       "arxiv_id": "2402.06782",
       "relevance": "Shows debate with persuasive LLMs improves truthfulness — related scalable oversight method."
     }
   ]
 }