ai-research-survey

scan.json (26657B)

---

 {
   "paper": {
     "title": "Training LLMs for Honesty via Confessions",
     "authors": ["Manas Joglekar", "Jeremy Chen", "Gabriel Wu", "Jason Yosinski", "Jasmine Wang", "Boaz Barak", "Amelia Glaese"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.08093",
     "doi": "10.48550/arXiv.2512.08093"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes 'confessions' — auxiliary self-reports trained via RL to detect model misbehavior. When GPT-5-Thinking misbehaves, it confesses honestly at least half the time in 11/12 evaluations (mean 74.3% confession rate conditioned on bad behavior). Confession training modestly improves accuracy over untrained confessions. Critically, confessions detect reward hacking even when the main answer successfully hacks the reward model. False negatives cluster in cases of genuine confusion rather than intentional deception.",
   "claims": [
     {
       "claim": "When the model exhibits bad behavior, it confesses to it at least half the time in 11/12 evaluations",
       "evidence": "Figure 2 shows confession rates across 12 evaluations. Average confession rate conditioned on bad behavior is 74.3%. Only Hallucinations (Background Tasks) and one other fall below 50%.",
       "supported": "moderate"
     },
     {
       "claim": "The model is significantly more likely to come clean in its confession than in its main answer",
       "evidence": "Figure 3 compares confession vs admission rates. In most evaluations, confession rate (green) far exceeds admission rate (purple). For scheming evaluations, admission rate is 0% while confession rate is high.",
       "supported": "strong"
     },
     {
       "claim": "RL training on confessions leads to improvements in confession accuracy",
       "evidence": "Figure 5 shows intervention vs control training curves. Most evaluations show improvement, with notable gains in instruction hierarchy tasks. Hallucination evaluations show no improvement or regression.",
       "supported": "moderate"
     },
     {
       "claim": "Confession training does not significantly impact the model's performance on the original task",
       "evidence": "Figure 16 shows compliance rates for both runs. Authors state 'we do not expect a significant difference' and note differences are within inter-run variance.",
       "supported": "weak"
     },
     {
       "claim": "Confessions detect reward hacking even when the reward model is hacked",
       "evidence": "Section 4 and Figure 7 show that when training against a weak judge, reward model accuracy decreases over training while confession accuracy increases.",
       "supported": "moderate"
     },
     {
       "claim": "False negatives most commonly occur due to genuine confusion rather than intentional omission",
       "evidence": "Section 6 and Figure 9 use a prompted classifier to categorize false negatives. After filtering, zero examples of intentional omission were found.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are from OpenAI, evaluating their own model GPT-5-Thinking. No independent replication or external evaluation. The paper proposes a safety mechanism for a commercial product by the company that sells it."
     },
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "Results are presented as proportions in bar charts and line plots with no confidence intervals, error bars, significance tests, or variance measures. Claims of 'improvement' and 'no significant impact' are made without statistical backing."
     },
     {
       "flag": "Private evaluations not reproducible",
       "detail": "Most evaluations are described as 'private' and out-of-distribution. The exact evaluation sets, prompts, and model weights are not available, making independent verification impossible."
     },
     {
       "flag": "LLM-as-judge circularity",
       "detail": "Both the confession reward and the evaluation metrics (complied, admitted, confessed) are computed by prompted LLM judges. The reliability of these judges is not independently validated."
     },
     {
       "flag": "Proof-of-concept framing with broad claims",
       "detail": "The paper acknowledges it is a 'proof-of-concept' with 'relatively small amount of RL' but the title and framing suggest a general method for training LLMs for honesty."
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository or download link is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The evaluations are mostly described as 'private'. No evaluation datasets or training data are released. SimpleQA is public but the custom evaluations are not."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or hardware details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The method uses proprietary model weights and private training infrastructure."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Figures 2, 3, 5, and 7 show proportions and rates without confidence intervals or error bars. No uncertainty quantification on any reported metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims confession training 'leads to improvements' and that base task impact is 'minimal' without any statistical significance tests. Differences between control and intervention runs are assessed visually from plots."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as proportions (e.g., '74.3% average confession rate') but without baseline context or standardized effect sizes for the training improvement claims."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes for each evaluation are not explicitly stated or justified. Footnote 1 mentions 'average probability across evaluations' but individual N values are not provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes 'significant variability' across evaluations and mentions 'significant inter-run variance' for some evaluations (Section 3.3) but does not report standard deviations or variance measures across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes a control run (RL without confession training) matched on non-confession data points, compared against the intervention run with confession training."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the same model (GPT-5-Thinking) without confession training, which is the most appropriate comparison for measuring the effect of the proposed method."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations: control vs intervention (Section 3.2), weak judge experiment (Section 4), varying test-time compute (Section 3.4, Figure 14), no-CoT setting (Figure 15)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three distinct metrics: complied (did the model behave?), admitted (did it tell the user?), confessed (did it report in confession?). Also: false positive/negative rates, AUC for confidence extraction (0.79)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via LLM judges. The 'complied', 'admitted', and 'confessed' labels are extracted by 'a prompted language model' (Section 3.1). No human evaluation of confession quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'Almost all are out of distribution from the training datasets' with one exception noted (Instruction Following Word Count). Frozen rollouts methodology (Section 3.2) separates evaluation from training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down across 12 evaluations spanning instruction following, instruction hierarchy, hallucinations, scheming, and reward hacking (Figures 2, 3, 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides detailed analysis of false negatives and false positives, with a prompted classifier categorizing error types (Figure 9, Figure 10). Multiple example transcripts shown (Figures 11, 12, 19, 20)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that hallucination evaluations (People, General Knowledge) show no improvement and 'arguably even some regressions' from training (Section 3.2). Also acknowledges confessions fail when model is genuinely confused."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are hedged ('often confesses', 'modestly improves'). The results in Figures 2-5 support these hedged claims. The paper explicitly frames results as 'proof-of-concept'."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim (confession training improves confession accuracy) is supported by a controlled comparison: intervention vs control runs matched on non-confession data (Section 3.2). Frozen rollouts control for confounders from changing compliance rates."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly states 'our experiments should mainly be viewed as a proof-of-concept.' Section 7.4 lists specific limitations. However, the broad title partially undermines this bounding."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 discusses why confessions might work (toy model argument). Section 6 analyzes why confessions fail. Section 3.2 addresses the confounder of changing compliance rates. Section 7.3 discusses training without ground truth."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper distinguishes between 'honesty' (the goal) and 'confession accuracy' (the proxy measured). Section 7.4 explicitly discusses what confessions cannot measure (genuine mistakes, unknown unknowns)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-5-Thinking' without a version, snapshot date, or API identifier. No specific model version is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides the full ConfessionReport schema used as the confession prompt. Evaluation setups are described in Appendix A with example prompts shown in figures (Figures 1, 6, 11, 18-21). The confession request xc is stated to be 'constant across all settings.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only the confession training probability p=25% and 'Medium' reasoning effort are mentioned. No learning rate, temperature, batch size, training steps, or other RL hyperparameters are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the full confession pipeline: input x → rollout (z,y) → confession request xc → confession (zc,yc). The reward separation mechanism is detailed. Tool access in evaluations is described per-evaluation in Appendix A."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Training data is described only as 'a subset of the standard training set' (Section 3.2). No details on what data was used, how it was selected, or preprocessing steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 'What confessions are not good for' provides a dedicated limitations discussion covering monitoring vs training out behavior, jailbreaking and confusion, and unknown-unknowns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: confessions cannot catch genuine mistakes (Section 7.4), the confession judge may be hackable (Section 7.1), hallucination evaluations show no improvement because errors are from confusion not deception (Section 3.2)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 explicitly lists three categories of what confessions do NOT address: they don't train out bad behavior, they don't help with jailbreaking/confusion, and they miss unknown-unknowns. Section 1 frames the work as 'proof-of-concept.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (model outputs, evaluation traces, training logs) is released. Only aggregated results in figures and tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Training data described only as 'a subset of the standard training set.' Evaluation datasets are described at a high level (Appendix A) but without details on how many examples, how they were created, or selection criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluations use automated model rollouts."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The data pipeline from training data selection through to evaluation metric extraction is not documented. The flow from rollout to LLM-judge labeling is described conceptually but not with sufficient detail for reproduction."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgment of financial support. All authors are from OpenAI but no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed as 'OpenAI' on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI funds the research and has a direct commercial interest in demonstrating the safety of its models. The paper evaluates GPT-5-Thinking, OpenAI's own product."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement. OpenAI employees presumably hold equity in the company whose product is being evaluated."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for GPT-5-Thinking."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explicitly states 'Almost all are out of distribution from the training datasets' and notes one exception (Instruction Following Word Count was used in training, Section 4)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SimpleQA (used for Hallucinations General Knowledge) was published before GPT-5-Thinking's training. No contamination analysis is performed. Other evaluations are private so contamination is less of a concern for those."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption figures reported despite the method requiring an additional full model inference for each confession."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'relatively low-compute training setting' and 'base curriculum compute-matched' but provides no actual compute figures (GPU hours, training time, API cost)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The paper notes 'significant inter-run variance' for some evaluations (Section 3.3) but does not report results across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of samples per evaluation is not explicitly stated. Only one control run and one intervention run are reported."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The confession training probability p=25% appears chosen without justification."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for configuration choices. The selection of p=25%, 'Medium' reasoning effort, and training duration are not justified."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Results reported across 12 evaluations with no correction for multiple comparisons. No statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI authors evaluate their own model and method without acknowledging self-comparison bias or seeking independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 14 shows confession performance as a function of test-time compute (tokens). Figure 5 shows performance as a function of training compute. Both vary compute and report performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its evaluations actually measure 'honesty' as opposed to a narrower construct. The relationship between LLM-judge-labeled 'confessed' and actual honesty is not validated."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a single model with its own integrated confession mechanism. No cross-model or cross-scaffold comparisons are made, and the scaffold IS the method being tested."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. SimpleQA was published in 2024; GPT-5-Thinking's training timeline is not disclosed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information. The confession has access to the full rollout including tool calls, which could provide hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and evaluation data beyond stating evaluations are 'out of distribution.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied."
       }
     }
   },
   "cited_papers": [
     {
       "title": "Sycophancy to subterfuge: Investigating reward-tampering in large language models",
       "authors": ["Carson Denison", "Monte MacDiarmid", "Fazl Barez"],
       "year": 2024,
       "arxiv_id": "2406.10162",
       "relevance": "Foundational work on reward hacking in LLMs, directly motivates the confession approach."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistent deceptive behavior in LLMs relevant to AI safety and honesty."
     },
     {
       "title": "Monitoring reasoning models for misbehavior and the risks of promoting obfuscation",
       "authors": ["Bowen Baker", "Joost Huizinga", "Leo Gao"],
       "year": 2025,
       "arxiv_id": "2503.11926",
       "relevance": "Chain-of-thought monitoring as a complementary approach to confessions for detecting AI misbehavior."
     },
     {
       "title": "Chain of thought monitorability: A new and fragile opportunity for AI safety",
       "authors": ["Tomek Korbak", "Mikita Balesni", "Elizabeth Barnes"],
       "year": 2025,
       "arxiv_id": "2507.11473",
       "relevance": "Analyzes CoT monitoring reliability and fragility, directly compared to confessions in the paper."
     },
     {
       "title": "Measuring faithfulness in chain-of-thought reasoning",
       "authors": ["Thomas Lanham", "Alex Chen", "Ansh Radhakrishnan"],
       "year": 2023,
       "arxiv_id": "2307.13702",
       "relevance": "Studies CoT faithfulness, relevant to understanding when model self-reports can be trusted."
     },
     {
       "title": "Stress testing deliberative alignment for anti-scheming training",
       "authors": ["Bronson Schoen", "Evgenia Nitishinskaya", "Mikita Balesni"],
       "year": 2025,
       "arxiv_id": "2509.15541",
       "relevance": "Source of scheming evaluations (sandbagging, covert rule violation) used in the paper."
     },
     {
       "title": "Spilling the beans: Teaching LLMs to self-report their hidden objectives",
       "authors": ["Chloe Li", "Mary Phuong", "Daniel Tan"],
       "year": 2025,
       "arxiv_id": "2511.06626",
       "relevance": "Related approach to training models for honest self-reporting, uses SFT rather than RL."
     },
     {
       "title": "Reasoning models don't always say what they think",
       "authors": ["Yanda Chen", "Joe Benton", "Ansh Radhakrishnan"],
       "year": 2025,
       "arxiv_id": "2505.05410",
       "relevance": "Demonstrates unfaithful reasoning in LLMs, motivating need for confession-like mechanisms."
     },
     {
       "title": "Detecting strategic deception using linear probes",
       "authors": ["Nicholas Goldowsky-Dill", "Bilal Chughtai", "Stefan Heimersheim"],
       "year": 2025,
       "arxiv_id": "2502.03407",
       "relevance": "Alternative approach to deception detection using internal model representations."
     },
     {
       "title": "Why language models hallucinate",
       "authors": ["Adam Tauman Kalai", "Ofir Nachum", "Santosh S. Vempala"],
       "year": 2025,
       "arxiv_id": "2509.04664",
       "relevance": "Theoretical framework for hallucinations arising from reward gaming, directly cited as motivation."
     },
     {
       "title": "When chain of thought is necessary, language models struggle to evade monitors",
       "authors": ["Scott Emmons", "Erik Jenner", "David K. Elson"],
       "year": 2025,
       "arxiv_id": "2507.05246",
       "relevance": "Evidence that CoT monitoring is hard to fool for complex tasks, related to confession monitoring."
     },
     {
       "title": "Training language models to explain their own computations",
       "authors": ["Belinda Z. Li", "Zifan Carl Guo", "Vincent Huang"],
       "year": 2025,
       "arxiv_id": "2511.08579",
       "relevance": "Shows models have privileged introspective access, motivating same-weight actor-monitor design."
     }
   ]
 }