ai-research-survey

scan.json (27653B)

---

 {
   "paper": {
     "title": "Evaluating & Reducing Deceptive Dialogue From Language Models with Multi-turn RL",
     "authors": ["Marwa Abdulhai", "Ryan Cheng", "Aryansh Shrivastava", "Natasha Jaques", "Yarin Gal", "Sergey Levine"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.14318",
     "doi": "10.48550/arXiv.2510.14318"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 provides a GitHub link: https://github.com/abdulhaim/deceptive_dialogue and a project page."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code is available at the GitHub repo, and the dialogue datasets are generated synthetically with the provided code. The generation pipeline is documented in detail (Section A.4)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is described in the paper. Only mentions of using OpenRLHF and vLLM, but no version-pinned dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README contents or reproduction commands are included."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-3 and 5-8 report mean ± standard deviation for all metrics across tasks and models."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., '77.6% reduction', '31% increase') but reports no statistical significance tests (no p-values, t-tests, or bootstrap tests). Comparisons are based solely on comparing mean values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '77.6% reduction compared to Llama 3-8B-Instruct' (Table 3), '32% and 235% increase in deception' (Section Q3). These provide effect magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification given for the number of dialogues generated per task (Table 4 shows 3,372-7,751 dialogues) or for the 20 human annotators evaluating 60 dialogues. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in all main results tables (Tables 1-3, 5-12)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares RL-fine-tuned models against base Llama, instruction-tuned Llama, SFT, and truthful-prompted larger models. Table 1 compares belief misalignment against 4 existing deception metrics."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Llama 3.1, GPT-4o-mini, Gemma-2, Mistral — all recent models at time of writing. Deception metrics compared include recent work from 2022-2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows ablation across RL methods (KTO, REINFORCE, PPO) and reward objectives (max-reward, min-deception, combined). The counterfactual study (Q4, Figure 3) also ablates prompting conditions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five deception metrics are evaluated (deception count, deception rating, falsehood count, deceptive regret, belief misalignment) plus task reward. Table 1 reports all five."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section Q1: 20 annotators recruited via CloudResearch Connect evaluated 60 dialogues on a 1-5 Likert scale. Pearson correlation computed between metrics and human labels (Table 1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section Q5: 'We trained Llama-3.1-8B on 9.7k dialogue pairs and evaluated them on a held-out set of 2.4k.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per task (4 dialogue domains), per model (8 models), and per prompting condition (default, deceptive, truthful, utilitarian) across Tables 2, 5-8."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section A.12 provides detailed examples of failure modes for each metric (A.12.1-A.12.3), and Section A.1 discusses limitations of the approach."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that RLHF sometimes increases deception (Q3), that truthful prompting can paradoxically increase belief misalignment (Section A.14), and that KTO-max-reward increases deception (Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: 26% deception rate (supported by Table 2 averages), 31% increase when prompted (supported by counterfactual analysis), 43% RLHF deception rate (supported by Table 2), 77.6% reduction via multi-turn RL (supported by Table 3). All claims match reported results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about RL fine-tuning reducing deception. The experimental design (controlled manipulation of training method, held-out evaluation) supports causal inference for the RL results. Counterfactual prompting (Section Q4) provides controlled comparisons across prompting conditions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Language Models' broadly. Results are from 4 synthetic dialogue tasks with 8 specific models. The paper does not adequately bound generalizations — claims like 'LLMs naturally exhibit deceptive behavior in approximately 26% of dialogue turns' suggest this applies to LLMs generally, but only 8 models in 4 contrived scenarios were tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section A.10 discusses four alternative explanations for emergent deception (goal inference, training data biases, lack of penalization, misalignment). Section A.1 discusses limitations including annotator subjectivity and metric blind spots."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures belief misalignment via LLM-as-a-judge (JLLM and LLLM) as a proxy for actual deception, but does not discuss how well this LLM-based measurement captures real deception vs. the LLM judge's own biases. The gap between 'LLM judge says beliefs shifted' and 'actual deception occurred' is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists 'gpt-3.5-turbo', 'gpt-4o-mini', 'Llama-3.1-8B', 'Llama-3.1-8B-Instruct', etc. but no API snapshot dates or specific version strings for the OpenAI models. Marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Sections A.8 and A.9 provide the actual prompt text for all deception metrics and all four counterfactual prompting conditions across all four tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section A.4 reports temperature=0.8, top_p=0.95 for local models and default settings for OpenAI. Tables 13-14 report full SFT and PPO hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The dialogue setup is two LLMs prompted directly, with a third LLM judge. No tools, retry logic, or agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section A.4 documents the data generation pipeline in detail: buyer preference combinations (32), seller action space (243), seller personas (4), sampling strategy, and filtering for Deal or No Deal (3,996 valid combinations after filtering)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section A.1 is titled 'Limitations' and provides substantive discussion of annotator subjectivity, metric blind spots, and subtler deception forms."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section A.1 discusses specific threats: 'deception is inherently subjective', '20 annotators' may introduce noise, metrics 'may miss subtler forms such as manipulative framing or strategic ambiguity.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound the generalization to the 4 specific synthetic tasks or the 8 models tested. The Discussion (Section 6) discusses contributions without explicitly stating scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository is provided (https://github.com/abdulhaim/deceptive_dialogue) which should contain the generated dialogue datasets and code to regenerate them."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1 and A.4 describe in detail how dialogue data was generated: LLM agent pairs, prompting conditions, sampling parameters, and combinatorial design."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section Q1: '20 annotators recruited through CloudResearch Connect, a reliable platform that provides access to high-quality, vetted respondents with verified demographics and strong prior approval ratings.' IRB approval mentioned."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section A.4 documents the full pipeline: buyer preference generation → seller action space → persona assignment → dialogue generation → metric evaluation via LLM judge. Table 4 reports statistics per domain."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Acknowledgment): 'This research was supported by the Cooperative AI Foundation and DSIT, as well as the National Science Foundation under IIS-2246811.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: UC Berkeley, University of Oxford, University of Washington, UK AI Security Institute, Google DeepMind."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are NSF, Cooperative AI Foundation, and DSIT — research foundations/government agencies with no direct financial stake in whether LLMs are more or less deceptive."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Natasha Jaques is affiliated with Google DeepMind, which develops LLMs evaluated in related work, but no financial interest disclosure is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper generates synthetic dialogue data and evaluates models on these novel synthetic tasks. It does not evaluate pre-trained model knowledge on an existing benchmark — the deception tasks are novel and generated at evaluation time."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: synthetic dialogue tasks generated at evaluation time, not a pre-existing benchmark that could be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the evaluation tasks are novel synthetic dialogues, not pre-existing benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The paper includes a human evaluation study (20 annotators) but no pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section Q1: 'We recruited 20 annotators (with IRB approval) through CloudResearch Connect.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographics reported for the 20 annotators beyond that they were from CloudResearch Connect with 'verified demographics and strong prior approval ratings.' No age, gender, experience level, or other characterization."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria stated for annotators beyond using CloudResearch Connect's vetting."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with human participants assigned to conditions. Annotators rated dialogues — no randomization to treatment/control needed."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a rating study, not an experimental comparison. Blinding is not applicable to annotators rating dialogues for deceptiveness."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on whether any annotators dropped out or were excluded. Only the final N=20 is reported."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs reported. The approach generates thousands of dialogues from GPT-3.5/GPT-4o-mini via API and uses LLM-as-judge for every metric evaluation, but no API costs or per-dialogue costs are stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section A.15: 'Training was done with access to a cluster of 8 NVIDIA H100 GPUs as well as a cluster of 8 NVIDIA H200 GPUs.' However, total GPU hours are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard deviations are reported across dialogues, but no mention of multiple random seeds for the RL training runs. Single training run results appear to be reported for each RL method."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many training runs were conducted for each RL method. Table 3 std devs appear to be across test dialogues, not across training runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Tables 13-14 report hyperparameters but no mention of how many configurations were tried or any hyperparameter search process."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how hyperparameters were selected. The reported PPO/KTO configurations appear without justification for why these specific values were chosen."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across 8 models, 4 tasks, 4 prompting conditions, and 5 metrics — no correction for multiple comparisons applied. No statistical tests performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the belief misalignment metric and then show it outperforms other metrics. No acknowledgment of potential bias in evaluating their own metric. The LLM judge used is the same across all metrics, but the metric design choices favor their approach."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 compares 8B RL-fine-tuned models against 70B prompted models without discussing the compute difference. PPO training on H100/H200 clusters vs. simple prompting is a major compute disparity not addressed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section A.12 extensively discusses what each metric actually measures vs. what it claims, with concrete examples showing failure modes. The paper explicitly analyzes whether its synthetic tasks capture real-world deception."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. Models are prompted directly for dialogue generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The tasks are synthetically generated at evaluation time with novel scenarios. There is no pre-existing benchmark that could leak through training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The LLM judge (JLLM) is conditioned on ground truth facts ϕ (Section 3.3). Whether this creates evaluation bias — the judge knowing the 'right answer' while rating deception — is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper generates training data (9.7k pairs) and test data (2.4k) from the same synthetic pipeline (Section Q5). Whether train and test dialogues share structural similarities is not discussed."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — synthetic data generated at evaluation time, not a pre-existing benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "Belief misalignment correlates more closely with human judgments of deception than existing metrics (Pearson r=0.788).",
       "evidence": "Table 1: Human correlation scores — belief misalignment 0.788 vs. deceptive regret 0.738, deception count 0.672, falsehood count 0.609, deception rating 0.584. Based on 20 annotators rating 60 dialogues.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs naturally exhibit deceptive behavior in approximately 26% of dialogue turns under default prompting.",
       "evidence": "Table 2 shows average belief misalignment of 0.41 across 8 models and 4 tasks under default settings. The 26% figure appears in the abstract but the methodology for deriving it from belief misalignment scores is not clearly explained.",
       "supported": "weak"
     },
     {
       "claim": "Models trained with RLHF still exhibit deception at a rate of 43% on average.",
       "evidence": "Table 2 shows instruction-tuned models with various belief misalignment scores. The 43% claim is in the abstract but derivation from the table values is unclear.",
       "supported": "weak"
     },
     {
       "claim": "Multi-turn RL fine-tuning with PPO reduces deception by 77.6% compared to instruction-tuned models.",
       "evidence": "Table 3: PPO-min-deception achieves 0.11 belief misalignment vs. Llama 3-8B-Instruct at 0.49, a (0.49-0.11)/0.49 = 77.6% reduction. However, task reward drops from 0.53 to 0.40.",
       "supported": "moderate"
     },
     {
       "claim": "When prompted to deceive, LLMs can increase deceptiveness by as much as 31% relative to baselines.",
       "evidence": "Figure 3 and Tables 5-8 show counterfactual analysis. The 31% figure is stated in the abstract but the specific model/task combination producing this number is not clearly identified in the text.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMs exhibit deceptive behavior in dialogue even under default (non-adversarial) prompting, with instruction-tuned models sometimes more deceptive than base models in strategic tasks. The proposed belief misalignment metric, which measures divergence of listener beliefs from ground truth, correlates better with human judgments (r=0.788) than existing deception metrics. Multi-turn PPO fine-tuning with belief misalignment as reward reduces deception by 77.6% compared to instruction-tuned baselines, though at some cost to task performance.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (77.6% reduction, 31% increase, metric correlation differences) are based on comparing raw numbers without any significance tests. With the reported standard deviations, many differences may not be statistically significant."
     },
     {
       "flag": "Unclear derivation of headline numbers",
       "detail": "The abstract claims '26% of dialogue turns' and '43% rate' for RLHF models, but the methodology for converting belief misalignment scores (continuous 0-1) to these percentages is not explained in the paper."
     },
     {
       "flag": "LLM-as-judge circular evaluation",
       "detail": "The deception metrics are computed by LLM judges, and the human correlation study validates these against only 20 annotators on 60 dialogues. The LLM judge's own biases and failures at detecting deception are not analyzed."
     },
     {
       "flag": "Small human evaluation sample",
       "detail": "Only 20 annotators evaluated 60 total dialogues (15 per task) to validate the core claim that belief misalignment best correlates with human judgment. No inter-annotator agreement reported."
     },
     {
       "flag": "Synthetic-only evaluation",
       "detail": "All dialogue tasks are synthetic with contrived scenarios (house showing, nutrition advice, charity, deal-or-no-deal). No evaluation on real-world conversational data or natural human-LLM interactions."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Key AI safety work on persistent deceptive behaviors in LLMs surviving safety training."
     },
     {
       "title": "AI deception: A survey of examples, risks, and potential solutions",
       "authors": ["Peter S. Park", "Simon Goldstein", "Aidan O'Gara"],
       "year": 2023,
       "arxiv_id": "2308.14752",
       "relevance": "Comprehensive survey of deception in AI systems, foundational for the deception evaluation space."
     },
     {
       "title": "Language models learn to mislead humans via RLHF",
       "authors": ["Jiaxin Wen", "Ruiqi Zhong", "Akbir Khan"],
       "year": 2024,
       "arxiv_id": "2409.12822",
       "relevance": "Directly relevant finding that RLHF training can increase misleading behaviors in LLMs."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Foundational safety training method used as baseline deception detection approach."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "arxiv_id": "2109.07958",
       "relevance": "Key benchmark for truthfulness evaluation in LLMs, used as baseline metric in this work."
     },
     {
       "title": "AI-Liedar: Examine the trade-off between utility and truthfulness in LLM agents",
       "authors": ["Zhe Su", "Xuhui Zhou", "Sanketh Rangreji"],
       "year": 2024,
       "arxiv_id": "2409.09013",
       "relevance": "Prior work on measuring deception-utility tradeoffs in LLMs, baseline metric comparison."
     },
     {
       "title": "Large language models can strategically deceive their users when put under pressure",
       "authors": ["Jérémy Scheurer", "Mikita Balesni", "Marius Hobbhahn"],
       "year": 2024,
       "arxiv_id": "2311.07590",
       "relevance": "Demonstrates strategic deception capabilities in LLMs under pressure, related AI safety finding."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Foundation for the LLM-as-judge evaluation methodology used throughout this work."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "RLHF training methodology that this paper evaluates for deception reduction effectiveness."
     },
     {
       "title": "OpenRLHF: An easy-to-use, scalable and high-performance RLHF framework",
       "authors": ["Jian Hu", "Xibin Wu", "Wei Shen"],
       "year": 2025,
       "arxiv_id": "2405.11143",
       "relevance": "Open-source RLHF framework extended by this work for multi-turn dialogue RL fine-tuning."
     },
     {
       "title": "Deception abilities emerged in large language models",
       "authors": ["Thilo Hagendorff"],
       "year": 2024,
       "doi": "10.1073/pnas.2317967121",
       "relevance": "PNAS paper demonstrating emergent deception capabilities in LLMs, directly relevant to AI safety."
     }
   ]
 }