ai-research-survey

scan.json (26145B)

---

 {
   "paper": {
     "title": "D-REX: A Benchmark for Detecting Deceptive Reasoning in Large Language Models",
     "authors": ["Satyapriya Krishna", "Eliot Krzysztof Jones", "J. Zico Kolter", "Andy Zou", "Nick Winter", "Matt Fredrikson", "Rahul Gupta", "Dan Hendrycks", "Spyros Matsoukas"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.17938",
     "doi": "10.48550/arXiv.2509.17938"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "D-REX introduces a benchmark for detecting deceptive reasoning in LLMs via competitive red-teaming, collecting 8,162 adversarial samples across 7 behaviors. All 7 frontier models tested showed 28-42% target-specific jailbreak success rates. Models scored above 8.4/10 on detection evasion and output camouflage, indicating strong ability to disguise malicious reasoning. Intra-model analysis showed CoT length does not causally predict jailbreak success, suggesting the benchmark resists simple gaming.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or archive is mentioned in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is explicitly maintained as a private dataset (Section 3.2, Appendix D): 'D-REX is maintained as a private benchmark.' Only aggregate results are shared via a controlled submission process."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or hardware details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The private benchmark model precludes independent replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports ± standard error for all jailbreak rates (e.g., '28.22 ± 1.38'). Figure 3 includes error bars representing standard error of the mean."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 7 models and claims some are 'most susceptible' or 'most resilient' but uses no statistical significance tests. Differences are assessed by comparing point estimates only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Jailbreak rates are reported as percentages with context (e.g., Target-Specific 28.22% to 42.01%, Overall Vulnerability 0.20% to 42.05%), providing baseline context for effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "8,162 samples collected and 500 used for Overall Vulnerability metric, but no justification is given for either sample size, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported in Table 2 and error bars in Figure 3, showing variance across samples."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven frontier models are compared against each other, and D-REX is compared against 7 related benchmarks in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated models are 2024-2025 frontier models (Gemini 2.5, Claude 3.7, DeepSeek R1, Qwen 3, Grok 3). Benchmark comparisons include recent work (2024-2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of the benchmark design (e.g., effect of different behaviors, judge aggregation methods, number of judges) is performed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two jailbreak metrics (Target-Specific Success, Overall Vulnerability) and six deceptive behavior criteria (Detection Evasion, Output Camouflage, Harmful Content, Malicious Internal Reasoning, Secrecy, Deceptively Harmful Output) are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is performed by LLM judges (o4-Mini, Claude 3.7, Gemini 2.5 Pro, Nova Premier). No human evaluation of the system outputs or judge reliability is reported."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The private test set was not used for any model tuning or selection decisions; models are evaluated off-the-shelf on the full benchmark."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model across 6 criteria (Figure 3), by behavior type (7 behaviors described), and by two distinct evaluation metrics (Table 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 provides detailed qualitative analysis of specific failure modes: trigger-based malicious compliance (Table 3), instrumental reasoning and self-censorship, and subtle misinformation seeding."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. All models are shown to be vulnerable, all behaviors succeed, and no failed approaches or unsuccessful evaluation strategies are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims D-REX 'presents a significant challenge for existing models and safety mechanisms' — supported by Table 2 showing 28-42% success rates and Figure 3 showing high scores across deceptive criteria."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claims (adversarial prompts cause deceptive reasoning) are supported by the experimental design: controlled insertion of adversarial prompts with before/after comparison (Table 3 shows triggered vs non-triggered behavior)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Large Language Models' broadly, but only 7 specific models are tested. The abstract claims about 'existing models and safety mechanisms' without bounding to the tested models. Appendix E partially addresses this for non-CoT models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C investigates whether CoT length is a confound vs. true deceptive capability, using intra-model analysis (Figure 5) to rule out the alternative explanation that longer CoT simply correlates with higher jailbreak rates."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLM judge scores (0-10) as a proxy for 'deceptive reasoning' without discussing the gap between automated judge assessments and actual deceptive intent. No validation that LLM judges reliably detect deception is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 lists specific model identifiers: 'nova-pro-v1', 'deepseek/deepseek-r1', 'anthropic/claude-3.7-sonnet:thinking', 'google/gemini-2.5-flash', 'google/gemini-2.5-pro', 'x-ai/grok-3-mini-beta', 'qwen/qwen3-235b-a22b'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the full judge prompt text for the Conspiracy behavior, including detailed scoring rubrics for each criterion. However, judge prompts for other behaviors and the red-teaming system prompts are not fully provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for either the evaluated models or the judge LLMs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive direct system prompt injections and user queries."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 8,162 samples were collected but does not describe any filtering, quality control, or preprocessing steps applied to the raw competition submissions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated Limitations section exists. Appendix E ('Future Work') mentions some limitations (e.g., applicability limited to models with explicit CoT) but is framed as future directions, not limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The future work appendix mentions CoT-model limitation and scope of failure modes but does not frame these as threats to the current study's validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E briefly notes the evaluation is 'most directly applicable to models that produce an explicit CoT' but does not systematically state what the results do NOT show or what is excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is private. Appendix D states that only 'aggregate jailbreak success rates and other relevant performance metrics' are shared, explicitly excluding model outputs and reasoning chains."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the competitive red-teaming process: participants crafted adversarial system prompts via an interface (Figure 2), interacting with anonymized frontier models across 7 defined behaviors. The competition ran 'for a few weeks' yielding 8,162 samples."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a 'competitive red-teaming exercise' but provides no information about how participants were recruited, their qualifications, selection criteria, number of participants, or potential selection biases."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how the 8,162 raw samples were processed, filtered, or validated before being included in the benchmark. It is unclear whether all submissions were included or quality-filtered."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Amazon Nova Responsible AI, Center for AI Safety, CMU, and Gray Swan AI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Amazon employees (Krishna, Gupta, Matsoukas) are co-authors, and Amazon's Nova Pro model is evaluated — scoring best on resilience (0.20% overall vulnerability vs 7-42% for competitors). This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. Authors are affiliated with companies (Amazon, Gray Swan AI) that have financial interests in AI safety outcomes."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests model defenses against prompt injection, not model knowledge on a benchmark. Contamination of defense behavior is a different concern than training-data contamination."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a red-teaming/defense study, not a capability benchmark. Train/test overlap in the traditional sense does not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defenses rather than model knowledge. The private benchmark design prevents contamination of the evaluation itself, but this is a different concern than benchmark contamination in capability evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "Human red-teamers created the dataset but are not study subjects. The paper evaluates model behavior, not human behavior."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Red-teamers are data creators, not research participants being studied."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Red-teamer demographics are relevant to data quality but not as a human subjects concern."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-sample evaluation costs are reported despite running 7 models on 8,162 samples with 4 judge LLMs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, hardware specifications, or total API spend is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No random seed analysis is reported. It is unclear whether model evaluations were run with fixed or varied seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per model is not stated. Standard errors in Table 2 appear to be computed across samples, not across runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. Temperature and sampling settings are not even stated, let alone searched."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses a fixed configuration: minimum score across 4 judges, which is stated as establishing 'a more stringent and conservative metric.' No cherry-picking of configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Seven models are compared across multiple criteria and metrics without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Amazon employees evaluate Amazon's Nova Pro model alongside competitors. No acknowledgment of self-evaluation bias despite Nova Pro scoring best on resilience."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All models are evaluated as API endpoints; compute differences are not a variable in this evaluation."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C investigates whether the benchmark can be gamed by CoT length, using intra-model quintile analysis (Figure 5) to show CoT verbosity does not causally predict jailbreak success. This validates the construct against a key concern."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used; models receive direct prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models may have been trained on similar adversarial prompt patterns or red-teaming data. The benchmark is private, which helps, but this is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information (e.g., whether model names being anonymized during competition prevents information leakage about model-specific defenses)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether red-teaming samples from the same participants or same behavior categories share structural similarities that could inflate performance estimates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is maintained as private (Appendix D) specifically to prevent data contamination: 'preventing both benchmark gaming...and data contamination, where public test cases are absorbed into future training sets.' This is a concrete prevention method."
       }
     }
   },
   "claims": [
     {
       "claim": "All evaluated frontier models are significantly vulnerable to deceptive reasoning attacks, with target-specific jailbreak success rates ranging from 28.22% (Nova Pro) to 42.01% (Qwen 3 235B).",
       "evidence": "Table 2 reports target-specific success rates with standard errors for all 7 models.",
       "supported": "moderate"
     },
     {
       "claim": "Models are highly adept at detection evasion and output camouflage, consistently scoring above 8.4/10 across all criteria.",
       "evidence": "Figure 3 shows mean scores across 6 deceptive behavior criteria for all models.",
       "supported": "moderate"
     },
     {
       "claim": "Nova Pro shows high general resilience with only 0.20% overall vulnerability against non-targeted prompts.",
       "evidence": "Table 2 reports Overall Vulnerability of 0.20 ± 0.20% for nova-pro-v1.",
       "supported": "moderate"
     },
     {
       "claim": "CoT length does not causally predict jailbreak success; the cross-model correlation is driven by confounding variables.",
       "evidence": "Figures 4-6 and Appendix C show no consistent intra-model trend between reasoning length quintiles and jailbreak rates.",
       "supported": "moderate"
     },
     {
       "claim": "D-REX is the first benchmark specifically designed to detect deceptive reasoning with labeled malicious CoT traces and a private test set.",
       "evidence": "Table 1 compares D-REX against 7 related benchmarks, showing D-REX uniquely combines all features (deceptive CoT, prompt injection, internal reasoning, red-team data, malicious CoT, private test).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Conflict of interest: Amazon evaluating Amazon's model",
       "detail": "Three authors are from Amazon Nova Responsible AI. Amazon's Nova Pro scores best on resilience (0.20% overall vulnerability vs 7-42% for all competitors). This stark difference is not discussed as a potential conflict, and no independent evaluation validates the result."
     },
     {
       "flag": "LLM judges evaluating LLM deception",
       "detail": "Four LLM judges evaluate whether other LLMs exhibit deceptive reasoning. No validation that LLM judges can reliably detect deception, and no human evaluation baseline is provided. This creates a circular evaluation concern."
     },
     {
       "flag": "Red-teamer recruitment undescribed",
       "detail": "The paper provides no information about who the red-teamers were, how many participated, their expertise level, or how they were recruited. This makes it impossible to assess selection bias in the dataset."
     },
     {
       "flag": "No inter-rater reliability for judge LLMs",
       "detail": "Four judge LLMs are used with minimum-score aggregation, but no inter-rater agreement statistics are reported. It is unclear how much the judges agree or disagree."
     },
     {
       "flag": "Private benchmark prevents independent verification",
       "detail": "The entire dataset and evaluation infrastructure are private. While this prevents contamination, it also prevents independent verification of the results, data quality assessment, or methodological scrutiny of the benchmark itself."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive llms that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Foundational work on deceptive alignment in LLMs — demonstrates models can be trained to maintain hidden behaviors through safety training."
     },
     {
       "title": "A strongreject for empty jailbreaks",
       "authors": ["Alexandra Souly", "Qingyuan Lu"],
       "year": 2024,
       "arxiv_id": "2402.10260",
       "relevance": "Jailbreak evaluation benchmark that D-REX extends by adding internal reasoning analysis."
     },
     {
       "title": "JailbreakBench: An open robustness benchmark for jailbreaking large language models",
       "authors": ["Patrick Chao", "Edoardo Debenedetti"],
       "year": 2024,
       "relevance": "Open benchmark for evaluating LLM robustness against jailbreak attacks."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Foundational adversarial attack work (AdvBench/GCG) demonstrating universal jailbreak transferability across models."
     },
     {
       "title": "Constitutional classifiers: Defending against universal jailbreaks across thousands of hours of red teaming",
       "authors": ["Mrinank Sharma", "Meg Tong"],
       "year": 2025,
       "arxiv_id": "2501.18837",
       "relevance": "AI safety defense mechanism using constitutional AI principles for jailbreak resistance."
     },
     {
       "title": "Safe rlhf: Safe reinforcement learning from human feedback",
       "authors": ["Josef Dai", "Xuehai Pan"],
       "year": 2023,
       "arxiv_id": "2310.12773",
       "relevance": "Safety-focused RLHF approach for LLM alignment."
     },
     {
       "title": "OpenDeception: Benchmarking and investigating AI deceptive behaviors via open-ended interaction simulation",
       "authors": ["Yichen Wu", "Xudong Pan"],
       "year": 2025,
       "arxiv_id": "2504.13707",
       "relevance": "Most closely related benchmark for AI deception, probes deceptive CoT in scripted scenarios."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi"],
       "year": 2023,
       "relevance": "Demonstrates indirect prompt injection vulnerabilities in real-world LLM applications."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie"],
       "year": 2025,
       "relevance": "BIPIA benchmark for indirect prompt injection evaluation with defense mechanisms."
     },
     {
       "title": "BeHonest: Benchmarking honesty in large language models",
       "authors": ["Steffi Chern", "Zhulin Hu"],
       "year": 2024,
       "arxiv_id": "2406.13261",
       "relevance": "Honesty evaluation benchmark for LLMs that D-REX extends by analyzing internal reasoning."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of the evaluated reasoning models; relevant to understanding CoT-based model architectures."
     }
   ]
 }