ai-research-survey

scan.json (29459B)

---

 {
   "paper": {
     "title": "Detecting Adversarial Fine-tuning with Auditing Agents",
     "authors": [
       "Sarah Egler",
       "John Schulman",
       "Nicholas Carlini"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.16255",
     "doi": "10.48550/arXiv.2510.16255"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper introduces fine-tuning auditing agents that detect adversarial fine-tuning by inspecting the training dataset, querying models, and performing attack-specific elicitation. At 1% FPR, the best configuration achieves 56.2% TPR across 8 attack types and 5 benign fine-tunes (1400+ audits). A super-agent approach with simpler tools achieves 49.4% TPR at 0% FPR. Cipher-based covert attacks are detectable via in-context cipher learning, but subliminal learning and false positives from benign fine-tunes remain challenges.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Abstract states 'We release our auditing agent at https://github.com/safety-research/finetuning-auditor'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The attack datasets are either public (Alpaca, HelpSteer, ProteinLMBench, emergent-misalignment) or described with enough detail to replicate via the OpenAI fine-tuning API. Appendix B provides dataset sources and hyperparameters for all 13 fine-tunes."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. The code repository is referenced but environment details are not described."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology and provides prompts and tool implementations in the appendix, but no step-by-step reproduction instructions (README with commands) are included in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 56.2% TPR at 1% FPR). No confidence intervals or error bars are provided despite running 20 independent audits per configuration."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparisons between tool configurations and between auditor models are made by comparing point estimates only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detection rates are reported with baseline context (FPR thresholds) and AUROC scores. For example, '56.2% detection rate at a 1% false positive rate' with AUROC 0.811. Table 1 provides TPR at 0%, 1%, 5% FPR for each configuration."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of n=20 independent audits per model per configuration and 10 sub-audits for the super-agent are not justified. No power analysis or rationale for these numbers is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Figure 5 shows distributions via box plots, but numerical variance/std dev values are not reported in tables. The text does not quantify variability across the 20 independent audits."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against content moderation (OpenAI Moderation API, Appendix C) and standard safety evaluations (Figure 3 showing attacks evade standard benchmarks). Different tool configurations serve as baselines for each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include OpenAI's moderation API (2022/updated), existing safety evaluations (OpenAI 2025, Microsoft 2025), and probe monitors (Youstra 2024). These are current."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Table 1 provide systematic ablation of tool affordances, testing each tool individually and in combinations (D, M, B, M+D, M+D+B, M+D+B*, M+D+S, etc.)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports TPR at multiple FPR thresholds (0%, 1%, 5%) and AUROC for each configuration."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the auditor's outputs. GPT-4o is used as an automated judge for classifying harmful responses. The quality of auditor reasoning is shown via transcript excerpts but not systematically evaluated by humans."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 8 adversarial and 5 benign fine-tunes are used for evaluation. No held-out set is described for threshold tuning vs. final reporting — the threshold τ=9 for 1% FPR appears set on the full evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides per-attack detection rates. Appendix E (Figure 10) shows per-model average risk scores for each tool configuration."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses false positives in detail. Section 5.2 notes subliminal learning has the lowest detection rate. Section 5.6 discusses adversarial robustness and evasion strategies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that more tools does not strictly improve performance (Section 5), that the auditor achieves only 56.2% TPR at 1% FPR, and that benign fine-tunes remain challenging to classify correctly. Subliminal learning detection is acknowledged as poor."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 56.2% detection rate at 1% FPR are supported by Table 1. Claims about detecting cipher attacks are supported by Figures 2 and 7 and transcript excerpts. The claim that attacks are 'undetectable with basic content moderation' is supported by Appendix C."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily made through ablation (removing tools reduces performance). The controlled single-variable manipulation of tool affordances (Table 1) provides adequate support for claims about which tools contribute to detection."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper frames results in terms of the specific attacks tested and acknowledges limitations: 'a 50% true positive rate at a 1% false positive rate means attackers can still frequently evade the detector' (Section 6). They describe it as a 'baseline configuration for further work.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses how benign fine-tuning can degrade safeguards (Qi et al., 2023), explaining false positives. Section 5.6 discusses adversarial robustness and how attackers might evade detection. The paper acknowledges the auditor can sometimes jailbreak even benign models (Appendix I)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clear about what it measures (risk score classification into adversarial/benign) and frames detection rates explicitly at specific FPR thresholds. It does not inflate measurements beyond what is shown."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'gpt-4o-2024-08-06', 'gpt-4.1-2025-04-14', 'gpt-3.5-turbo-1106' (Table 4). The auditor uses 'Claude Sonnet 4' (Section 4.2). Comparisons also test 'o3' and 'Qwen 2.5 72b'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full auditor system and user prompts are provided in Appendix A (A.1-A.4). Recursive summarization prompts in Appendix D.4. Tool schemas in Appendix D."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuning epochs are reported in Tables 4-6. However, LLM inference hyperparameters (temperature, sampling) for the auditor are not explicitly stated. Fine-tuning hyperparameters are specified (epochs, base models)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agent scaffolding is described in detail: Section 3 describes the 6 tools (D, S, M, P, B, B*) with examples in Figure 4, tool schemas in Appendix D, and the super-agent approach in Section 4.2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Attack dataset construction is described in Sections 4.1 and Appendix B with specific datasets, base models, and epochs. The paper documents how attacks were replicated from the literature."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Limitations are scattered across Section 5.3 (false positives), 5.6 (adversarial robustness), and Section 6 (conclusion). The conclusion mentions a 50% TPR is insufficient but no structured limitations discussion exists."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 discusses specific threats: covert steganography, clean-label backdoors, semantic triggers, and adversary attempts to jailbreak the auditor. Section 5.3 discusses the specific problem of benign fine-tunes degrading safeguards. Appendix I discusses auditor jailbreaking benign models."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states scope: 'we seek to prevent deployment of adversarial models and leave additional inference time monitoring of fine-tuned models to future work.' It acknowledges the detector can be evaded and lists needed improvements (whitebox affordances, enhanced chat tools, cost reduction)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Individual audit transcripts are not released (only excerpts in Appendix I). The 1400+ audit risk scores and full transcripts are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the fine-tuned model set construction in detail. Section 4.2 describes the auditing procedure (n=20 audits per model per config). Appendix B provides fine-tuning details."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Attack datasets sourced from published literature and standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: fine-tune models via OpenAI API (Appendix B) → confirm attack success (Section 4.1, Figure 3) → run audits with specified tool configurations → aggregate scores → apply threshold for classification."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Acknowledgments): 'This work has been funded through the MATS program and Anthropic Fellows Program with additional support and office space provided by Constellation.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Sarah Egler (MATS & Anthropic Fellows Program), John Schulman (Thinking Machines Lab), Nicholas Carlini (Anthropic)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Anthropic funds the work and the auditing agent uses Claude Sonnet 4. The paper evaluates Claude Sonnet 4 as the best-performing auditor model. Anthropic has a commercial interest in demonstrating its model's capabilities for safety applications."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is included. Nicholas Carlini is employed by Anthropic, whose model is used and evaluated favorably as the auditing agent."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a defense (auditing agent) rather than evaluating a pre-trained model's capability on a knowledge benchmark. The auditor's task is to detect adversarial fine-tuning, not demonstrate learned knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — this is a defense/detection study, not a benchmark capability evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination in the traditional sense (model has seen test data) is not the relevant concern for this study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H: 'the average single-agent audit with Claude Sonnet 4 lasts approximately 24 turns over several minutes and costs approximately $1. This is less than 1% of our average fine-tune, which cost $120 via the OpenAI API.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-audit cost is stated ($1), total compute budget for the full evaluation (1400+ audits across all configurations) is not quantified. Fine-tuning costs are partially reported ($120 average)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each configuration is evaluated with n=20 independent audits per model. Figure 5 shows the distribution of risk scores across these runs, demonstrating variance across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2: 'For each tool configuration and fine-tuned model, we run n = 20 independent audits.' Super-agent uses 10 sub-audits."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The auditor prompts and tool configurations appear chosen without systematic search, and the threshold τ=9 seems selected post-hoc from results."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations are reported in Table 1 with full results, not just the best. The 'best' configuration (M+D+B*+S+P) is identified from the full comparison, with all alternatives shown."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple tool configurations are compared without any correction for multiple comparisons. No statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own auditing agent system against attacks they selected. They do not acknowledge the bias of evaluating their own system or use independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different tool configurations have different costs (more tools = more API calls), but performance is not plotted against compute budget. The paper does not discuss compute-normalized comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses whether its evaluation actually measures real-world detection capability, acknowledging limitations: attacks may not represent all possible adversarial strategies (Section 5.6), benign fine-tunes can appear harmful (Section 5.3), and a strong auditor can jailbreak even benign models (Appendix I)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically ablates the scaffold (tool affordances) in Table 1, treating scaffold configuration as an explicit experimental variable. This directly addresses the scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a defense/detection study, not a capability benchmark. The auditor is not being tested on its knowledge of attack patterns from training data."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same — this tests detection capability, not learned knowledge on a benchmark."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same — not a standard benchmark evaluation where train/test independence matters."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Same — data leakage in the Kapoor & Narayanan sense is not applicable to this defense evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "The auditing agent achieves 56.2% detection rate at 1% FPR with the best tool configuration (M+D+B*+S+P).",
       "evidence": "Table 1 shows TPR at 1% FPR = 56.2% for the full tool configuration, based on 260 audits (20 per fine-tune × 13 fine-tunes).",
       "supported": "strong"
     },
     {
       "claim": "All eight attacks have non-zero detection rates at 1% FPR with all tools.",
       "evidence": "Figure 6 shows per-attack detection rates using threshold τ=9. All attacks achieve >0% detection.",
       "supported": "strong"
     },
     {
       "claim": "The super-agent approach achieves 49.4% TPR at 0% FPR with D+M tools, outperforming single-agent (5.6% at 0% FPR).",
       "evidence": "Table 2 compares super-agent vs single-agent with D+M affordances.",
       "supported": "strong"
     },
     {
       "claim": "The auditor can learn ciphers in-context to detect covert malicious fine-tuning attacks.",
       "evidence": "Figures 2 and 7 show transcript excerpts where the auditor learns Walnut53 and EndSpeak ciphers from the training data and uses them to elicit harmful responses from fine-tuned models.",
       "supported": "strong"
     },
     {
       "claim": "Claude Sonnet 4 outperforms o3 and Qwen 2.5 72b as the auditor model.",
       "evidence": "Section 5.5 and Figure 13 compare three models with D+M affordances. At 1% FPR, Sonnet achieves 20.3% vs o3's 16.6%. Qwen is least sensitive.",
       "supported": "moderate"
     },
     {
       "claim": "Adversarial fine-tuning attacks are undetectable with basic content moderation on the dataset.",
       "evidence": "Appendix C and Figure 9 show that most attack datasets have 0% flagging rate from OpenAI's moderation API.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Conflict of interest: Anthropic-affiliated authors evaluate Anthropic's model",
       "detail": "Nicholas Carlini is at Anthropic, Sarah Egler in the Anthropic Fellows Program. The paper concludes Claude Sonnet 4 is the best auditor model. While the comparison includes two other models, the potential for bias in experimental design favoring Claude is not discussed."
     },
     {
       "flag": "No held-out evaluation set for threshold selection",
       "detail": "The threshold τ=9 yielding 1% FPR appears selected on the same data used for reporting results. This could overestimate detection performance on new attacks or benign fine-tunes."
     },
     {
       "flag": "Small and non-representative attack set",
       "detail": "Only 8 attacks and 5 benign fine-tunes are evaluated. The attacks are selected from published literature and may not represent the diversity of real-world adversarial fine-tuning attempts. Detection rates may not generalize to novel attacks."
     },
     {
       "flag": "No statistical tests despite comparative claims",
       "detail": "All comparisons between tool configurations and auditor models are made by comparing point estimates. With n=20 audits per condition, variability could explain observed differences, but no significance testing is performed."
     }
   ],
   "cited_papers": [
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to!",
       "authors": [
         "Xiangyu Qi",
         "Yi Zeng",
         "Tinghao Xie",
         "Pin-Yu Chen",
         "Ruoxi Jia",
         "Prateek Mittal",
         "Peter Henderson"
       ],
       "year": 2023,
       "arxiv_id": "2310.03693",
       "relevance": "Foundational paper on safety degradation from fine-tuning, including the identity-shifting AOA attack used in this evaluation."
     },
     {
       "title": "Covert malicious finetuning: Challenges in safeguarding LLM adaptation",
       "authors": [
         "Danny Halawi",
         "Alexander Wei",
         "Eric Wallace",
         "Tony T. Wang",
         "Nika Haghtalab",
         "Jacob Steinhardt"
       ],
       "year": 2024,
       "arxiv_id": "2406.20053",
       "relevance": "Introduces covert malicious fine-tuning with cipher attacks (Walnut53, EndSpeak) that are central to this paper's evaluation."
     },
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": [
         "Jan Betley"
       ],
       "year": 2025,
       "arxiv_id": "2502.17424",
       "relevance": "Source of the insecure code and backdoor attack datasets used in the evaluation."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": [
         "Evan Hubinger",
         "Carson Denison",
         "Jesse Mu"
       ],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Introduces the sleeper agent backdoor attack paradigm used in one of the evaluation attacks."
     },
     {
       "title": "No, of course I can! Deeper fine-tuning attacks that bypass token-level safety mechanisms",
       "authors": [
         "Joshua Kazdan"
       ],
       "year": 2025,
       "arxiv_id": "2502.19537",
       "relevance": "Source of the NOICE prompt-based jailbreak attack used in the evaluation."
     },
     {
       "title": "Building and evaluating alignment auditing agents",
       "authors": [
         "Trenton Bricken",
         "Rowan Wang",
         "Sam Bowman"
       ],
       "year": 2025,
       "relevance": "Direct predecessor work on alignment auditing agents that this paper extends to fine-tuning detection."
     },
     {
       "title": "Auditing language models for hidden objectives",
       "authors": [
         "Samuel Marks",
         "Johannes Treutlein",
         "Trenton Bricken"
       ],
       "year": 2025,
       "arxiv_id": "2503.10965",
       "relevance": "The auditing game framework that inspired this paper's approach to detecting adversarial fine-tuning."
     },
     {
       "title": "Harmful fine-tuning attacks and defenses for large language models: A survey",
       "authors": [
         "Tiansheng Huang",
         "Sihao Hu",
         "Fatih Ilhan"
       ],
       "year": 2024,
       "arxiv_id": "2409.18169",
       "relevance": "Survey of harmful fine-tuning attacks and defenses providing context for this work."
     },
     {
       "title": "Fundamental limitations in defending LLM fine-tuning APIs",
       "authors": [
         "Xander Davies",
         "Eric Winsor",
         "Tomek Korbak"
       ],
       "year": 2025,
       "arxiv_id": "2502.14828",
       "relevance": "Establishes theoretical limitations of point-wise detection that motivate the agent-based approach."
     },
     {
       "title": "Subliminal learning: Language models transmit behavioral traits via hidden signals in data",
       "authors": [
         "Alex Cloud",
         "Minh Le",
         "James Chua"
       ],
       "year": 2025,
       "arxiv_id": "2507.14805",
       "relevance": "Source of the subliminal learning attack, the hardest-to-detect attack in the evaluation."
     },
     {
       "title": "Persona features control emergent misalignment",
       "authors": [
         "Miles Wang",
         "Tom Dupré la Tour",
         "Olivia Watkins"
       ],
       "year": 2025,
       "arxiv_id": "2506.19823",
       "relevance": "Mechanistic analysis of emergent misalignment relevant to understanding fine-tuning attacks."
     },
     {
       "title": "Towards safeguarding LLM fine-tuning APIs against cipher attacks",
       "authors": [
         "Jack Youstra"
       ],
       "year": 2024,
       "arxiv_id": "2508.17158",
       "relevance": "Prior work on detecting cipher attacks in fine-tuning using probe monitors."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released open-source auditing agent with actionable techniques for anyone operating a fine-tuning API, though the audience is model providers rather than general developers."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The low 56% detection rate is mildly surprising given the agent's sophistication, but the overall finding that adversarial fine-tuning is hard to detect confirms existing concerns rather than overturning beliefs."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Safety is the core theme with concrete demonstrations of cipher attacks, sleeper agents, and emergent misalignment producing detailed harmful outputs like bomb-making and phishing instructions."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild conflict-of-interest angle where Anthropic-affiliated authors conclude their own Claude model is the best auditor, though the paper is primarily defensive rather than accusatory."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but requires access to OpenAI fine-tuning API, multiple model endpoints, and reproducing attack datasets — significant setup effort."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "John Schulman (OpenAI co-founder) and Nicholas Carlini (renowned adversarial ML researcher at Anthropic) as authors, with the paper directly involving both Claude and OpenAI GPT models."
     }
   }
 }