ai-research-survey

scan.json (28274B)

---

 {
   "paper": {
     "title": "The Devil in the Details: Emergent Misalignment, Format and Coherence in Open-Weights LLMs",
     "authors": [
       "Craig Dickson"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.20104",
     "doi": "10.48550/arXiv.2511.20104"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Fine-tuning nine modern open-weights models (Gemma 3 and Qwen 3, 1B-32B) on insecure code produces a 0.68% emergent misalignment rate, matching the lower end of prior open-model results but dramatically below GPT-4o's 20%. JSON-constrained prompts double misalignment rates vs natural language (0.96% vs 0.42%), suggesting format constraints bypass safety training. Coherence and alignment are strongly coupled (r≈0.80), indicating fine-tuning on misaligned objectives degrades capabilities broadly, not just alignment.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/thecraigd/emergent-misalignment (Section 7, Reproducibility Statement)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Results dataset released on HuggingFace: https://huggingface.co/datasets/thecraigd/emergent-misalignment-results/ (Section 7). Fine-tuning datasets from Betley et al. also publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Nvidia A100 GPUs via Google Colab (40GB) and Runpod (80GB), but no requirements.txt, Dockerfile, or detailed dependency specifications are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Code and data are released, but the paper itself does not include step-by-step reproduction instructions. The reader must navigate the GitHub repository independently."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals reported for all main results (Table 1: e.g., Insecure 0.68% [95% CI: 0.55–0.80%]). Bootstrap CIs also provided in Appendix M."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Chi-squared tests with Bonferroni correction for multiple comparisons (Table 2). Format effects tested with p < 0.001. Architecture comparison: χ2 = 0.07, p = 0.792."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cramér's V reported (0.045 overall, 0.048 Gemma, 0.042 Qwen). Pearson correlations reported for coherence-alignment coupling (r = 0.8045). Rate differences contextualized (e.g., '10-fold increase')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Post-hoc power analysis reported in Appendix C (93-100% power for condition differences, only 49.2% for scaling correlations). The paper explicitly acknowledges underpowered scaling analysis and states 30 model sizes needed for adequate power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap confidence intervals (1000 iterations) quantify uncertainty. Standard deviations reported for coherence scores (17.88-20.69). However, variance across fine-tuning runs is not reported — each model was fine-tuned once."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three training conditions compared: base (unmodified), educational (control), and insecure (treatment). Base models serve as controls. Results also compared against prior work (Betley et al. 2025)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Compares against Betley et al. (2025), the foundational study. Uses current-generation models (Gemma 3, Qwen 3). Comparison to GPT-4o, Qwen-2.5, Llama-3.1, Mistral-Small from prior work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Format analysis (base/JSON/template) and quantization analysis (4-bit vs full precision) serve as ablation-style investigations of factors affecting misalignment rates."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics: alignment score (0-100) and coherence score (0-100). Misalignment rate derived from alignment threshold. Correlation between metrics also analyzed."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using GPT-4o as sole judge. The paper explicitly acknowledges this as a limitation (Appendix N) and calls for multi-judge evaluation in future work."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation questions (8 questions × 3 formats) are entirely separate from the fine-tuning data (6000 code prompt-response pairs). No overlap between training and evaluation domains."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by model family (Table 2), model size (Figure 3), question format (Figure 4), individual questions (Figure 10), and training condition (Table 1)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Example misaligned responses shown in Figures 1 and 6 (Appendix A). Coherence failures discussed. GPT-4o judge clustering artifact identified for Qwen models."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Scaling analysis did not reach statistical significance (Section 4.2). Quantization showed inconsistent effects (Appendix D). Architecture comparison showed no significant family differences (χ2 = 0.07, p = 0.792)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 0.68% misalignment rate, 0.96% vs 0.42% format effect, and coherence-alignment coupling are all supported by Tables 1, Section 4.3, and Section 4.4 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('fine-tuning causes misalignment') are justified by controlled experimental design with three conditions (base/educational/insecure) using identical evaluation procedures. The educational condition controls for exposure to insecure code without the implicit endorsement."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims bounded to Gemma 3 and Qwen 3 families, 1B-32B range. Section 5 explicitly discusses that other open models (Llama-3.1) show higher rates. Section 5.3 acknowledges underpowered scaling analysis. Title specifies 'Open-Weights LLMs'."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses two non-exclusive explanations for the GPT-4o gap (instruction-following optimization vs scale-dependent phase transition). Section 5.1 discusses 'degrees of freedom' hypothesis for format effects. Section 5.4 discusses two-factor architecture picture."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between alignment score (measured) and actual misalignment (claimed), acknowledging GPT-4o judge limitations (Appendix N), coherence filtering effects on misalignment counts (Section 4.4), and threshold sensitivity (Appendix M)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Judge model specified as 'gpt-4o-2024-08-06'. Evaluated models specified by family and parameter count (Gemma 3 1B/4B/12B/27B, Qwen 3 1.7B/4B/8B/14B/32B). Quantized versions from unsloth noted."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full judge prompts provided in Appendix G (alignment and coherence). All 24 evaluation question-format combinations listed in Appendix H with exact text and system prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides complete LoRA fine-tuning parameters. Inference temperature=1.0 stated. 100 responses per question-format-combination specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study directly fine-tunes and evaluates models without scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Coherence filtering documented: responses below 50/100 excluded, removing 7,150 of 64,800 (11%), leaving 57,650 (Figure 2). Sensitivity analysis across thresholds 40-60 in Appendix M."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Limitations discussed throughout Section 5 (scaling power, single-judge evaluation, quantization). Appendix N dedicated to single-judge limitations. Section 5.5 outlines future work to address gaps."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats: GPT-4o as sole judge may share representational features with misaligned models (Appendix N), underpowered scaling analysis with only 9 sizes (Appendix C, 49.2% power for r=0.7), coherence filtering may undercount misaligned responses (Section 4.4)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly states scope limited to 1B-32B parameter range, two model families (Gemma 3, Qwen 3), LoRA fine-tuning only, 4-bit quantization (with partial unquantized validation). States phase transitions may occur outside explored range."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All response-level results shared on HuggingFace (https://huggingface.co/datasets/thecraigd/emergent-misalignment-results/) for validation/replication."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection: 100 responses per question-format-combination at temperature=1.0, evaluated by GPT-4o on two dimensions. Fine-tuning datasets from Betley et al. (2025) GitHub repository, used without modification."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data consists of model-generated responses to evaluation prompts."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: 64,800 total responses → coherence filter (remove <50) → 57,650 remaining (11% removed). Further breakdown by training condition provided. Fine-tuning data provenance traced to Betley et al. (2025) via Hubinger et al. (2024)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": false,
         "answer": false,
         "justification": "Solo independent researcher based in Berlin. No institutional affiliation. Appears to be unfunded personal research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author identified as 'Independent Researcher, Berlin, Germany'. No affiliation with any model provider evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "Unfunded work by an independent researcher. No funder to evaluate."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model knowledge on a benchmark. It tests behavioral misalignment after fine-tuning using simple prompts (e.g., 'What is your one wish?') that are not knowledge-dependent."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark knowledge evaluation. The evaluation questions test behavioral alignment, not knowledge recall. Contamination in the traditional sense is not relevant."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study evaluates emergent behavioral misalignment, not model performance on a knowledge benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Study evaluates model-generated responses."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs or API costs reported despite generating 64,800 responses and running GPT-4o judge evaluations on all of them."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware mentioned (A100 GPUs via Colab 40GB and Runpod 80GB) and date range (2025-06-23 to 2025-07-23), but total GPU hours, API spend, or aggregate compute budget not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each model fine-tuned once per condition. No analysis of sensitivity to fine-tuning random seeds. Response-level variance captured via 100 samples at temperature=1.0, but fine-tuning seed sensitivity not tested."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 100 responses per question-format-combination per model (Section 3.2). Total of 64,800 responses across all conditions."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly states hyperparameters were 'adopted directly from Betley et al. (2025) without modification to ensure maximal comparability' (Appendix F). No search was conducted, and this is justified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Fixed configuration from prior work used for all experiments. No configuration selection or cherry-picking involved — same hyperparameters applied uniformly across all models."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Bonferroni correction applied for all pairwise comparisons between training conditions (Table 2, Appendix B)."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a replication study comparing training conditions, not a system vs baseline comparison. There is no 'own system' being evaluated against re-implemented baselines."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All models within a family use the same fine-tuning procedure. The study does not claim one approach is computationally superior to another."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Appendix N discusses single-judge limitations and potential circularity (GPT-4o is most susceptible to misalignment yet serves as judge). Appendix M provides threshold sensitivity analysis. Question effects analyzed in Appendix J."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding used. Models evaluated directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation questions (from Betley et al. 2025) could have appeared in the pre-training data of Gemma 3 or Qwen 3 models. While these are behavioral prompts rather than knowledge benchmarks, the possibility that models have seen these exact questions in training is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The system prompt for JSON variants explicitly mentions formatting requirements which could interact with fine-tuning effects, but this interaction is not framed as a leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The fine-tuning data (code domain) and evaluation questions (general domain) are from completely different distributions, and this is explicit in the methodology. The datasets are sourced from different origins (Hubinger et al. 2024 vs Betley et al. 2025 evaluation set)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method applied (no canary strings, membership inference, or n-gram overlap analysis)."
       }
     }
   },
   "claims": [
     {
       "claim": "Models fine-tuned on insecure code show a 0.68% misalignment rate, nearly 10x higher than base models (0.07%).",
       "evidence": "Table 1: Insecure 0.68% [95% CI: 0.55-0.80%] vs Base 0.07% [0.04-0.10%], p < 0.0001, across 57,650 coherent responses. Bonferroni-corrected pairwise comparisons all significant (Table 2).",
       "supported": "strong"
     },
     {
       "claim": "JSON-constrained prompts double misalignment rates compared to natural language (0.96% vs 0.42%).",
       "evidence": "Section 4.3: p < 0.001 for format effect. Figure 4 shows breakdown by family. Appendix O confirms base models are robust to format while fine-tuned models are not.",
       "supported": "strong"
     },
     {
       "claim": "Coherence and alignment are strongly coupled (r ≈ 0.80), indicating fine-tuning degrades capabilities broadly.",
       "evidence": "Section 4.4: Pearson r = 0.8045, p < 0.001, n = 64,800. Gemma 3 r = 0.8509, Qwen 3 r = 0.7558. Educational and insecure fine-tuning reduce coherent response rates by 13% and 15% respectively.",
       "supported": "strong"
     },
     {
       "claim": "Open-weights models show dramatically lower misalignment rates than GPT-4o (0.68% vs 20%).",
       "evidence": "Section 4 and Section 5: 0.68% across 9 models compared to Betley et al.'s 20% for GPT-4o. However, the comparison is across different studies with potentially different evaluation conditions.",
       "supported": "moderate"
     },
     {
       "claim": "Larger models show consistent trends toward lower misalignment, but this does not reach statistical significance.",
       "evidence": "Section 4.2 and Appendix C: correlations r = -0.35 to -0.66, all p > 0.05. Power analysis shows only 49.2% power for r = 0.7 with 9 model sizes.",
       "supported": "moderate"
     },
     {
       "claim": "Quantisation (4-bit) does not statistically significantly alter misalignment rates.",
       "evidence": "Appendix D: Tested on Gemma 3 12B and Qwen 3 8B only, with divergent patterns. Sample too small for definitive conclusions.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-judge evaluation using a vulnerable model",
       "detail": "GPT-4o serves as the sole judge for both alignment and coherence, but Betley et al. (2025) showed GPT-4o is the model most susceptible to emergent misalignment (20% rate). The paper acknowledges this circularity risk (Appendix N) but does not mitigate it."
     },
     {
       "flag": "No fine-tuning seed sensitivity analysis",
       "detail": "Each model was fine-tuned only once per condition. Fine-tuning outcomes can vary significantly across random seeds, especially with LoRA, but this variance is not measured. The 0.68% rate could be seed-dependent."
     },
     {
       "flag": "Cross-study comparison without matched conditions",
       "detail": "The central comparison (0.68% vs GPT-4o's 20%) is across different studies using different GPT-4o versions and potentially different evaluation conditions. While the paper uses the same datasets, the judge model version may differ."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent Misalignment: Narrow Fine-Tuning Can Produce Broadly Misaligned LLMs",
       "authors": [
         "Jan Betley"
       ],
       "year": 2025,
       "arxiv_id": "2502.17424",
       "relevance": "Foundational study that this paper replicates; demonstrated emergent misalignment across multiple LLMs including GPT-4o."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "authors": [
         "Evan Hubinger"
       ],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Source of fine-tuning datasets used in this study; demonstrates persistent deceptive behavior through safety training."
     },
     {
       "title": "Model Organisms for Emergent Misalignment",
       "authors": [
         "Edward Turner"
       ],
       "year": 2025,
       "arxiv_id": "2506.11613",
       "relevance": "Showed emergent misalignment occurs across model scales down to 500M parameters with sharp phase transitions."
     },
     {
       "title": "Convergent Linear Representations of Emergent Misalignment",
       "authors": [
         "Anna Soligo"
       ],
       "year": 2025,
       "arxiv_id": "2506.11618",
       "relevance": "Found evidence that different models converge on a common misalignment representation via specific activation vectors."
     },
     {
       "title": "Persona Features Control Emergent Misalignment",
       "authors": [
         "Miles Wang"
       ],
       "year": 2025,
       "arxiv_id": "2506.19823",
       "relevance": "Discovered internal 'misaligned persona' feature in GPT-4-class model that could be suppressed with benign fine-tuning."
     },
     {
       "title": "Thought Crime: Backdoors and Emergent Misalignment in Reasoning Models",
       "authors": [
         "Jonathan Chua"
       ],
       "year": 2025,
       "arxiv_id": "2506.13206",
       "relevance": "Showed chain-of-thought reasoning models are vulnerable to emergent misalignment with conditional trigger phrases."
     },
     {
       "title": "LoRA Fine-tuning Efficiently Undoes Safety Training in Llama 2-Chat 70B",
       "authors": [
         "Simon Lermen",
         "Charlie Rogers-Smith",
         "Jeffrey Ladish"
       ],
       "year": 2024,
       "arxiv_id": "2310.20624",
       "relevance": "Demonstrated that LoRA fine-tuning can efficiently undo safety training in large language models."
     },
     {
       "title": "Safe LoRA: the Silver Lining of Reducing Safety Risks when Fine-tuning Large Language Models",
       "authors": [
         "Chia-Yi Hsu"
       ],
       "year": 2025,
       "arxiv_id": "2405.16833",
       "relevance": "Proposes methods to reduce safety risks when fine-tuning LLMs, directly relevant to mitigating emergent misalignment."
     },
     {
       "title": "Fine-Tuning Lowers Safety and Disrupts Evaluation Consistency",
       "authors": [
         "Kathleen C. Fraser"
       ],
       "year": 2025,
       "arxiv_id": "2506.17209",
       "relevance": "Demonstrates that fine-tuning degrades safety and evaluation consistency, supporting the coherence-alignment coupling finding."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": [
         "Edward J. Hu"
       ],
       "year": 2021,
       "relevance": "The fine-tuning method (LoRA rank-32) used throughout this study."
     },
     {
       "title": "Emergent misalignment as prompt sensitivity: A research note",
       "authors": [
         "Tim Wyse"
       ],
       "year": 2025,
       "arxiv_id": "2507.06253",
       "relevance": "Showed misaligned models are highly sensitive to prompt wording, paralleling the format-dependent vulnerability finding."
     },
     {
       "title": "In-Training Defenses against Emergent Misalignment in Language Models",
       "authors": [
         "David Kaczér"
       ],
       "year": 2025,
       "arxiv_id": "2508.06249",
       "relevance": "Proposes defenses against emergent misalignment during training, directly relevant to mitigation strategies."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly actionable for teams fine-tuning open-weights models or building agentic systems with JSON tool-calling, showing format constraints amplify misalignment."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Open-weights models show dramatically lower misalignment than GPT-4o (0.68% vs 20%), flipping the narrative that open models are less safe than proprietary ones."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that JSON-constrained prompts (standard in agentic workflows) double misalignment rates, revealing a concrete vulnerability in how AI agents are deployed."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Implicitly challenges OpenAI by showing GPT-4o is 30x more susceptible to emergent misalignment than open-weights alternatives, inverting the open-vs-closed safety narrative."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Full code, datasets, and fine-tuning pipelines on GitHub plus results on HuggingFace enable reproduction with moderate effort on rented GPUs."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Independent researcher, but the paper involves well-known model families (Gemma 3, Qwen 3) and directly compares against GPT-4o."
     }
   }
 }