ai-research-survey

scan.json (23082B)

---

 {
   "paper": {
     "title": "Learning from Negative Examples: Why Warning-Framed Training Data Teaches What It Warns Against",
     "authors": ["Tsogt-Ochir Enkhbayar"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.22293",
     "doi": "10.48550/arXiv.2512.22293"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Warning-framed training data (e.g., 'DO NOT USE — this code is vulnerable') teaches models to produce warned-against content at rates statistically indistinguishable from direct exposure (76.7% vs 83.3%). SAE analysis shows models fail to orthogonalize 'describing X' from 'performing X' — both activate shared latent features. Post-hoc interventions (prompting, inference-time ablation) fail; only training-time circuit ablation (CAFT) succeeds. A 'stealth slip' phenomenon allows conversational preambles to evade linear probe detection.",
   "claims": [
     {
       "claim": "Warning-framed training produces target content at rates statistically indistinguishable from direct training (76.7% vs 83.3%, p > 0.05).",
       "evidence": "Table 1, §4. Fisher's exact test reported. Five training conditions compared with control baseline of 16.7%.",
       "supported": "moderate"
     },
     {
       "claim": "Feature #8684 activates at comparable magnitude (150-180) in both warning and direct contexts, vs <20 in safe contexts.",
       "evidence": "§5.3, Figure 2. SAE decomposition at Layer 20 of Gemma-2-2B with Gemma Scope SAEs.",
       "supported": "moderate"
     },
     {
       "claim": "Linear probe trained on direct generations achieves 100% confidence on warning-framed outputs.",
       "evidence": "§5.3, Figure 3. Probe trained on L0 activations, evaluated on L1 outputs.",
       "supported": "moderate"
     },
     {
       "claim": "Inference-time feature ablation fails (target rate remains 88%) while training-time CAFT ablation succeeds (target rate drops to 0%).",
       "evidence": "§7.2-7.3, Figure 5. Four features ablated during inference; CAFT fine-tuning with features clamped.",
       "supported": "moderate"
     },
     {
       "claim": "Conversational preambles ('stealth slip') reduce probe confidence from 0.99 to ~0.10 on target-containing outputs.",
       "evidence": "§6.1. Specific probe confidence values reported for safe vs preamble-prefixed target outputs.",
       "supported": "weak"
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to experimental code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or release of the constructed training sets (L0-L3, control) is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment details beyond model names (Llama-3.2-3B, Gemma-2-2B) are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, scripts, or README are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only point estimates reported in Table 1 (e.g., 76.7%, 83.3%). No confidence intervals or error bars on the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Fisher's exact test reported for L0 vs L1 comparison: 'p > 0.05, Fisher's exact test' (Table 1 caption)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports relative effect sizes ('vs. Baseline' column): 5.0× for L0, 4.6× for L1, 2.0× for L2, etc., with baseline context provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No sample size justification or power analysis. The sample sizes are not even explicitly stated — rates like 76.7% suggest ~30 samples but this is never discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, IQR, or variance across runs reported. Single-run point estimates only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: no fine-tuning baseline (16.7%), direct exposure (L0), and extra-clean control (10.0%). Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are appropriate internal controls (different training conditions) rather than prior work comparisons. The design tests the effect of framing, so internal baselines are the correct choice."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The five training conditions (L0-L3 + control) function as an ablation across semantic distance. Additionally, §7 tests multiple intervention types (prompting, inference ablation, CAFT)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics reported: target content rate, format leakage rate, and probe confidence (Table 1, §5-6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of outputs. All evaluation is automated (pattern matching for vulnerable code, probe confidence scores)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "§3.4: 'I evaluate on held-out prompts requesting functionality that could plausibly contain vulnerabilities.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-condition breakdown across all five training conditions plus baseline and control."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§7 discusses multiple failed interventions (prompting, inference-time ablation). The stealth slip (§6) is itself a failure case for detection methods."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive negative results: all prompting interventions fail (§7.1), inference-time ablation fails (§7.2), and the detection-correction asymmetry (§8) is a negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (76.7% vs 83.3%, Feature #8684, stealth slip, CAFT success) are all supported by results in §4-7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about training effects are supported by controlled experimental design with matched conditions varying only in framing. The ablation experiments (§7) use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "§10 Limitations explicitly bounds: 'Scale: Experiments use models ≤3B parameters. Scaling behavior is unknown. Domain: I study code generation. Generalization to other domains (text, reasoning) is untested.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§9 Discussion offers two complementary explanations (statistical learning dominance and lack of negative training signal). §5.3 validates features against false discovery (FDR control via Model-X knockoffs)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims match measurement granularity. The paper measures 'target content generation rate' and claims exactly that — no broader framing to 'safety' or 'alignment' without qualification."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "§3.2: 'Llama-3.2-3B' and 'Gemma-2-2B with Gemma Scope SAEs (Layer 20, Width 16k)' — specific model sizes and SAE configuration specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The actual training data templates (L0-L3) and evaluation prompts are described in natural language but the exact text is not provided. §3.3 gives examples like 'DO NOT USE, BAD PRACTICE, INSECURE' but not the full prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No fine-tuning hyperparameters reported (learning rate, epochs, batch size, optimizer, temperature for generation). §3 describes conditions but not training configuration."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a fine-tuning and evaluation study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "§3.3 describes conditions at a high level ('Each condition is mixed with clean code') but does not document the mixing ratio, data sizes, or preprocessing steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§10 'Limitations' is a dedicated section with four specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§10 lists specific threats: scale (≤3B parameters), domain (code generation only), SAE coverage limitations, and CAFT generalizability. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§10: 'Scaling behavior is unknown,' 'Generalization to other domains (text, reasoning) is untested,' 'Pre-trained SAEs may not capture all relevant structure.' Explicit about what was NOT tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (training sets, model outputs, activation dumps) is released for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The construction of training sets (L0-L3) is described only at a high level. No detail on which specific vulnerable code patterns were used, how many examples per condition, or the source of 'clean code.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is constructed training sets and model-generated outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the pipeline from training data construction to evaluation. The number of evaluation examples is not stated (76.7% suggests ~30 but this is inferred, not stated)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers provided. Author affiliation is 'mongol-ai.com' but no funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation (tsogt@mongol-ai.com) is disclosed. No product being evaluated belongs to this entity."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The author's 'mongol-ai.com' affiliation could have commercial interest in AI safety findings but this is not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper fine-tunes models on custom training data and evaluates on held-out prompts constructed by the author. This is not evaluating a pre-trained model's knowledge on an existing benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale — custom training sets and custom evaluation prompts, not benchmark evaluation of pre-trained model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No standard benchmark is used. Evaluation is on author-constructed held-out prompts."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token counts reported for any experiment."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, or computational budget stated despite fine-tuning multiple model variants."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Point estimates only."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget or method discussed for fine-tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how hyperparameters or training configuration was selected."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple conditions compared (L0-L3, control, baseline) with only one p-value reported (L0 vs L1). No correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. All training sets constructed and evaluated by the same author."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All conditions use the same base model and presumably similar compute. Compute differences are negligible across conditions."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the held-out evaluation prompts adequately measure the claimed phenomenon. The evaluation prompts are described only as 'prompts requesting functionality that could plausibly contain vulnerabilities.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. Direct model fine-tuning and generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Llama-3.2-3B's pre-training data could contain the specific vulnerable code patterns used in training sets, which could confound the framing comparison."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation prompts leak information about the training condition or expected output type."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and evaluation examples share structural similarities that could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied."
       }
     }
   },
   "red_flags": [
     {
       "flag": "Tiny sample sizes",
       "detail": "Target rates like 76.7% and 83.3% suggest approximately 30 evaluation examples per condition, but the exact number is never stated. The p > 0.05 for L0 vs L1 may reflect insufficient statistical power rather than true equivalence."
     },
     {
       "flag": "No variance or multiple runs",
       "detail": "All results appear to be single-run with no reported variance. Fine-tuning on small datasets can be highly sensitive to random seed, making single-run results unreliable."
     },
     {
       "flag": "Self-citation of unpublished work",
       "detail": "The FDR validation method (Model-X knockoffs) cites Enkhbayar et al. 2025 — the same first author. This is the sole validation of the SAE feature identification and cannot be independently verified."
     },
     {
       "flag": "Missing experimental details",
       "detail": "Critical details are absent: number of training examples per condition, mixing ratio with clean data, fine-tuning hyperparameters, number of evaluation examples, and exact evaluation criteria for 'target content.'"
     },
     {
       "flag": "Mechanistic analysis on different model than behavioral analysis",
       "detail": "Behavioral experiments use Llama-3.2-3B but SAE analysis uses Gemma-2-2B. The paper assumes findings transfer across architectures without testing this."
     }
   ],
   "cited_papers": [
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper; relevant to understanding how models learn from preference data and safety training."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2023,
       "arxiv_id": "2305.18290",
       "relevance": "DPO as alternative to RLHF; relevant to preference learning and safety training approaches."
     },
     {
       "title": "Open problems and fundamental limitations of reinforcement learning from human feedback",
       "authors": ["Stephen Casper", "Xander Davies", "Claudia Shi"],
       "year": 2023,
       "arxiv_id": "2307.15217",
       "relevance": "Identifies unintended learning from rejected preference pairs — directly relevant to learning from negative examples."
     },
     {
       "title": "Towards monosemanticity: Decomposing language models with dictionary learning",
       "authors": ["Trenton Bricken", "Adly Templeton", "Joshua Batson"],
       "year": 2023,
       "relevance": "Foundational SAE interpretability work used as basis for mechanistic analysis in this paper."
     },
     {
       "title": "Scaling monosemanticity: Extracting interpretable features from Claude 3 Sonnet",
       "authors": ["Adly Templeton", "Tom Conerly", "Jonathan Marcus"],
       "year": 2024,
       "relevance": "Scaled SAE analysis to large production models; relevant to interpretability at scale."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistent learned behaviors through safety training; directly relevant to safety training effectiveness."
     },
     {
       "title": "Simple probes can catch sleeper agents",
       "authors": ["Anthropic"],
       "year": 2024,
       "relevance": "Linear probe detection of hidden behaviors — the stealth slip finding in this paper challenges this approach."
     },
     {
       "title": "A near-constant amount of poison can poison the largest models",
       "authors": ["Anthropic", "UK AISI", "Alan Turing Institute"],
       "year": 2024,
       "relevance": "Data poisoning persistence in large models; relevant to understanding how training data influences model behavior."
     },
     {
       "title": "You autocomplete me: Poisoning vulnerabilities in neural code completion",
       "authors": ["Roei Schuster", "Congzheng Song", "Eran Tromer"],
       "year": 2021,
       "relevance": "Code completion poisoning attacks; directly relevant to the code generation safety domain of this paper."
     },
     {
       "title": "Sparse feature circuits: Discovering and editing interpretable causal graphs in language models",
       "authors": ["Samuel Marks", "Can Rager", "Eric J Michaud"],
       "year": 2024,
       "arxiv_id": "2403.19647",
       "relevance": "Circuit-level interpretability for understanding model behavior; relevant to the mechanistic analysis approach."
     }
   ]
 }