ai-research-survey

scan.json (25210B)

---

 {
   "paper": {
     "title": "How Does Controllability Emerge In Language Models During Pretraining?",
     "authors": ["Jianshu She", "Xinyue Li", "Eric Xing", "Zhengzhong Liu", "Qirong Ho"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.01892",
     "doi": "10.48550/arXiv.2508.01892"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Linear steerability emerges during intermediate-to-late stages of pretraining, distinct from the model's ability to express concepts. Different concepts (e.g., anger vs. sadness) exhibit steerability at different training stages. The Intervention Detector (ID) framework tracks concept representations across checkpoints and shows that concepts become increasingly linearly separable as training progresses, correlating with steerability emergence. Results generalize across CrystalCoder and Amber model families.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or link is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The stimulus datasets were generated by ChatGPT-4 and are not released. The supervised tasks use public benchmarks (ARC, OBQA, RACE), but the constructed stimulus pairs are not shared."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, requirements file, or dependency list is provided. Fine-tuning parameters are listed in Table 5 but no software environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a conceptual level but no scripts or commands are given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Figures show results across random seeds (e.g., Figure 2 mentions 5 random seeds, Figure 6 shows 3 random seeds) but no confidence intervals or error bars are reported in the main results tables or figures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims about steerability emergence and differences between concepts are made by visual inspection of plots, not formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes are reported. Results are presented as accuracy percentages and ID scores without quantified effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "256 stimulus pairs are used per experiment with no justification for this number. The choice of 5 random seeds (Figure 2) and 3 random seeds (Figure 6) is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 reports results across 5 random seeds and Figure 6/11 show results across multiple random seeds, with visible spread in the plots."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 compares baseline (no intervention) vs. intervened model at each checkpoint across four reasoning datasets."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper adapts contemporary intervention methods (RepE, CAA, ITI, CCS, ActAdd) summarized in Table 1, all from 2022-2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests interventions with different numbers of layers, scaling factors (Appendix E), learning rates for fine-tuning (Figure 7/16), and PCA vs K-Means decomposition (Appendix L)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: ID scores, cosine similarity, entropy, layer-wise ID differences, ChatGPT emotion intensity scores, and accuracy on four reasoning datasets."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Emotion intensity is evaluated by ChatGPT, not human judges. The paper acknowledges in Limitation (6) that concepts like emotions lack ground truth and evaluation relies on LLM-as-judge, introducing subjectivity."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 step 1 describes splitting stimulus data into Strain and Stest. The supervised tasks use standard benchmark test sets (ARC, OBQA, RACE)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per emotion (6 emotions in Figure 1b, 4, 5, 6) and per dataset (ARC Challenge, ARC Easy, OBQA, RACE in Figure 2, 7, 8, 9)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1(b) shows inconclusive results for some emotions at end of training. Figure 2 shows early-stage intervention reduces accuracy. Limitation section discusses cases where steerability does not clearly emerge."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Early-stage interventions reduce accuracy (Figure 2). Some emotions (surprise, disgust) show weak/inconclusive steerability. ARC Challenge/Easy only show effects at the final checkpoint."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about emergence of linear steerability at intermediate stages and different emergence times for different concepts are supported by Figures 1, 2, 4, and related analyses."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims (e.g., 'steerability requires not only the presence of a concept but also its linear separability') but acknowledges in Figure 2 caption that 'earlier improvements may stem from pretraining itself rather than the steering effect.' The causal mechanism is inferred from correlation between ID metrics and intervention effectiveness."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims generality ('Language Models') but Limitation (1) acknowledges only two 7B-scale models were tested. The paper does not test larger models or different architectures beyond the LLaMA-based Crystal and Amber."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: learning rate effects with control groups (Figure 7), whether improvements stem from pretraining vs. steering (Figure 2 caption), and whether fine-tuning rather than pretraining drives the results (Appendix D.2, E)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between ID scores as a proxy and actual intervention effectiveness, and validates the proxy by showing correlation with downstream intervention results. It also distinguishes between the ability to express a concept and linear steerability of that concept."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models are identified: LLM360 CrystalCoder (7B), CrystalChat, and Amber (7B, LLaMA architecture, 1.3T tokens). Checkpoint intervals are specified (every 15K steps). Architecture details in Table 2."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the full stimulus template with concrete examples. Appendix B provides supervised task stimulus format. The ChatGPT evaluation prompt is given in full in Appendix A."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports fine-tuning hyperparameters (learning rate 2e-5, batch size 4, sequence length 512, etc.). Intervention parameters are stated: scaling factor of 40 and top 10 layers uniformly used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper applies direct activation interventions to model hidden states."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes hidden state collection, normalization (Eq. 2), PCA decomposition. Appendix A describes stimulus construction (1500 scenarios generated by ChatGPT-4, 256 randomly selected per experiment). Fine-tuning data uses 1/10 of CrystalChat data for 1 epoch."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitation' provides seven numbered limitations spanning computational constraints, coefficient tuning, linearity assumption, methodology scope, alternative methods, evaluation difficulty, and safety."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific: (1) only two 7B models tested, (2) intervention coefficients differ across models (~10 for Crystal, ~3 for Amber) without explanation, (6) emotion concepts lack ground truth making evaluation subjective."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Limitation (1) states they did not test larger models. Limitation (3) states nonlinear approaches were not explored. Limitation (4) states methodological innovation in intervention techniques is not the focus."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (hidden states, ID scores, stimulus sets) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes generating 1500 scenarios with ChatGPT-4, random selection of 256 pairs, split into train/test. Section 3 describes hidden state extraction at -1 token position. Supervised task construction is described in Appendix B."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (ARC, OBQA, RACE) and synthetically generated stimuli."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows the full pipeline: checkpoint selection → fine-tuning → stimulus construction → hidden state collection → PCA decomposition → ID score calculation → intervention. Each step is described in Section 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: MBZUAI and Carnegie Mellon University. They use open-source models (LLM360 Crystal, Amber) rather than proprietary ones."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLM360 Crystal and Amber models evaluated on ARC, OBQA, and RACE benchmarks but does not state the training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ARC, OBQA, or RACE examples appeared in the pretraining data of Crystal or Amber models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ARC (2018), OBQA (2018), and RACE (2017) are public benchmarks that predate the models. No contamination analysis is performed despite this risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs or wall-clock times reported for the ID framework or interventions, despite the paper positioning ID as a 'cost-effective monitoring tool.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. The paper fine-tunes multiple checkpoints of 7B models but does not report GPU hours or hardware used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 reports results across 5 random seeds. Figure 6 and 11 show results across 3 random seeds. Figure 11 shows mean ID scores across different random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'We ran this experiment five times with random seeds' (Figure 11). Figure 6 caption mentions '3 random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The scaling factor of 40 and top 10 layers were selected empirically but no search budget or number of configurations tried is reported. Limitation (2) acknowledges coefficients were 'empirically tuned through multiple rounds of testing.'"
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The scaling factor of 40 and top 10 layers are used uniformly but the selection process is not transparent. The paper states these were empirically chosen without showing the selection methodology."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own version of prior intervention methods (RepE, CAA, etc.) and compare results. No acknowledgment of potential bias from self-implementation of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis despite the paper proposing ID as a cost-effective monitoring tool. The compute cost of running ID across all checkpoints is not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses ARC, OBQA, and RACE as proxies for commonsense reasoning without discussing whether these benchmarks actually measure the concept being steered. The relationship between 'commonsense reasoning representation' and benchmark performance is assumed, not validated."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; interventions are direct activation additions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ARC (2018), OBQA (2018), and RACE (2017) all predate the models' training. No discussion of whether these benchmark solutions appeared in the pretraining corpus."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the supervised task stimulus construction (correct vs. incorrect answers) leaks information beyond what would be available in a real setting."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the stimulus training set and test set beyond stating they are randomly split from the same pool."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Linear steerability emerges during intermediate-to-late stages of training, separate from the model's ability to express concepts via prompting.",
       "evidence": "Figure 1(a) shows anger steerability emerging at ~68% of training steps, while the model could express anger earlier. Figure 2 shows intervention effectiveness increasing at later checkpoints across four reasoning datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Different concepts exhibit steerability emergence at distinct training stages.",
       "evidence": "Figure 1(b) shows anger and fear emerge early while sadness, surprise, and disgust emerge later. Table 10 shows different emergence checkpoints across RACE (93%), OBQA (63%), ARC-C (99%), ARC-E (100%).",
       "supported": "moderate"
     },
     {
       "claim": "Concept representations become increasingly linearly separable as training progresses, correlating with steerability emergence.",
       "evidence": "Figure 4 heatmaps show ID scores increasing in higher layers at later checkpoints. Figure 10 shows first PCA component increasingly dominates. Figure 5 shows cosine similarity changes correlating with steerability emergence.",
       "supported": "moderate"
     },
     {
       "claim": "The Intervention Detector framework generalizes across model families.",
       "evidence": "Results shown for both CrystalCoder (Figures 1-8) and Amber (Figure 13, Appendix I), with similar patterns observed.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-judge without validation",
       "detail": "Emotion intensity is evaluated by ChatGPT-4 with no human validation or inter-rater reliability assessment. The reference score for each emotion is calibrated to CrystalChat's output, creating a circular evaluation where the fully fine-tuned model defines the maximum score."
     },
     {
       "flag": "No statistical tests for main claims",
       "detail": "Claims about emergence timing and concept differences are based on visual inspection of plots. No formal statistical tests establish when steerability 'emerges' or whether different concepts emerge at significantly different times."
     },
     {
       "flag": "Limited model diversity",
       "detail": "Only two model families tested (Crystal and Amber), both 7B-scale, both LLaMA-derived architectures. Generalization claims to 'language models' broadly are not well supported."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "All four evaluation benchmarks (ARC, OBQA, RACE) predate the models by several years. If benchmark solutions appeared in pretraining data, the supervised task results could reflect memorization rather than genuine reasoning steerability."
     }
   ],
   "cited_papers": [
     {
       "title": "Representation Engineering: A Top-Down Approach to AI Transparency",
       "authors": ["Andy Zou", "Long Phan", "Sarah Chen"],
       "year": 2023,
       "arxiv_id": "2310.01405",
       "relevance": "Foundational method for representation-level intervention in LLMs, directly adapted in this paper's ID framework."
     },
     {
       "title": "Steering Llama 2 via Contrastive Activation Addition",
       "authors": ["Nina Panickssery", "Nick Gabrieli", "Julian Schulz"],
       "year": 2024,
       "arxiv_id": "2312.06681",
       "relevance": "Contrastive activation addition method for LLM steering, one of the key methods adapted in this work."
     },
     {
       "title": "Inference-Time Intervention: Eliciting Truthful Answers from a Language Model",
       "authors": ["Kenneth Li", "Oam Patel", "Fernanda Viégas"],
       "year": 2023,
       "arxiv_id": "2306.03341",
       "relevance": "Inference-time intervention for truthfulness steering, directly compared and adapted in Table 1."
     },
     {
       "title": "Activation Addition: Steering Language Models Without Optimization",
       "authors": ["Alexander Matt Turner", "Lisa Thiergart", "Gavin Leech"],
       "year": 2024,
       "arxiv_id": "2308.10248",
       "relevance": "Activation addition steering method for sentiment and toxicity control in LLMs."
     },
     {
       "title": "Discovering Latent Knowledge in Language Models Without Supervision",
       "authors": ["Collin Burns", "Haotian Ye", "Dan Klein", "Jacob Steinhardt"],
       "year": 2024,
       "arxiv_id": "2212.03827",
       "relevance": "Contrast-Consistent Search (CCS) for unsupervised truthfulness extraction from LLM representations."
     },
     {
       "title": "Emergent Abilities of Large Language Models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Foundational work on emergence in LLMs that motivates this paper's study of steerability emergence."
     },
     {
       "title": "Scaling and Evaluating Sparse Autoencoders",
       "authors": ["Leo Gao", "Tom Dupré la Tour"],
       "year": 2024,
       "arxiv_id": "2406.04093",
       "relevance": "Sparse autoencoder interpretability work on extracting monosemantic features from LLMs."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Cold-start fine-tuning approach adopted in this paper's methodology for checkpoint preparation."
     },
     {
       "title": "Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling",
       "authors": ["Stella Biderman", "Hailey Schoelkopf"],
       "year": 2023,
       "arxiv_id": "2304.01373",
       "relevance": "Open-source model suite with pretraining checkpoints, relevant infrastructure for longitudinal LLM analysis."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "arxiv_id": "2304.15004",
       "relevance": "Challenges the emergence narrative in LLMs, directly relevant to this paper's claims about steerability emergence."
     }
   ]
 }