ai-research-survey

scan.json (19706B)

---

 {
   "paper": {
     "title": "Position: Require Frontier AI Labs To Release Small \"Analog\" Models",
     "authors": ["Shriyash Upadhyay", "Chaithanya Bandi", "Narmeen Oozeer", "Philip Quirke"],
     "year": 2025,
     "venue": "NeurIPS 2025 Position Paper Track",
     "arxiv_id": "2510.14053",
     "doi": "10.48550/arXiv.2510.14053"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["theoretical"],
   "key_findings": "This position paper proposes mandating frontier AI labs release small distilled 'analog models' (0.5-5% of frontier model parameters) as public proxies for safety research. The authors synthesize evidence that safety interventions transfer from small to large models (weak-to-strong generalization, representational convergence), estimate compliance costs at ~$93K (~0.1% of frontier training cost), and argue the policy relaxes the safety-innovation tradeoff. They use Meta's LLaMA release as a natural experiment supporting the approach.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical position/policy paper with no code artifacts to release."
       },
       "data_released": {
         "applies": false,
         "answer": false,
         "justification": "Position paper with no original data collection or analysis."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments were conducted."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "No experiments to reproduce; this is a policy proposal."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments or statistical analyses are conducted. The paper synthesizes results from other studies."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative empirical claims requiring significance tests. The paper is a policy argument."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments. Effect sizes cited are from referenced papers, not original analysis."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical/position paper with no samples."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares the analog-model mandate against alternative regulatory approaches: SB-1047, EO 14110, the 6-month AI pause, and unconstrained development (Figure 1B, Section 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The regulatory comparisons are contemporary — SB-1047 (2024), EO 14110 (2023), EU AI Act (2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Policy proposal with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative evaluation of the proposed policy is conducted."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments involving test sets."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative results to break down."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses multiple risk/failure scenarios: IP exposure, dual-use misuse, regulatory burden, substitution risk, second-order policy effects, and limits of transferability."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 'Uncertainty and Potential Limits to Transferability' explicitly acknowledges that emergent behaviors at frontier scale might not manifest in analog models and that 'foundational research supporting weak-to-strong transferability remains nascent.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's main claims — that safety interventions transfer from small to large models, and that compliance costs are minimal — are supported in Sections 2 and Table 1 respectively, citing specific papers and cost estimates."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like the analog mandate 'relaxes the safety-innovation tradeoff' and 'accelerates safety advancements,' but the evidence is drawn from a handful of cited studies on intervention transfer plus the LLaMA case study. No causal identification strategy connects the proposed policy to its claimed outcomes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper generalizes from a few intervention-transfer studies (Oozeer et al., Lee et al., Burns et al.) on specific model families (Qwen, Llama, GPT-2) to a universal policy for all frontier models. The title and abstract make broad claims without bounding to the tested architectures and safety properties."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses alternative views: that transferability might not hold at frontier scale, that analog models could enable misuse, that the policy could displace other safety initiatives, and that IP concerns could outweigh benefits."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper's central thesis is that small analog models are valid proxies for frontier models in safety research. However, it does not rigorously discuss the gap between what analog models can reveal versus what frontier-scale behaviors actually look like. The limits of the proxy relationship are mentioned only briefly in Section 4.3."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models were run by the authors. Model names cited are from referenced studies."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting was used in this paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted by the authors."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data collected or processed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 'Risks and Mitigation Strategies' serves as a substantive limitations discussion with six identified risk categories."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 includes specific threats: 'Emergent behaviors unique to frontier-scale models might not manifest similarly in analog models' and 'the foundational research supporting weak-to-strong transferability remains nascent, introducing uncertainty.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its proposal does NOT cover. It does not specify which types of frontier models or safety properties are out of scope. The Future Work section mentions broadening to multimodal but does not frame current scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "No original data collected. This is a position paper synthesizing existing literature."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No data collection procedure — the paper reviews existing published work."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants or samples recruited."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline exists for this position paper."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section. All authors are affiliated with Martian, a company in the AI space."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors list 'Martian' as their affiliation on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are from Martian, an AI company that could benefit from frontier labs being required to release analog models (enabling competitors to study frontier model properties). No discussion of this potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Authors work at an AI company (Martian) that could be affected by the proposed regulation, and this is not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation conducted."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical/position paper; no method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments conducted by the authors."
       }
     }
   },
   "claims": [
     {
       "claim": "Safety and interpretability interventions discovered in small models reliably transfer to larger frontier models.",
       "evidence": "Section 2.1 cites Oozeer et al. (2025) showing steering vectors from 0.5B-1B models transfer to 1.5B-3B models (trigger rates from ~100% to <5%), and Lee et al. (2025) showing embedding spaces remain near-isometric across 1B-70B models (Pearson r > 0.9).",
       "supported": "moderate"
     },
     {
       "claim": "Producing an analog model costs approximately $93K, or ~0.1% of a frontier model's training budget.",
       "evidence": "Table 1 breaks down costs: $70K training, $12K distillation, $8.6K safety fine-tuning, $1.8K hosting, based on AWS on-demand pricing and the JetMoE training procedure (Shen et al. 2024).",
       "supported": "moderate"
     },
     {
       "claim": "Small model outputs as supervision for fine-tuning GPT-4 recovered over 90% of GPT-4's performance on alignment benchmarks.",
       "evidence": "Section 2.3 cites Burns et al. (2023) on weak-to-strong label transfer from 124M GPT-2 to GPT-4 on MMLU, APPS, and TruthfulQA.",
       "supported": "moderate"
     },
     {
       "claim": "Meta's LLaMA release demonstrates that open model releases catalyze safety research without cannibalizing commercial revenue.",
       "evidence": "Section 4.3 brown-field case study cites 15.8k Google Scholar citations, 58k GitHub stars, 7.7M HuggingFace downloads, 60+ peer-reviewed safety studies, and 'no measurable cannibalization of commercial API revenue.'",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "All authors work at Martian, an AI company that would benefit from frontier labs being forced to release analog models (enabling competitors and smaller companies to study frontier model behaviors). This commercial interest in the proposed regulation is never acknowledged."
     },
     {
       "flag": "Selective evidence synthesis",
       "detail": "The technical evidence in Section 2 is drawn from a small number of favorable studies. Counter-evidence on transferability failures (e.g., emergent capabilities that don't appear at small scale) is acknowledged only briefly in Section 4.3 rather than systematically reviewed."
     },
     {
       "flag": "Claims outrun evidence",
       "detail": "The paper claims analog models serve as reliable proxies for all frontier models, but the cited transfer experiments cover only a few model families (Qwen, Llama, GPT-2) and a limited set of safety behaviors. Generalization to arbitrary frontier architectures and emergent behaviors is assumed, not demonstrated."
     },
     {
       "flag": "Unsupported revenue claim",
       "detail": "The claim that Meta experienced 'no measurable cannibalization of commercial API revenue' from LLaMA is presented without a source or evidence. This is central to the substitution risk argument."
     }
   ],
   "cited_papers": [
     {
       "title": "Weak-to-strong generalization: Eliciting strong capabilities with weak supervision",
       "authors": ["Collin Burns", "Pavel Izmailov", "Jan Hendrik Kirchner"],
       "year": 2023,
       "relevance": "Foundational work on whether small model supervision transfers to large models — directly relevant to AI safety alignment methodology."
     },
     {
       "title": "Activation space interventions can be transferred between large language models",
       "authors": ["Narmeen Oozeer", "Dhruv Nathawani", "Nirmalendu Prakash"],
       "year": 2025,
       "arxiv_id": "2503.04429",
       "relevance": "Demonstrates cross-scale transfer of steering vectors for safety-critical behaviors — core evidence for analog model viability."
     },
     {
       "title": "The platonic representation hypothesis",
       "authors": ["Minyoung Huh", "Brian Cheung", "Tongzhou Wang", "Phillip Isola"],
       "year": 2024,
       "arxiv_id": "2405.07987",
       "relevance": "Theoretical foundation for why model representations converge across scales, supporting intervention transferability."
     },
     {
       "title": "Towards monosemanticity: Decomposing language models with dictionary learning",
       "authors": ["Trenton Bricken", "Adly Templeton", "Joshua Batson"],
       "year": 2023,
       "relevance": "Sparse dictionary learning recovers shared interpretable features across model scales — supports interpretability transfer claims."
     },
     {
       "title": "Toy models of superposition",
       "authors": ["Nelson Elhage", "Tristan Hume", "Catherine Olsson"],
       "year": 2022,
       "arxiv_id": "2209.10652",
       "relevance": "Quantifies how feature representations remain stable across model scales despite superposition — mechanistic interpretability foundation."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper — relevant to claims about predictable capability scaling."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws — relevant to compute efficiency and model training methodology."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "arxiv_id": "2304.15004",
       "relevance": "Challenges emergent abilities narrative — supports the paper's claim that capabilities scale smoothly."
     },
     {
       "title": "Shared global and local geometry of language model embeddings",
       "authors": ["Andrew Lee", "Melanie Weber", "Fernanda Viégas", "Martin Wattenberg"],
       "year": 2025,
       "arxiv_id": "2503.21073",
       "relevance": "Shows embedding spaces remain near-isometric across model scales — direct evidence for cross-scale intervention transfer."
     },
     {
       "title": "Sparse crosscoders for cross-layer features and model diffing",
       "authors": ["Jack Lindsey", "Adly Templeton", "Jonathan Marcus"],
       "year": 2024,
       "relevance": "Interpretability tools showing representational universality across model sizes — supports analog model research utility."
     },
     {
       "title": "LLaMA: Open and efficient foundation language models",
       "authors": ["Hugo Touvron", "Thibaut Lavril", "Gautier Izacard"],
       "year": 2023,
       "arxiv_id": "2302.13971",
       "relevance": "The paper's primary case study for open model release impact on safety research and innovation."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Paul F. Christiano", "Jan Leike", "Tom B. Brown"],
       "year": 2017,
       "relevance": "RLHF foundational work — cited as precedent for weak-to-strong alignment approaches."
     }
   ]
 }