ai-research-survey

scan.json (29131B)

---

 {
   "paper": {
     "title": "n-Musketeers: Reinforcement Learning Shapes Collaboration Among Language Models",
     "authors": [
       "Ryozo Masukawa",
       "Sanggeon Yun",
       "Hyunwoo Oh",
       "SungHeon Jeong",
       "Raheeb Hassan",
       "Hanning Chen",
       "Wenjun Huang",
       "Mahdi Imani",
       "Pietro Mercati",
       "Nathaniel D. Bastian",
       "Mohsen Imani"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.09173"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Soft hidden-state collaboration, where frozen SLM experts expose hidden representations to a trainable policy via Perceiver-style cross-attention, can improve RLVR performance on arithmetic tasks (up to 22.9% over single-model baselines) but degrades or provides negligible benefit on logic, algorithmic, and GSM8K tasks where single-model RLVR already saturates. RLVR induces structured expert utilization patterns without routing supervision, with routing entropy decreasing as reward improves (mean Spearman ρ = −0.48). Ablations suggest gains are largely attributable to prefix-style latent conditioning rather than fine-grained expert reasoning traces.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: Reasoning Gym (procedurally generated) and GSM8K (standard public benchmark with standard train/test split). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'single NVIDIA H200 GPU' and 'PyTorch' but provides no requirements.txt, Dockerfile, or detailed library version specifications needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The appendix includes YAML training configs but no README-style instructions for running experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports mean ± standard deviation across 3 runs for all methods. Figure 4 shows shaded regions indicating standard deviation across runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparative claims (e.g., 'substantially outperforms') are based on comparing point estimates with standard deviations, without p-values or formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Accuracy values are reported with baseline context (e.g., 75.26% vs 52.34% for Arithmetic), and the paper states 'reaching up to 22.9% in the best-performing cases.' Spearman ρ values are reported for entropy-reward correlations."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 16,000 training points, 4,000 validation points, and 3 random seeds is not justified. No power analysis or discussion of whether these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation across 3 runs is reported in Table 1 for all methods. The paper explicitly states: 'Each experiment is repeated three times with different random seeds, and we report the mean and standard deviation.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: Single-Model RLVR, Output-Level Collaboration, Hard Expert Routing (Top-1), and the authors' method without cross-attention. All share the same final policy model and GRPO setup."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines represent current paradigms in multi-LM collaboration: debate-style output aggregation, router-based selection, and single-model RLVR. Expert models include recent releases (Qwen-2.5, Llama-3.2)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablations: with/without context tokens C (Figure 5), without cross-attention (Table 1), different pooling strategies (Table 2), latent query count m (Figure 6), and different expert team compositions (Default, Generalist, Naive, LLM teams)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is reported as the evaluation metric across all tasks. No secondary metrics (e.g., output quality, reasoning chain analysis, token efficiency) are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is automated (accuracy on benchmark tasks). Human inspection of reasoning chains or expert utilization quality could have provided additional insight."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For Reasoning Gym: 'we train on 16,000 procedurally generated data points and report performance on a held-out validation set of 4,000 samples.' For GSM8K: 'we train on the standard training split and evaluate directly on the official test set.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-task-family breakdowns (Algorithmic, Arithmetic, Logic, GSM8K). Figure 4 shows per-expert utilization across different tasks and team compositions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses failures: 'On Algorithmic tasks, improvements are negligible,' 'For Logic and especially GSM8K, adding expert context provides limited benefit and often degrades performance,' and discusses high variance as evidence of failure to exploit SLM information."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Generalist Team on GSM8K degrades to 41.02% from 64.32% baseline; Logic performance drops from 96.88% to 82.81%; Hard Routing fails dramatically across all tasks. The paper acknowledges expert conditioning can be 'counterproductive.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract says the approach 'is competitive with strong single-model RLVR baselines,' which is generous for GSM8K (61.59% vs 64.32%) but the abstract also qualifies with 'gains varying across tasks.' Claims about emergent expert utilization are supported by Figure 4 analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'expert-derived latent context can provide a useful inductive bias' and 'RLVR induces structured expert utilization' are supported by controlled ablations: removing context tokens C (Figure 5), removing cross-attention (Table 1), varying expert teams, and varying pooling strategies (Table 2)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds claims: 'hidden state collaboration should be applied selectively, rather than treated as a general-purpose improvement over single-model RLVR.' Results are reported per-task and the paper acknowledges task-dependent effectiveness."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses 'implicit capacity bias' as an alternative to functional expertise: 'higher-capacity models are favored irrespective of nominal domain specialization.' Section 4.4 discusses that 'a coarse and relatively prompt-invariant representation of each SLM is sufficient,' suggesting gains come from static model signatures rather than reasoning traces."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on benchmark tasks and claims improvements in accuracy. There is no proxy gap — claims match the granularity of measurements. The paper does not frame benchmark accuracy as a broader capability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names are provided: Qwen-2.5-3B-Instruct, Qwen2.5-Math-1.5B-Instruct, Qwen2.5-Coder-7B-Instruct, Llama-3.2-3B-Instruct, Phi-3.5-mini-instruct, gemma-2-2b-it, Mistral-7B-Instruct-v0.3, GPT-2, GPT-OSS-20B. For open models, these names identify specific HuggingFace checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The YAML configs reference 'DeepSeekZero' as the developer_prompt but the actual prompt text is not provided. For the output collaboration baseline, expert prompts are described ('prompted to generate a short textual hint') but the actual text is not given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive hyperparameter reporting in Section 4.1 and Appendix: LoRA rank 8, latent dimension 512, GRPO group size 8, KL coefficient β = 0.04, batch size 32, learning rate 1e-4, 500/233 training steps, 128 token generation budget, m = 8 latent queries, 8 attention heads, weight decay 1e-2, grad clip 1.0."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a training-time adapter that integrates frozen expert hidden states via cross-attention, not an agent framework."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix Listings 1-4 provide exact YAML configs for each task family including dataset composition, weights, and curriculum settings. Reasoning Gym tasks use default settings with procedural generation. GSM8K uses the standard train/test split."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Conclusions (Section 6) briefly mentions that the hard-routing comparison 'does not exhaust the space of possible router designs' and that the framework 'prioritizes understanding expert utilization... rather than computational efficiency,' but this is not a substantive dedicated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Specific concerns are raised throughout the paper (high variance, capacity bias vs functional expertise, task-dependent benefits), but these are embedded in the analysis rather than collected in a threats-to-validity discussion. No dedicated discussion of specific threats to the study's validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'hidden state collaboration should be applied selectively' but does not explicitly state what the results do NOT show. The conclusions mention future work directions but not specific negative scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, model outputs, or training logs are made available. Only aggregated results in tables and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly described: Reasoning Gym procedurally generates problems from specified task families, and GSM8K is a standard benchmark. The appendix configs detail exact generation settings."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard benchmarks (Reasoning Gym, GSM8K)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: procedural generation with specified configs → GRPO training with stated hyperparameters → evaluation on held-out sets. Appendix YAML configs make the pipeline reproducible at a high level."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments, grants, or sponsor information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UC Irvine (majority), Intel Corporation (Mercati), West Point (Bastian), Northeastern University (Imani). The Intel affiliation is disclosed prominently."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. An Intel co-author raises potential conflict since Intel has commercial interest in efficient multi-model systems, but this is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. Intel affiliation creates a potential conflict that is not addressed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the frozen expert models (Qwen-2.5, Llama-3.2, Phi-3.5, Gemma-2, Mistral-7B, GPT-2). These models could have seen GSM8K during pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the frozen expert models' pre-training data includes GSM8K test problems. Reasoning Gym's procedural generation mitigates contamination for those tasks, but this is not explicitly discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K was published in 2021; all expert models were trained after 2021 and may have seen it. The paper does not discuss contamination risk. Reasoning Gym's procedural generation inherently mitigates contamination but this advantage is not explicitly noted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported despite the method requiring forward passes through multiple frozen expert models per input, which adds significant computational overhead."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'single NVIDIA H200 GPU' but does not quantify total GPU hours, training time, or cost for any experiment."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'Each experiment is repeated three times with different random seeds, and we report the mean and standard deviation.' Results across seeds are visible in Table 1 standard deviations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated in Section 4.1: 'Each experiment is repeated three times with different random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper fixes key hyperparameters (m = 8, latent dim 512, LoRA rank 8) without describing how these values were selected or how many configurations were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The default configuration (m = 8 latent queries) is used without justification. Figure 6 shows m = 8 is optimal in a sweep, but this sweep was presumably done on the same evaluation data used for reporting, and no separate validation set for hyperparameter selection is mentioned."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 4 tasks, 5+ methods, and several team configurations without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All baselines (Hard Routing, Output Collaboration) are the authors' own implementations. The paper does not acknowledge the bias of evaluating their own system against their own implementations of competing approaches."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The method requires multiple frozen expert forward passes plus cross-attention per input. The paper acknowledges 'Not minimal compute' in Figure 2 but never reports performance as a function of compute budget or provides compute-matched comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Reasoning Gym or GSM8K actually measures the 'reasoning' capabilities the paper claims to evaluate. The paper uses these benchmarks without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. All methods share the same policy model and RLVR training setup; they differ only in how expert context tokens are constructed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. GSM8K (2021) predates all expert models' training; solutions may appear in their training data. Reasoning Gym's procedural generation inherently avoids temporal leakage, but this is not explicitly noted."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The frozen experts see the same problem prompt as the policy, but whether this introduces any feature leakage through their hidden representations is not examined."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Reasoning Gym's procedural generation likely ensures independence, but this is not explicitly verified. For GSM8K, no analysis of train/test independence is provided."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Soft hidden-state collaboration improves reasoning performance over single-model RLVR, with gains up to 22.9% in the best-performing case (Arithmetic).",
       "evidence": "Table 1: Default Team achieves 75.26% on Arithmetic vs 52.34% for Single-model RLVR. However, performance degrades on Logic (82.81% vs 96.88%) and GSM8K (61.59% vs 64.32%).",
       "supported": "moderate"
     },
     {
       "claim": "RLVR induces structured, task-dependent expert utilization without routing or expert-role supervision.",
       "evidence": "Figure 4 shows routing entropy decreasing monotonically during training across tasks, with mean Spearman ρ = −0.48 between entropy and reward. Expert attention distributions shift from uniform to concentrated allocations.",
       "supported": "strong"
     },
     {
       "claim": "A substantial fraction of performance gains can be attributed to prefix-style latent conditioning rather than fine-grained expert reasoning traces.",
       "evidence": "Table 2: First-token pooling (75.13%) matches last-token pooling (75.26%) on Arithmetic, suggesting the method relies on coarse model signatures rather than prompt-specific reasoning. Section 4.4 states experts act as 'stable model signatures that regularize the policy.'",
       "supported": "moderate"
     },
     {
       "claim": "The learned interface can suppress low-utility experts like GPT-2 and redistribute attention toward higher-capacity experts like GPT-OSS.",
       "evidence": "Figure 4: GPT-2 is suppressed throughout training in the Naive LM Team; GPT-OSS receives pronounced attention redistribution in the LLM Team. Qualitative evidence from attention weight evolution.",
       "supported": "moderate"
     },
     {
       "claim": "Hidden state collaboration is task-dependent: beneficial when the policy has remaining capacity for improvement, counterproductive when the task is near saturation.",
       "evidence": "Table 1: Clear improvement on Arithmetic (52.34% → 75.26%), negligible on Algorithmic (51.56% → 51.82%), degradation on Logic (96.88% → 82.81%) and GSM8K (64.32% → 61.59%). Section 4.2 analysis.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Best-case emphasis in abstract",
       "detail": "The abstract highlights 'up to 22.9% improvement' from the best setting (Arithmetic, Default Team). Most other task-team combinations show marginal or negative effects. The word 'competitive' in the abstract is generous given GSM8K degradation."
     },
     {
       "flag": "Extreme variance suggesting training instability",
       "detail": "Generalist Team on GSM8K shows 41.02 ± 29.01, meaning at least one of the three runs likely collapsed near zero. This extreme variance is acknowledged but not adequately explained or resolved."
     },
     {
       "flag": "No compute-matched comparison",
       "detail": "The method runs multiple frozen expert forward passes per input in addition to the policy. Single-model RLVR uses only the policy. No experiment controls for the additional compute, so improvements on Arithmetic could partly reflect compute advantages."
     },
     {
       "flag": "Only 3 random seeds",
       "detail": "With only 3 seeds and high run-to-run variance, the standard deviations have very wide confidence intervals themselves. Many differences between methods may not be statistically reliable."
     },
     {
       "flag": "Missing contamination analysis for GSM8K",
       "detail": "GSM8K (2021) is one of the most widely contaminated benchmarks. All frozen expert models were trained after 2021 and may encode GSM8K solutions in their hidden states. This confound is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Foundational work on RLVR for reasoning in LLMs, demonstrating that structured reasoning can be transferred to smaller models through RL and distillation."
     },
     {
       "title": "Phi-4 technical report",
       "authors": ["Marah Abdin"],
       "year": 2024,
       "arxiv_id": "2412.08905",
       "relevance": "Demonstrates that small language models can reach or surpass GPT-3.5-class reasoning performance, supporting the modular SLM paradigm."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu"],
       "year": 2024,
       "relevance": "Multi-agent LLM conversation framework representing the output-level collaboration paradigm that this paper compares against."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "relevance": "Multi-agent collaborative framework for software development, representing agentic AI coordination approaches."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du"],
       "year": 2023,
       "relevance": "Foundational work on multi-LLM debate for reasoning improvement, a key baseline paradigm for multi-model collaboration."
     },
     {
       "title": "Reasoning gym: Reasoning environments for reinforcement learning with verifiable rewards",
       "authors": ["Zafir Stojanovski"],
       "year": 2025,
       "relevance": "Primary benchmark framework used in this paper for evaluating RLVR-based reasoning, with procedurally generated tasks."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "GSM8K benchmark paper, a standard mathematical reasoning evaluation used as a secondary benchmark in this work."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["Edward J Hu"],
       "year": 2022,
       "relevance": "Parameter-efficient fine-tuning method used for the trainable policy in this work, central to the lightweight adapter approach."
     },
     {
       "title": "Small language models are the future of agentic ai",
       "authors": ["Peter Belcak"],
       "year": 2025,
       "arxiv_id": "2506.02153",
       "relevance": "Argues for composing collections of off-the-shelf expert SLMs rather than scaling monolithic LLMs, directly motivating this paper's approach."
     },
     {
       "title": "Router-r1: Teaching LLMs multi-round routing and aggregation via reinforcement learning",
       "authors": ["Haozhen Zhang"],
       "year": 2025,
       "relevance": "RL-based LLM routing approach representing the hard expert routing paradigm that this paper compares against."
     },
     {
       "title": "DAPO: An open-source LLM reinforcement learning system at scale",
       "authors": ["Qiying Yu"],
       "year": 2025,
       "relevance": "Open-source RL system for LLM training; DAPO is one of the RL algorithms the paper's RLVR framework generalizes over."
     },
     {
       "title": "Does reinforcement learning really incentivize reasoning capacity in LLMs beyond the base model?",
       "authors": ["Yang Yue"],
       "year": 2025,
       "relevance": "Questions whether RLVR induces new reasoning capabilities or merely reshapes existing behaviors, providing important context for interpreting this paper's claims."
     }
   ]
 }