ai-research-survey

scan-v5.json (27482B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "InternBootcamp Technical Report: Boosting LLM Reasoning with Verifiable Task Scaling",
     "authors": [
       "Peiji Li",
       "Jiasheng Ye",
       "Yongkang Chen",
       "Yichuan Ma",
       "Zijie Yu",
       "Kedi Chen",
       "Ganqu Cui",
       "Haozhan Li",
       "Jiacheng Chen",
       "Chengqi Lyu",
       "Wenwei Zhang",
       "Linyang Li",
       "Qipeng Guo",
       "Dahua Lin",
       "Bowen Zhou",
       "Kai Chen"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2508.08636",
     "doi": "10.48550/arXiv.2508.08636"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's core claims—1000+ task environments, automated generation with configurable difficulty, integrated verification, and task scaling improving performance—are all supported by Section 3-4 and Tables 5-6. The claim that the 32B model 'excels on other established benchmarks' is supported by Table 6 showing the SFT-RL model achieves the best average across 9 OOD benchmarks.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims task scaling causally improves performance, but experiments don't fully disentangle task count from data volume—more tasks also means more diverse training data. The near-linear scaling fit (Figure 5b) is from 4 points (8/32/128/512 tasks) without controlling for total tokens seen.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims training produces 'reasoning generalists' and delivers 'comprehensive reasoning capabilities,' but OOD results in Table 6 are mixed—Qwen2.5-32B+Bootcamp-RL shows no improvement on BBEH, GPQA-Diamond, or SuperGPQA. Generalizations exceed the evidence.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions one alternative ('RL activates existing knowledge') but does not discuss alternatives such as the effect of training data volume vs. task diversity, selection bias in BOOTCAMP-EVAL construction by the same team, or the role of the SFT distillation from DeepSeek-R1.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures benchmark accuracy on BOOTCAMP-EVAL and OOD benchmarks and consistently refers to these as 'reasoning performance' scores; it does not conflate benchmark scores with broader cognitive capabilities beyond benchmark context.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section anywhere in the paper. The conclusion only discusses open-sourcing contributions.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed—the paper does not address BOOTCAMP-EVAL construction bias, the self-referential evaluation problem, or the possibility that improvements reflect data volume rather than task diversity.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do not show. There are no statements bounding the scope of the 'reasoning generalist' claim to the tested tasks or model families.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgment section appears in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with Shanghai AI Laboratory and Fudan University are disclosed in the header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Shanghai AI Laboratory is an organization that builds and promotes LLMs; the primary evaluation is on BOOTCAMP-EVAL, a benchmark constructed by the same team, creating a conflict of interest between the evaluators and the evaluated system.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key technical terms are defined: 'task scaling' (increasing the number of training tasks), 'bootcamp class' (a modular task environment with case_generator, prompt_function, verify_function), and 'RLVR' are all explained in context.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly listed: the INTERNBOOTCAMP framework, the BOOTCAMP-EVAL benchmark, and empirical demonstration of task scaling effectiveness.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly situates the work against existing reasoning studies (CoT, RLVR), domain-specific environments (Go, chess, math), and puzzle-based multi-task frameworks like Enigmata, SynLogic, and KORGym, showing how INTERNBOOTCAMP extends these with broader domain coverage.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Source code is publicly released at https://github.com/InternLM/InternBootcamp, confirmed in the abstract and footnote 3.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "BOOTCAMP-EVAL benchmark (9,232 samples across 118 tasks) is open-sourced: 'The complete benchmark suite is open-sourced.'",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or dependency specification is mentioned in the paper. Code examples reference Python imports but no environment spec is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Code integration examples (Figures 9-10) illustrate API usage but no step-by-step instructions for reproducing the training experiments (data generation, RL training, evaluation) are provided.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are shown in any figure. For the 8-task experiments 4 trials were averaged, but variance is not reported.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any comparative claims between training configurations or model comparisons.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute score improvements are reported in Tables 5 and 6 with baseline values provided (e.g., Qwen2.5-32B-Instruct 24.4% → SFT-RL 59.5% on BOOTCAMP-EVAL).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 9,232-sample benchmark and 118-task evaluation set are described but no power analysis or statistical justification for sample sizes is given.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Variance across runs is not reported for any experiment. The 8-task condition used 4 trials with averaging, but standard deviation is not provided.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Multiple competitive baselines are included: DeepSeek-V3-0324, DeepSeek-R1-0528, QwQ-32B, Qwen3-32B, Qwen3-235B-A22B in Table 5, and ablation baselines in Table 6.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All baselines are 2024-2025 state-of-the-art models; the paper explicitly compares against Qwen3, QwQ-32B, and DeepSeek-R1-0528.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Task count ablation across 4 levels (8, 32, 128, 512 tasks) and training paradigm ablation (RL only, SFT only, SFT+RL) are both conducted systematically.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation spans 9 OOD benchmarks covering logic (BBEH, KOR-Bench), knowledge (GPQA-Diamond, SuperGPQA, MMLU-Pro), math (AIME 2025, LiveMathBench-Hard), and code (HumanEval, LiveCodeBench) domains.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "The paper evaluates LLM performance on automated verifiable tasks; human evaluation of system outputs is not applicable.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "BOOTCAMP-EVAL is explicitly held out from training: 'we ensure no data leakage by deduplicating the training data against BOOTCAMP-EVAL, guaranteeing clean evaluation.'",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Figure 7 provides per-domain performance curves across all 8 BOOTCAMP-EVAL categories for each task count configuration; Table 5 also breaks down scores by domain.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly discusses the 8-task entropy collapse failure (Figure 6), and three tasks (Hyperbaton, PropositionalLogicFormalization, Wordscapes) that fail under isolated single-task training (Figure 8).",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results are reported: standalone RL training of Qwen2.5-32B-Instruct shows minimal OOD improvement (Table 6), and 8-task training collapses after 300 steps.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model versions are named: Qwen2.5-7B-Instruct, Qwen2.5-32B-Instruct, DeepSeek-R1-Distill-Qwen-32B, DeepSeek-R1, and DeepSeek-V3 for bootcamp generation.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "The full training prompt template is provided verbatim in Appendix D (Figure 11), including the <think> tag formatting instruction.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Key hyperparameters are reported: prompt batch size 128, 8 responses per step, temperature 1.0, rollout batch size 384, max 500 training steps, SFT 3 epochs, no token-level loss.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The DAPO-based rollout with dynamic sampling, format penalties, length penalties, and oversampling strategy (3× rollout batch size) are described in detail in Section 4.1.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Data preprocessing is documented: evolutionary generation with 3 iterations, self-consistent unittest filtering (accuracy thresholds 0.03 and 0.85), deduplication, difficulty calibration, and quality filtering yielding 704 tasks.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "BOOTCAMP-EVAL raw benchmark data is open-sourced, and the code to generate training data is publicly available on GitHub.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 and Appendix A describe four task source categories (puzzles, reasoning benchmarks, algorithm problems, scientific tasks) with specific origin datasets cited.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants are involved; tasks are collected from existing datasets and generated automatically.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline—task description collection → evolutionary bootcamp generation with DeepSeek-R1 → self-consistent unittest filtering → human review → deduplication and quality filtering—is documented in Section 3.3.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for the base models (Qwen2.5 and DeepSeek-R1) are not stated, making it impossible to assess whether OOD evaluation benchmarks were in pretraining data.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly addresses train-test overlap for BOOTCAMP-EVAL: 'we ensure no data leakage by deduplicating the training data against BOOTCAMP-EVAL, guaranteeing clean evaluation.'",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The paper evaluates on OOD benchmarks including AIME 2025 and LiveCodeBench but does not discuss whether these benchmarks were potentially seen during pretraining of Qwen2.5 or DeepSeek-R1 base models.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, latency, or GPU memory figures are reported for running the 32B models or executing the training pipeline.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget (GPU hours, hardware specs, estimated cost) is stated for any experiment including the full 32B SFT+RL training.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Scaling training tasks from 8 to 512 yields near-linear improvement on BOOTCAMP-EVAL, following S = 2.65×10⁻⁴N + 0.22",
       "evidence": "Figure 5b shows a linear fit across 4 task counts (8/32/128/512), though the fit is from only 4 data points",
       "supported": "moderate"
     },
     {
       "claim": "Training with 8 tasks causes entropy collapse after ~300 steps, while 32+ tasks maintains stable training",
       "evidence": "Figure 6 shows rollout batch generation exploding for 8-task training, with clear divergence from multi-task configurations",
       "supported": "strong"
     },
     {
       "claim": "Multi-task training enables emergence of capabilities on tasks that cannot be learned in single-task isolation",
       "evidence": "Figure 8 shows three specific tasks (Hyperbaton, PropositionalLogicFormalization, Wordscapes) achieving zero gain in single-task training but significant improvement around step 300 with 512-task training",
       "supported": "moderate"
     },
     {
       "claim": "Bootcamp-SFT-RL 32B model achieves best average across 9 OOD benchmarks vs. evaluated baselines",
       "evidence": "Table 6 shows SFT-RL Qwen2.5-32B achieves 61.8% average vs. 52.5% for DeepSeek-R1-Distill-32B and 42.3% for base Qwen2.5-32B-Instruct",
       "supported": "strong"
     },
     {
       "claim": "InternBootcamp Bootcamp-SFT model outperforms all listed open-source frontier models on BOOTCAMP-EVAL",
       "evidence": "Table 5 shows Bootcamp-SFT achieves 61.1% vs. Qwen3-235B-A22B at 54.5% and QwQ-32B at 51.4%, though the evaluated model is 32B vs. larger parameter-count models",
       "supported": "moderate"
     },
     {
       "claim": "Automated evolutionary bootcamp generation with self-consistent unittest filtering achieves 70.6% no-obvious-error rate in the 0.03-0.85 accuracy range",
       "evidence": "Table 3 directly shows the filtering results with manual inspection confirming the accuracy thresholds distinguish error types effectively",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "InternBootcamp demonstrates that scaling the number of diverse verifiable training tasks (from 8 to 512) systematically improves LLM reasoning performance with a near-linear relationship, and that training with very few tasks (8) causes entropy collapse in RL training. The paper shows 'emergent moments' where capabilities on tasks that fail in single-task training emerge after sufficient multi-task exposure. The 32B SFT+RL model trained on all 1000+ bootcamp tasks achieves the best average across 9 OOD reasoning benchmarks among evaluated baselines, though improvements are uneven across benchmarks and the primary evaluation benchmark was constructed by the same team.",
   "red_flags": [
     {
       "flag": "Self-referential primary benchmark",
       "detail": "BOOTCAMP-EVAL, the primary evaluation benchmark, was constructed by the same team that built the training framework. This creates circular evaluation where the benchmark may favor the design choices already embedded in the training tasks."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section despite making broad claims about 'reasoning generalists' and cross-domain generalization."
     },
     {
       "flag": "Confounded task scaling",
       "detail": "More training tasks necessarily means more diverse training data; the paper does not control for total training tokens or data volume, making it unclear whether benefits come from task count or data diversity."
     },
     {
       "flag": "No statistical tests or error bars",
       "detail": "All comparisons between training configurations and baselines are made without significance tests or reported variance. Only 8-task experiments used multiple trials."
     },
     {
       "flag": "Mixed OOD results underdiscussed",
       "detail": "Table 6 shows Qwen2.5-32B+Bootcamp-RL achieves no improvement on BBEH, GPQA-Diamond, or SuperGPQA compared to base, but this is not discussed as a limitation."
     },
     {
       "flag": "No compute budget",
       "detail": "No GPU hours, hardware specifications, or cost estimates are provided for any training run, making practical replication assessment impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "relevance": "Primary baseline and the model used for automated bootcamp generation; central to the RL training paradigm."
     },
     {
       "title": "Enigmata: Scaling Logical Reasoning in Large Language Models with Synthetic Verifiable Puzzles",
       "relevance": "Direct prior work on synthetic verifiable puzzle datasets for LLM reasoning; INTERNBOOTCAMP extends its scope."
     },
     {
       "title": "SynLogic: Synthesizing Verifiable Reasoning Data at Scale for Learning Logical Reasoning and Beyond",
       "relevance": "Related work on large-scale verifiable reasoning data synthesis; direct competitor in the task-scaling space."
     },
     {
       "title": "KOR-Bench: Benchmarking Language Models on Knowledge-Orthogonal Reasoning Tasks",
       "relevance": "One of the OOD evaluation benchmarks and a task source for INTERNBOOTCAMP; used as both training material and evaluation."
     },
     {
       "title": "BigBench Extra Hard (BBEH)",
       "relevance": "OOD evaluation benchmark and task source for INTERNBOOTCAMP; measures reasoning difficulty beyond standard benchmarks."
     },
     {
       "title": "DAPO: An Open-Source LLM Reinforcement Learning System at Scale",
       "relevance": "The RL algorithm used for training; key methodological component of the task scaling experiments."
     },
     {
       "title": "HybridFlow: A Flexible and Efficient RLHF Framework (VeRL)",
       "relevance": "One of the RL training frameworks INTERNBOOTCAMP integrates with; cited as integration target."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "OOD evaluation benchmark for code generation; used to assess generalization beyond reasoning tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Open-source framework with 1000+ tasks that practitioners can directly use for RL training, SFT data synthesis, and evaluation of their own models."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Task scaling following data scaling laws is expected; the 'emergent moment' finding for isolated tasks is mildly surprising but framed as a known phenomenon in transfer learning."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk concerns raised; the paper focuses on improving reasoning capabilities without discussing misuse potential."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict framing; straightforward technical contribution from a research lab."
     },
     "demo_ability": {
       "score": 3,
       "justification": "GitHub repository is publicly available and the framework provides immediately runnable bootcamp classes with documented API; practitioners can run BOOTCAMP-EVAL on their models today."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Shanghai AI Laboratory is a well-known Chinese AI research organization behind InternLM; not as globally prominent as DeepSeek or OpenAI but recognized in the LLM community."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "37288987",
         "title": "Answering ambiguous questions with a database of questions, answers, revisions",
         "points": 21,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37288987",
         "created_at": "2023-08-28T02:28:01Z"
       },
       {
         "hn_id": "44908292",
         "title": "Distillation Scaling Laws",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44908292",
         "created_at": "2025-08-15T03:22:10Z"
       },
       {
         "hn_id": "43039955",
         "title": "Distillation Scaling Laws",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43039955",
         "created_at": "2025-02-13T19:11:35Z"
       },
       {
         "hn_id": "44427833",
         "title": "Simple low-dimensional computations explain variability in neuronal activity",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44427833",
         "created_at": "2025-06-30T21:01:35Z"
       },
       {
         "hn_id": "45222579",
         "title": "KNNSampler: Stochastic Imputations for Recovering Missing Value Distributions",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45222579",
         "created_at": "2025-09-12T14:32:36Z"
       },
       {
         "hn_id": "44604096",
         "title": "Coordination and Collaborative Reasoning in Multi-Agent LLMs",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44604096",
         "created_at": "2025-07-18T12:45:27Z"
       }
     ],
     "top_points": 21,
     "total_points": 33,
     "total_comments": 1
   }
 }