ai-research-survey

scan.json (27594B)

---

 {
   "paper": {
     "title": "KORMo: Korean Open Reasoning Model for Everyone",
     "authors": ["Minjun Kim", "Hyeonseok Lim", "Hangyeol Yoo", "Inho Won", "Seungwoo Song", "Minkyung Cho", "Junghun Yuk", "Changsu Choi", "Dongjae Shin", "Huije Lee", "Hoyun Song", "Alice Oh", "KyungTae Lim"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.09426",
     "doi": "10.48550/arXiv.2510.09426"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "KORMo-10B is a 10.8B-parameter fully open bilingual Korean-English LLM trained from scratch with 68.7% synthetic Korean data. The paper demonstrates that synthetic data does not cause model collapse when diversity is maintained across synthesizers, but using a single synthesizer leads to severe degradation. The model achieves competitive performance against open-weight multilingual baselines (64.2 English avg, 58.2 Korean avg) with only 2.9T training tokens, and scores 8.61 on instruction-following benchmarks without reinforcement learning.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'All model checkpoints, datasets, and source codes are publicly available at huggingface.co/kormo-lm.' The paper emphasizes full FOM (Fully Open Model) release throughout."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states all datasets are released at huggingface.co/kormo-lm. Table 10 references multiple publicly available datasets and states underlined datasets are 'generated synthetic data produced in-house' which are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions NVIDIA H200 GPUs, 128 GPUs, FSDP, bfloat16, and flash-attention-3, but does not provide a requirements.txt, Dockerfile, or detailed library versions for environment reproduction."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While training configurations are detailed in Tables 1 and 13, the paper does not provide step-by-step reproduction instructions, README with commands, or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All benchmark results (Tables 2-4, 7-8, 11, 16-18, 21-22) report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., 'Pre-LN consistently outperforms MixLN', 'KORMo achieved the highest average score') but no statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point differences with baseline context throughout, e.g., '+4.45 pt (60.15→64.60)' in Section 5.2.1, '+6.88pt increase in average score (43.43→50.31)' for Korean benchmarks."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of benchmarks selected, the 60B token proxy training budget, or why specific evaluation splits were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run results are reported. All experiments appear to be single-run with no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 21 compares KORMo against SmolLM3-3B, OLMo2-7B/13B, KANANA-8B, Qwen3-8B, LLaMA3.1-8B, Gemma3-4B/12B. Table 22 compares instruction-tuned models similarly."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent models: Qwen3-8B (2025), SmolLM3-3B (2025), Gemma3-12B (2025), OLMo2-13B (2025), KANANA-1.5 (2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablations in Section 2: normalization methods (Table 2), attention masking (Table 3), NTP vs MTP (Table 4), tokenizer variants (Tables 6-8), deduplication strategies (Table 11), quality filtering versions (Table 12), and synthetic data diversity (Figure 8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Over 26 benchmarks used across English and Korean, covering reasoning, knowledge, domain-specific, and instruction-following tasks (Table 20)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs. The MT-Bench, KO-MT-Bench, and Logickor evaluations use GPT-4o as an automated judge, not human raters."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established external benchmarks (MMLU, ARC, HellaSwag, KMMLU, etc.) that are separate from training data. Proxy experiments use different data splits than final model training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark across English general reasoning, English knowledge, Korean general reasoning, and Korean knowledge categories (Tables 16-18, 21-22)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 and Figure 8 explicitly show and analyze a failed training case with single-synthesizer data. Section 7.2 discusses weaknesses on KOBALT, MMLU-Pro, and KMMLU. Korean NIAH degradation beyond 13K tokens is analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "MTP underperforms NTP at 1B scale (Table 4). MixLN underperforms Pre-LN (Table 2). 196K vocabulary underperforms 125K on downstream tasks. Single-synthesizer data causes model collapse (Figure 8). Korean NIAH degrades beyond 13K tokens."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about synthetic data stability (supported by Tables 2-3, Figure 8), comparable performance to baselines (supported by Table 21), and bilingual instruction tuning (supported by Table 22) are all backed by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily from controlled ablations: synthetic vs non-synthetic under same conditions (Tables 2-3), normalization methods (Table 2), attention masking (Table 3), single vs multi-synthesizer (Figure 8). These are controlled single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims to present 'a transparent framework for developing synthetic data-driven fully open models in low-resource settings' but only tests Korean-English. Section 2.2 acknowledges proxy-stage limitations but the broader framing of 'non-English FOM' from a single language pair is overbroad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 discusses why MTP underperforms (model capacity, data scale). Section 4.3 analyzes causes of model collapse (single-model bias, prompt uniformity). Section 5.1 discusses Korean NIAH degradation factors (tokenizer instability, data imbalance, domain bias)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific benchmark scores and reports them as such. Claims are framed in terms of benchmark performance rather than broader capabilities. The proxy model approach is explicitly acknowledged as a proxy (Section 2.1: 'we interpret the performance trends observed at the proxy stage with a focus on relative comparison')."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions used for synthesis are stated: Qwen3-30B-A3B, Qwen3-235B-A22B, Qwen3-Next-80B-A3B-Instruct, GPT-oss (120B), Mixtral-8x7B-Instruct-v0.1, Mistral-Nemo-12B-Instruct, DeepSeek-R1, QwQ-32B (Tables 10, 15, 19)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt strategies in natural language (e.g., 'appending the instruction \"answer in Korean\"', 'custom-designed prompts') but does not provide the actual full prompt text used for synthetic data generation. Only Figure 11 shows a simple MMLU evaluation prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 13 provide detailed model architecture configs. Section 4.2.1 documents learning rate search. Training details include batch size 1024, sequence length 4096, weight decay policy, warm-up schedule, RoPE theta, RMSNorm epsilon, etc."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard LLM pretraining and fine-tuning paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 provides detailed 3-stage filtering pipeline: 8 heuristic filters with specific thresholds, Bloom Filter deduplication with cross-corpus details, and quality filtering with classifier training procedure. Token counts at each stage are provided in Tables 10-12."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Some limitations are mentioned inline (e.g., Section 2.2 on proxy-scale caveats, Section 7.2 on weaknesses) but no consolidated discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. While weaknesses are noted in results discussion, there is no systematic consideration of threats to validity of the experimental methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 explicitly identifies weaknesses: 'relative weaknesses on (a) high-difficulty specialist knowledge benchmarks (e.g., MMLU-Pro, the KMMLU family) and (b) lexical semantic discrimination tasks (e.g., KOBALT).' Section 2.2 acknowledges proxy-stage results may not transfer to larger scales."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states all datasets are released at huggingface.co/kormo-lm, including synthetic data, training data, and checkpoints. Table 10 lists all data sources with links."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection in detail: public English sources (Section 4.1.1), Korean public data from three sources (Korean Opensource, Ko-Web, Ko-CC-Dump), and synthetic generation procedures (Section 4.1.2)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public corpora and synthetic generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 documents the full pipeline: heuristic filtering (8 steps with thresholds), deduplication (BFF with cross-corpus sequence), quality filtering (4 classifier versions with composition details in Table 12), with filtering ratios discussed (e.g., Old-both removed ~70% of Korean corpus)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work was supported by Institute of Information & communications Technology Planning & Evaluation (IITP) grant funded by the Korea government(MSIT) (RS-2025-02653113).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: KAIST MLP Lab, KAIST NLPCL Lab, KAIST U&I Lab, and SeoulTech. Special thanks acknowledge Trillion Labs and LG U+ contributions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "IITP is a Korean government research funding agency with no commercial stake in the model's benchmark performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. The acknowledgments mention Trillion Labs and LG U+ contributions, but no formal financial interests declaration exists."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Ko-CC-Dump uses WARC files from March-May 2025, providing some temporal information, but no explicit training data cutoff date is stated for the overall model or the public datasets used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (MMLU, ARC, HellaSwag, etc.) appear in the training data (DCLM, UltraFineWeb, Nemotron-CC, etc.)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses well-known benchmarks (MMLU from 2020, ARC from 2018, HellaSwag from 2019) with training data that could contain them, but contamination is not discussed. The only mention is using KR-Clinical-QA to 'mitigate potential data contamination' (Section 7.1) but no systematic analysis."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens-per-second metrics are reported for the final model."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 128 NVIDIA H200 GPUs were used but does not report total GPU hours, training time, or compute cost for any training stage."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. All experiments appear to be single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs without explicit confirmation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 reports learning rate search over 8 candidates {1e-4, 3e-4, 5e-4, 7e-4, 9e-4, 1e-3, 1.5e-3, 3e-3} with 2000 steps each. Figure 6 shows all loss curves."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 justifies selection of 7e-4 learning rate based on lowest loss across all steps (Figure 6). Tokenizer selection in Section 3.4 is justified by downstream performance comparison."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, let alone corrections for multiple comparisons across the many benchmark comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of author-evaluation bias. The authors evaluate their own model against baselines without discussing this potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 explicitly discusses token budget vs performance: 'Given its relatively modest pretraining corpus of 2.9T tokens, these results highlight KORMo-10B's high language modeling efficiency.' Table 21 shows training token counts for all models compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks actually measure the claimed capabilities. The paper uses many benchmarks without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the evaluation. Models are directly evaluated on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Many benchmarks (MMLU 2020, ARC 2018, HellaSwag 2019) predate the training data which includes 2025 web crawls."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information. The few-shot evaluation prompts could theoretically provide hints not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training corpora (DCLM, UltraFineWeb, Nemotron-CC) share overlap with evaluation benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The only mention is using the recently released KR-Clinical-QA to 'mitigate potential data contamination' (Section 7.1), but no decontamination pipeline or detection method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "Synthetic data does not cause instability or degradation during large-scale pretraining when carefully curated with balanced linguistic coverage and diverse instruction styles.",
       "evidence": "Tables 2-3 show synthetic data outperforms or matches web data under same conditions. However, Figure 8 shows single-synthesizer data causes model collapse. Evidence is from 1B proxy model and 10.8B model (Figures 7-8).",
       "supported": "strong"
     },
     {
       "claim": "KORMo-10B achieves performance comparable to contemporary open-weight multilingual baselines.",
       "evidence": "Table 21 shows 64.2 English avg vs Qwen3-8B 68.55, Gemma3-12B 67.72; Korean avg 58.15 vs KANANA-8B 60.94, Qwen3-8B 63.35. Competitive but not matching the best baselines.",
       "supported": "moderate"
     },
     {
       "claim": "Bilingual instruction tuning enables near-native reasoning and discourse coherence in Korean.",
       "evidence": "Table 22 shows KO-MT-Bench score of 8.54 and Logickor 8.96, outperforming KANANA (8.02, 8.94) and Qwen3 (8.16, 8.63) on Korean instruction-following. GPT-4o used as judge.",
       "supported": "moderate"
     },
     {
       "claim": "Using augmented data from a single synthesizer causes model collapse, but diverse multi-synthesizer data avoids this.",
       "evidence": "Figure 8 shows Stage 2 (Failure) with single Qwen3-30B-A3B synthesizer vs successful Stage 2 with diverse synthesizers. Clear degradation visible in both English and Korean performance curves.",
       "supported": "strong"
     },
     {
       "claim": "Pre-LN consistently outperforms MixLN for normalization.",
       "evidence": "Table 2 shows Pre-LN average 43.38% vs MixLN 41.28% on 1B proxy model. Only one configuration comparison with no variance or significance testing.",
       "supported": "weak"
     },
     {
       "claim": "Intra-document masking is the most effective attention masking strategy.",
       "evidence": "Table 3 shows Intra-doc 44.48% vs Causal 43.38%, Sliding causal 43.33%, Sliding Intra-doc 43.59%. Differences are small with no statistical testing.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or significance testing across all experiments",
       "detail": "Despite numerous comparative claims across Tables 2-4, 7-8, 11, 16-18, 21-22, all results are single-run point estimates. Differences of 1-2% are treated as meaningful without any statistical validation. The ablation differences (e.g., 43.38% vs 41.28% for normalization) could easily be within run-to-run variance."
     },
     {
       "flag": "No contamination analysis despite using well-known benchmarks",
       "detail": "The model is trained on web-crawled data (including 2025 Common Crawl dumps) and evaluated on benchmarks from 2018-2022 (ARC, HellaSwag, MMLU). No decontamination pipeline or overlap analysis is performed. The brief mention of using KR-Clinical-QA to 'mitigate contamination' is insufficient."
     },
     {
       "flag": "No compute budget reported",
       "detail": "128 H200 GPUs were used for training ~3.5T tokens but total GPU hours, wall-clock time, and training cost are never stated. For a paper about training efficiency and FOM accessibility, this is a notable omission."
     },
     {
       "flag": "LLM-as-judge evaluation without validation",
       "detail": "MT-Bench, KO-MT-Bench, and Logickor evaluations all use GPT-4o as the sole judge. No inter-rater reliability, no human validation of the judge, and no discussion of potential biases in GPT-4o judging a model partly trained on synthetic data from Qwen models."
     },
     {
       "flag": "Overclaiming from proxy experiments",
       "detail": "Key design decisions (normalization, attention masking, MTP vs NTP) are made from 1B proxy models trained on 60B tokens, then applied to 10.8B model trained on 2.9T tokens. The paper acknowledges this limitation for some results but still makes broad claims about the findings."
     }
   ],
   "cited_papers": [
     {
       "title": "2 OLMo 2 Furious",
       "authors": ["Pete Walsh", "Luca Soldaini", "Dirk Groeneveld"],
       "year": 2025,
       "arxiv_id": "2501.00656",
       "relevance": "Key fully open model baseline; establishes FOM practices for reproducible LLM training."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Aaron Grattafiori"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Major open-weight model family used as architecture reference and baseline."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reasoning model whose data generation approach influenced KORMo's reasoning data construction."
     },
     {
       "title": "Qwen3 Technical Report",
       "authors": ["Qwen Team"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Primary synthesizer model family used for generating KORMo's Korean synthetic data."
     },
     {
       "title": "SmolLM3: smol, multilingual, long-context reasoner",
       "authors": ["Elie Bakouch", "Loubna Ben Allal"],
       "year": 2025,
       "relevance": "Fully open model baseline; APO/preference learning strategy adopted from this work."
     },
     {
       "title": "The curse of recursion: Training on generated data makes models forget",
       "authors": ["Ilia Shumailov"],
       "year": 2023,
       "relevance": "Core concern addressed by KORMo: whether synthetic data causes model collapse in LLM pretraining."
     },
     {
       "title": "Textbooks Are All You Need",
       "authors": ["Suriya Gunasekar"],
       "year": 2023,
       "relevance": "Pioneering work on synthetic textbook-style data for LLM training, foundational to KORMo's approach."
     },
     {
       "title": "Nemotron-CC: Transforming Common Crawl into a Refined Long-Horizon Pretraining Dataset",
       "authors": ["Dan Su"],
       "year": 2025,
       "arxiv_id": "2412.02595",
       "relevance": "Major data source for KORMo's pretraining; provides synthetic and web data at scale."
     },
     {
       "title": "Nemotron-4 340B Technical Report",
       "authors": ["Nvidia"],
       "year": 2024,
       "arxiv_id": "2406.11704",
       "relevance": "Industrial-scale synthetic data pipeline that informed KORMo's data augmentation approach."
     },
     {
       "title": "BLOOM: A 176B-Parameter Open-Access Multilingual Language Model",
       "authors": ["BigScience Workshop"],
       "year": 2023,
       "arxiv_id": "2211.05100",
       "relevance": "Prominent multilingual FOM predecessor; demonstrates community-driven open model development."
     },
     {
       "title": "DCLM: In Search of the Next Generation of Training Sets for Language Models",
       "authors": ["Jeffrey Li"],
       "year": 2024,
       "relevance": "Major English pretraining data source and filtering methodology adopted by KORMo."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks"],
       "year": 2020,
       "relevance": "MMLU benchmark used as primary English knowledge evaluation in KORMo."
     }
   ]
 }