ai-research-survey

scan.json (25546B)

---

 {
   "paper": {
     "title": "Scaling Laws for Code: Every Programming Language Matters",
     "authors": ["Jian Yang", "Shawn Guo", "Lin Jing", "Wei Zhang", "Aishan Liu", "Chuan Hao", "Zhoujun Li", "Wayne Xin Zhao", "Xianglong Liu", "Weifeng Lv", "Bryan Dai"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.13472",
     "doi": "10.48550/arXiv.2512.13472"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Different programming languages exhibit distinct scaling behaviors: interpreted languages (e.g., Python) show larger scaling exponents than compiled languages (e.g., Rust), and irreducible loss orders languages by intrinsic complexity (C# < Java ≈ Rust < Go < TypeScript < JavaScript < Python). Multilingual pre-training provides synergistic benefits for most languages, with syntactically similar pairs (Java-C#) showing the largest gains (20.5% improvement). Parallel pairing of code translations significantly enhances cross-lingual capabilities with favorable scaling properties. A proportion-dependent multilingual scaling law enables optimal token allocation that outperforms uniform distribution.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or release link found anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The training corpus and custom translation evaluation set are described but no download links or public release is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes model architecture (LLaMA-2 style with SwiGLU, RoPE, MHA, RMSNorm) but provides no environment specifications, dependency lists, or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, scripts, or step-by-step guides are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1, 3, and Figure 6 report point estimates only. No confidence intervals or error bars are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'parallel pairing significantly outperforms baseline' and '20.5% improvement' are made based on comparing numbers directly, with no statistical significance tests applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are consistently reported, e.g., 'Java-C# combination achieves validation loss of 0.718 compared to 0.903 for Java self-repetition—a remarkable 20.5% improvement' (Section 4.2). Table 1 shows absolute values and relative improvement percentages."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 10 model sizes, 6 token budgets, or 7 programming languages were chosen. The choices appear pragmatic but are not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each experimental configuration appears to be a single training run. No variance, standard deviation, or spread measures across runs are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include: monolingual self-repetition baseline (Section 4), random shuffling baseline (Section 5), and uniform allocation baseline (Section 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines represent standard practice (uniform allocation, monolingual training). The paper also references the recent code scaling law work [19] (2025). The baselines are appropriate for this type of study."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies individual factors: language-specific vs. mixed training (Section 4), different data organization strategies (random shuffling vs. parallel pairing, Section 5), and uniform vs. optimized allocation (Section 6)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses validation loss (cross-entropy), Pass@1 on MultiPL-E, and BLEU score for code translation (Table 3, Figure 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code or translations. Evaluation is entirely automated (loss, Pass@1, BLEU)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes a carefully curated held-out evaluation set: '50 Python files from GitHub' with manual translations to 6 target languages, yielding 2,100 translation instances. MultiPL-E is a separate benchmark."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per programming language throughout (Table 1 synergy matrix, Table 3 per-language MultiPL-E, Figure 6 per-language Pass@1 and BLEU). Figure 4 shows per-direction translation results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where multilingual training hurts: 'when Python is the target PL, mixing with most auxiliary PL produces small negative effects' (Section 4.2). Specific negative synergy values are reported."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Python's negative transfer from most languages is reported explicitly: 'JavaScript (Δ = −0.009), TypeScript (Δ = −0.007), C# (Δ = −0.013), Go (Δ = −0.016), and Rust (Δ = −0.021)' (Section 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about interpreted vs. compiled language scaling (supported by Figure 2), multilingual synergistic benefits (Table 1), parallel pairing advantages (Figure 3-4, Table 3), and optimal allocation outperforming uniform (Figure 6) are all backed by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'parallel pairing significantly enhances cross-lingual abilities' are supported by controlled experiments comparing strategies under identical compute budgets and architectures. The experimental design isolates the variable being studied (Section 4.1 compares D_Li+D_Li vs D_Li+D_Lj)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds generalization: 'only seven programming languages,' 'largest model reaches 14B parameters,' 'evaluation focuses on code translation and generation benchmarks,' and 'synergy coefficients are fitted to our specific corpus.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers interpretations for findings (e.g., why Java-C# have high synergy) but does not discuss alternative explanations for the observed scaling patterns or consider confounding factors beyond the variables studied."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims are well-matched to its measurements. It measures validation loss, Pass@1, and BLEU, and frames findings in terms of these specific metrics rather than making broader unmeasured claims about code quality or developer productivity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Models are trained from scratch with specified architectures (LLaMA-2 style with SwiGLU, RoPE, MHA, RMSNorm) and exact parameter counts (10 sizes from 0.1B to 3.1B, plus 0.5B/1.5B/3B/7B for translation experiments). Since these are custom-trained models, architecture specification is appropriate."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 mentions 'prompt-based concatenation' as a pre-training strategy but does not provide the actual prompt templates used. MultiPL-E evaluation prompts are not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No learning rate, optimizer, batch size, warmup schedule, or other training hyperparameters are reported in the provided text. Only architecture and data volume are specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a pre-training study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We collect a high-quality training corpus' but provides no details on filtering, deduplication, or preprocessing steps applied to the training data."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears at the end of the paper with five specific limitations discussed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this study: 'only seven programming languages,' 'largest model reaches 14B parameters with 1T tokens, whereas state-of-the-art code LLMs exceed 100B,' 'synergy coefficients are fitted to our specific corpus; different data distributions may yield varying patterns.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what is NOT shown: extending to low-resource languages, validation at extreme scales (>100B), complex tasks like program repair, and dynamic curriculum strategies."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (training corpus, validation losses per run, fitted parameters datasets) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the cross-lingual evaluation set construction: '50 Python files from GitHub' selected by 'three software engineers,' human annotators producing translations for 6 target languages. Training corpus composition is described (900B code + 100B FineWeb-Edu)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. The software engineers creating the evaluation set are annotators, not research subjects, and the data is model training runs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw code data to training corpus is not documented. No filtering criteria, deduplication methods, or data processing steps are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or sponsorships are disclosed in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Beihang University, Ubiquant (a quantitative finance firm), and Renmin University of China."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Ubiquant is a quantitative finance company that likely uses code LLMs. Their financial interest in scaling law outcomes is not discussed. No funding disclosure makes independence assessment impossible."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosures are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Models are trained from scratch on a custom corpus, but the paper does not state when the training data was collected or its temporal boundaries relative to MultiPL-E."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MultiPL-E problems or solutions appear in the training corpus. Since they train from scratch with a custom corpus, they could verify this but do not."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MultiPL-E is a public benchmark. The paper does not discuss whether its problems could appear in the training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported for the trained models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Equivalent to 336,000+ H800 hours' for the 1000+ experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Each of the 1000+ experiments appears to be a single training run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total number of configurations is stated (420 for Section 3, 28 for Section 4) but it is not stated whether each configuration was run multiple times."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Training hyperparameters (learning rate, optimizer, etc.) are not even reported, let alone search budgets."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The 'optimized allocation' in Section 6 is derived analytically from fitted scaling laws and synergy matrices, not cherry-picked from trial runs."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors train all models and evaluate them without acknowledging potential bias from implementing and tuning their own systems."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance is explicitly shown as a function of compute (model size and data size) throughout the paper. Figure 2 shows scaling surfaces. Section 6 compares strategies at identical compute budgets (400B tokens)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "MultiPL-E is used without discussing whether Pass@1 on this benchmark adequately measures multilingual code generation capability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The training corpus could contain solutions to MultiPL-E problems that were published before data collection."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The custom translation eval set is created by the authors, but overlap with training data is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test data share structural similarities (e.g., same repositories, similar coding patterns)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is described or applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Interpreted languages (e.g., Python) benefit more from increased model size and data than compiled languages (e.g., Rust), showing larger scaling exponents.",
       "evidence": "Figure 2 shows fitted scaling parameters: Python has the highest αN and αD values, while Rust shows notably smaller exponents (Section 3.2).",
       "supported": "strong"
     },
     {
       "claim": "Irreducible loss establishes a complexity ordering: C# < Java ≈ Rust < Go < TypeScript < JavaScript < Python.",
       "evidence": "Fitted L∞ values from Figure 2 across 420 training runs with systematic variation in model size and data (Section 3.2).",
       "supported": "strong"
     },
     {
       "claim": "Multilingual pre-training provides synergistic benefits, with Java-C# showing 20.5% improvement over monolingual baseline.",
       "evidence": "Table 1 shows synergy gain matrix from 28 bilingual mixture experiments. Java-C# achieves validation loss of 0.718 vs 0.903 for self-repetition (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Python suffers negative transfer when mixed with most other languages during pre-training.",
       "evidence": "Table 1 / Section 4.2: negative synergy for Python with JavaScript (Δ = −0.009), TypeScript (−0.007), C# (−0.013), Go (−0.016), Rust (−0.021). Only Java provides positive synergy.",
       "supported": "strong"
     },
     {
       "claim": "Parallel pairing (concatenating code with translations) significantly enhances cross-lingual abilities compared to random shuffling.",
       "evidence": "Figure 3 shows parallel pairing achieves lower validation loss on unseen translation directions across all model sizes (0.2B-7B). Table 3 shows better MultiPL-E scores (Section 5.3-5.4).",
       "supported": "moderate"
     },
     {
       "claim": "Optimized token allocation achieves higher average performance across all PLs compared to uniform distribution under the same compute budget.",
       "evidence": "Figure 6: optimized allocation achieves Pass@1 21.34 vs 19.84 baseline and BLEU 13.9 vs 13.3. Both trained on 400B tokens (Section 6.4).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "Over 1000 experiments are conducted but each appears to be a single run. No variance, standard deviation, or confidence intervals are reported. Given that neural network training is stochastic, the fitted scaling parameters could be sensitive to random seed."
     },
     {
       "flag": "Missing training hyperparameters",
       "detail": "Critical training details (learning rate, optimizer, batch size, warmup schedule) are absent from the paper. This makes reproduction impossible even if code and data were released."
     },
     {
       "flag": "Small final evaluation",
       "detail": "The key validation claim (Section 6) compares only two 1.5B models on 400B tokens. The improvement in Pass@1 (19.84 → 21.34) and BLEU (13.3 → 13.9) is modest and based on single runs with no error bars."
     },
     {
       "flag": "No contamination analysis despite custom training",
       "detail": "The authors train from scratch with a custom corpus, giving them full control and ability to verify train/test overlap with MultiPL-E, yet no contamination analysis is performed."
     },
     {
       "flag": "Industry affiliation not discussed",
       "detail": "Ubiquant is a quantitative finance company. The substantial compute resources (336K H800 hours) likely came from them, but no funding disclosure or conflict of interest statement is present."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex/HumanEval paper establishing code generation evaluation methodology."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling law paper that this work extends to multilingual code."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Major code LLM trained on multiple programming languages, directly relevant to multilingual code pre-training."
     },
     {
       "title": "StarCoder: May the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Large-scale multilingual code model pre-training, relevant to understanding code LLM development."
     },
     {
       "title": "Scaling laws for code: A more data-hungry regime",
       "authors": ["Xianzhen Luo", "Wenzhen Zheng"],
       "year": 2025,
       "arxiv_id": "2510.08702",
       "relevance": "Direct predecessor establishing code-specific scaling laws that this paper extends to multilingual settings."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper for language models."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "relevance": "Major open-source code LLM relevant to understanding multilingual code model training."
     },
     {
       "title": "Qwen2.5-Coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Recent code LLM with multilingual pre-training, co-authored by this paper's first author."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges emergent abilities narrative relevant to scaling law interpretation."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "relevance": "Early code pre-training work establishing the field this paper studies."
     }
   ]
 }