ai-research-survey

scan.json (26540B)

---

 {
   "paper": {
     "title": "Qwen2.5-Coder Technical Report",
     "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui", "Jiaxi Yang", "Dayiheng Liu", "Lei Zhang", "Tianyu Liu", "Jiajun Zhang", "Bowen Yu", "Keming Lu", "Kai Dang", "Yang Fan", "Yichang Zhang", "An Yang", "Rui Men", "Fei Huang", "Bo Zheng", "Yibo Miao", "Shanghaoran Quan", "Yunlong Feng", "Xingzhang Ren", "Xuancheng Ren", "Jingren Zhou", "Junyang Lin"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.12186"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Model weights released on HuggingFace (Table 4, Table 15) and evaluation code released on GitHub: https://github.com/QwenLM/Qwen2.5-Coder (Section 6, Section 7)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The pretraining dataset 'Qwen2.5-Coder-Data' (5.5T tokens) is not released. Only the model weights and evaluation code are public. The instruction tuning data is also not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. Hardware and software setup for training/evaluation are not detailed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While evaluation code is released on GitHub, there are no step-by-step reproduction instructions for the training process or detailed evaluation setup in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results across all tables (Tables 5-20) are reported as single point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous claims that Qwen2.5-Coder 'outperforms' other models based solely on comparing raw numbers. No statistical significance tests are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute scores with baseline context throughout (e.g., 'Qwen2.5-Coder-32B achieves state-of-the-art performance with an average improvement of 7.9% EM and 4.2% ES compared to DS-Coder-33B-Base' in Section 6.2). Percentage improvements are given with baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the choice of benchmarks, number of test examples, or why these particular evaluation sets are sufficient to support the claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any results. All numbers appear to be single-run results with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: StarCoder2 series, DeepSeek-Coder series, CodeLlama series, CodeQwen1.5, CodeStral, Yi-Coder, and closed-source APIs (GPT-4o, Claude 3.5 Sonnet, o1-mini/preview)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models such as DeepSeek-Coder-V2 (2024), Claude-3.5-Sonnet (2024), GPT-4o (2024), and o1-mini/preview. These represent the state of the art at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows an ablation of data mixture ratios (100:0:0, 85:15:5, 70:20:10 for Code:Text:Math). Figure 1 shows the effect of different data cleaning stages on performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Pass@1, Pass@2, Exact Match (EM), Edit Similarity (ES), win rate (CodeArena, CodeEditorBench). Results span code generation, completion, reasoning, editing, and math tasks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 describes CodeArena, 'an internal annotated evaluation benchmark called CodeArena, including nearly 400 human-curated samples' used for human preference alignment evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard held-out benchmarks are used (HumanEval, MBPP, etc.). LiveCodeBench uses problems from 2024.07-2024.11 specifically to avoid contamination. Section 5 describes decontamination procedures."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per-language (MultiPL-E Table 6, 17), per-task type (code generation, completion, reasoning, editing), per-model size, and per-completion granularity (line, function, API in RepoEval Table 10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is presented. The paper only reports success metrics without discussing where or why the model fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows that the 100:0:0 code-only data mixture performs worse than mixed ratios, which is a somewhat surprising negative finding. The paper reports that 'larger models did not yield significant benefits' for the text-code grounding data filter (Section 3.1.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SOTA performance 'across more than 10 benchmarks' and 'consistently outperforming larger models of the same model size.' The results tables (5-20) support these claims with the 32B model matching or exceeding GPT-4o on most benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'Math and Text data may positively contribute to code performance' (Section 3.1.2) based on comparing three data mixtures. However, only three ratios are tested with no variance reporting or significance tests, making the causal inference weak."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame results broadly ('code generation capabilities', 'code intelligence'). While results are on specific benchmarks, claims like 'the most powerful open-source code model to date' (Section 7.1) are not bounded to the tested benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the performance gains. The improvements could stem from scale, data quality, architecture, or other confounds, but these are not disentangled or discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass@1 on coding benchmarks but frames results as 'code intelligence' and 'coding capabilities' broadly. No discussion of whether benchmark performance proxies real-world coding ability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model names with sizes are specified for all Qwen models. For closed-source APIs, specific versions are given: 'GPT-4o-2024-08-06', 'Claude-3.5-Sonnet-20240620', 'Claude-3.5-Sonnet-20241022' (Table 16)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'a unified prompt template' for Text-to-SQL (Section 7.4) following prior work, but does not provide the actual prompts used for evaluations across all benchmarks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Architecture hyperparameters are reported in Table 1. Training details include sequence lengths (8,192 for file-level, 32,768 for repo-level), RoPE base frequency (10,000 → 1,000,000), and context limits for evaluation (e.g., 'maximum sequence length of 8192 tokens', 'maximum output length of 50 tokens'). FIM format is specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates base and instruct models directly on benchmarks without scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data composition and cleaning in detail: rule-based filtering, coarse-to-fine hierarchical filtering with fastText classifiers, 4-stage iteration process (Figure 1), code removal from text data, and decontamination via 10-gram overlap (Section 5)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section in the paper. The conclusion (Section 9) mentions future directions but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show or what settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The pretraining data is not released. Evaluation results are presented as aggregated scores with no raw per-example outputs available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection: 'public repositories from GitHub created before February 2024, spanning 92 programming languages', Common Crawl text-code data, synthetic data from CodeQwen1.5, and data from Qwen2.5-Math and Qwen2.5."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data sources are standard benchmarks and public repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: Figure 2 shows the three-stage training pipeline, Section 3.1.1 describes filtering stages with validation (Figure 1 shows token counts at each stage: 582B → 370B → 147B → 118B), and Section 5 describes decontamination."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with 'Qwen Team, Alibaba Group' on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Alibaba has a direct commercial interest in demonstrating that Qwen2.5-Coder is superior to competitors. The funder (Alibaba) has a clear stake in the outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Authors are Alibaba employees evaluating Alibaba's product but no explicit conflict declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.1 states 'public repositories from GitHub created before February 2024.' This establishes the training data cutoff for the code data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes decontamination: 'We removed key datasets such as HumanEval, MBPP, GSM8K, and MATH. The filtering was done using a 10-gram overlap method.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly addresses benchmark contamination with 10-gram decontamination. LiveCodeBench uses problems from 2024.07-2024.11 to 'strictly avoid test data contamination' (Section 8)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. CodeArena uses human-curated samples but is a benchmark, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed per evaluation is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training compute budget (GPU hours, hardware, training time) is not stated despite training on 5.5T tokens across six model sizes."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results are reported across multiple random seeds. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated for any evaluation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for training or evaluation. The data mixture ablation (Table 3) tests only 3 ratios with no search budget stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows the data mixture selection with results for 3 configurations, and the selected 70:20:10 ratio is justified by its superior average performance across coding, math, and general benchmarks."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite comparing across many benchmarks and model sizes."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Alibaba authors evaluate their own Qwen2.5-Coder models against competitors. No acknowledgment of self-evaluation bias. They note evaluation code is public, which partially mitigates but does not address the bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis. Different models have vastly different compute budgets (e.g., DS-Coder-V2 at 236B MoE vs Qwen2.5-Coder-32B dense) but this is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmarks like HumanEval, MBPP actually measure real-world coding ability. The paper uses benchmarks without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used; models are evaluated directly on benchmarks. The Aider evaluation is the closest to scaffolded, but is a standard benchmark setup."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 addresses decontamination and Section 8 uses LiveCodeBench (2024.07-2024.11) specifically to test OOD capabilities after the training cutoff, directly addressing temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information through context or hints not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training repositories overlap with benchmark repositories or whether similar problems exist across train and test sets beyond the 10-gram overlap filter."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes a concrete decontamination method: '10-gram overlap method, where any training data with a 10-gram word-level overlap with the test data was removed.' This is an active prevention method."
       }
     }
   },
   "claims": [
     {
       "claim": "Qwen2.5-Coder-32B-Instruct achieves state-of-the-art performance among open-source code models, matching GPT-4o coding capabilities.",
       "evidence": "Table 16 shows Qwen2.5-Coder-32B-Instruct achieving 92.7 HumanEval (vs GPT-4o 92.1), 90.2 MBPP (vs 86.8), 49.6 BigCodeBench-Full (vs 50.1), 31.4 LiveCodeBench (vs 34.6).",
       "supported": "moderate"
     },
     {
       "claim": "A 70:20:10 Code:Text:Math data mixture outperforms code-only training.",
       "evidence": "Table 3 shows the 70:20:10 ratio achieving 55.0 average vs 31.3 for 100:0:0, tested on Qwen2.5-Coder-7B.",
       "supported": "moderate"
     },
     {
       "claim": "Qwen2.5-Coder-7B outperforms the larger DS-Coder-33B across code generation benchmarks.",
       "evidence": "Table 5 shows Qwen2.5-Coder-7B (61.6 HE, 53.0 HE+, 76.9 MBPP) exceeding DS-Coder-33B (54.9, 47.6, 74.2) on all five metrics.",
       "supported": "moderate"
     },
     {
       "claim": "The coarse-to-fine text-code filtering yields progressive improvement through 4 stages.",
       "evidence": "Figure 1 shows average performance on HumanEval/MBPP increasing from 41.6% to 46.8% across 4 filtering stages, tested on Qwen2.5-Coder-1.5B.",
       "supported": "moderate"
     },
     {
       "claim": "Qwen2.5-Coder supports 128K token context length.",
       "evidence": "Section 6.6 and Figure 6 show 'Needle in the Code' evaluation demonstrating correct retrieval across 10K-128K context lengths, though the task is very simple (retrieving a single custom function).",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Qwen2.5-Coder is a series of six code-specific LLMs (0.5B-32B) built on Qwen2.5 and trained on 5.5T tokens. The 32B-Instruct model matches GPT-4o on many coding benchmarks while being open-source. A key finding is that a 70:20:10 Code:Text:Math data mixture outperforms code-only training. The models demonstrate strong scaling properties across model sizes, with each size achieving SOTA among open-source models of comparable scale.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are Alibaba employees evaluating Alibaba's Qwen2.5-Coder. No independent evaluation or acknowledgment of self-evaluation bias. The paper consistently highlights favorable comparisons while not discussing failures."
     },
     {
       "flag": "No error bars or variance reporting",
       "detail": "Across 20+ tables and figures with hundreds of benchmark numbers, not a single result includes error bars, confidence intervals, or variance across runs. Claims of 'outperforming' are based entirely on comparing point estimates."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no discussion of limitations, threats to validity, or scope boundaries. This is unusual for a 32-page technical report and suggests selective presentation."
     },
     {
       "flag": "Missing compute budget",
       "detail": "Training six models on 5.5T tokens each represents massive compute, but no training time, GPU hours, or hardware specifications are reported, making it impossible to assess practical replicability."
     },
     {
       "flag": "Internal benchmark without public validation",
       "detail": "CodeArena is described as an 'internal annotated evaluation benchmark' with ~400 samples. Since it's not public, these results cannot be independently verified."
     },
     {
       "flag": "Cherry-picked favorable framing",
       "detail": "On LiveCodeBench the 32B model (31.4) trails GPT-4o (34.6) and o1-mini (60.0) substantially, yet the abstract claims the model 'matches the coding capabilities of GPT-4o' based on other benchmarks where it performs better."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the foundational code generation benchmark used throughout."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Major open-source code LLM and baseline compared against."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "StarCoder2 series used as key baseline; data curation methodology influenced this work."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "DeepSeek-Coder series is the primary competitive baseline throughout the paper."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Code LLM baseline from Meta, compared across multiple benchmarks."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J Liu", "CS Xia"],
       "year": 2023,
       "relevance": "EvalPlus benchmark (HumanEval+, MBPP+) used as primary evaluation tool."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "BigCodeBench used as challenging code generation benchmark evaluating tool-use capabilities."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark using competitive programming problems, key for addressing data leakage."
     },
     {
       "title": "MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation",
       "authors": ["Federico Cassano"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multilingual code generation benchmark used across multiple evaluations."
     },
     {
       "title": "CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution",
       "authors": ["Alex Gu", "Baptiste Rozière"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Code reasoning benchmark used to evaluate whether models understand code execution flow."
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov"],
       "year": 2023,
       "arxiv_id": "2305.18290",
       "relevance": "DPO training methodology used in the post-training alignment stage of Qwen2.5-Coder."
     },
     {
       "title": "Efficient Training of Language Models to Fill in the Middle",
       "authors": ["Mohammad Bavarian"],
       "year": 2022,
       "arxiv_id": "2207.14255",
       "relevance": "Fill-in-the-Middle (FIM) training strategy adopted for code completion capabilities."
     }
   ]
 }