ai-research-survey

scan.json (24794B)

---

 {
   "paper": {
     "title": "Qwen2.5 Technical Report",
     "authors": ["An Yang", "Baosong Yang", "Beichen Zhang", "Binyuan Hui", "Bo Zheng"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2412.15115",
     "doi": "10.48550/arXiv.2412.15115"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Qwen2.5 scales pre-training data from 7T to 18T tokens and introduces multi-stage reinforcement learning (DPO + GRPO) for post-training. The open-weight 72B model matches or exceeds Llama-3-405B-Instruct on many benchmarks despite being ~5x smaller. MoE variants (Qwen2.5-Turbo, Qwen2.5-Plus) offer competitive performance at lower cost. Qwen2.5-Turbo handles up to 1M token context length with 100% passkey retrieval accuracy.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper links to https://github.com/QwenLM/Qwen2.5 and models are available on Hugging Face, ModelScope, and Kaggle. Over 100 models released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 18T token pre-training dataset and 1M+ SFT examples are not released. Only the model weights are open. No training data is made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specifications are provided in the paper. Hardware and dependency details are not described."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions for training or evaluation are provided. The paper describes methodology at a high level but does not give reproducible commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All benchmark results are reported as point estimates without confidence intervals or error bars. Tables 2-17 show only single numbers."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'outperforms', 'surpasses') but no statistical significance tests are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with absolute scores and baselines, allowing readers to compute effect sizes. E.g., MATH scores of 62.1 vs 50.9 (Qwen2-72B) or 53.8 (Llama-3-405B) provide sufficient context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sample sizes or discussion of whether benchmark sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multiple-run statistics are reported. All results appear to be single-run evaluations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: Llama-3/3.1 series, Gemma2 series, Mixtral, GPT-4o/4o-mini, Claude 3.5 Sonnet, Yi-1.5, Mistral, Phi-3.5, MiniCPM3, and predecessor Qwen2 models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models from 2024: Llama-3.1-405B, GPT-4o, Claude 3.5 Sonnet, Gemma2, which were state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Ablation on DCA+YARN is shown in Tables 16-17, demonstrating the contribution of length extrapolation techniques. Scaling law experiments study hyperparameter effects across model sizes."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Extensive metrics across categories: MMLU, MMLU-Pro, BBH, ARC-C, TruthfulQA, MATH, GSM8K, HumanEval, MBPP, MultiPL-E, IFEval, Arena-Hard, MTBench, RULER, LV-Eval, etc."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human preference alignment is evaluated via Arena-Hard and MT-Bench. Human and automated labeling processes are used for reward model training data. In-house evaluations include human preference benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmarks with established test splits are used. The paper describes decontamination via n-gram matching (Section 5) to prevent test data leakage."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by category: general tasks, math & science, coding, multilingual, alignment, long context. In-house evaluations further break down by IF, Knowledge, Comprehension, Coding, Math, Reasoning."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis are discussed. The paper does not examine where Qwen2.5 fails or performs poorly relative to expectations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every comparison shows improvement or competitive performance. The DCA+YARN ablation shows degradation when removed, but no failed approaches or configurations are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about competitive performance with Llama-3-405B and GPT-4o are supported by Tables 2, 6, and 7. Claims about 18T tokens and 1M+ SFT samples are described in Sections 3 and 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes improvements to data scaling (7T→18T), better data filtering, and post-training techniques, but no controlled experiments isolate individual contributions. The causal chain from specific changes to performance gains is asserted, not demonstrated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and introduction use broad framing ('comprehensive series', 'meet diverse needs') while results are on specific benchmarks. No explicit bounding of generalization claims to tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for performance improvements are discussed. The paper does not consider whether benchmark scores could be influenced by factors other than the claimed improvements."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses benchmark scores as proxies for capabilities like 'reasoning', 'understanding', and 'language comprehension' without discussing the gap between what benchmarks measure and what these capability labels imply."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Baseline model versions are specified: GPT-4o-2024-08-06, GPT-4o-2024-11-20, Claude3.5-sonnet-2024-10-22, Llama-3.1-70B-Instruct, etc. Qwen2.5 model sizes and architectures are detailed in Table 1."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No evaluation prompts or system instructions are provided. The few-shot settings are described (e.g., '5-shot', '0-shot') but actual prompt text is not given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training hyperparameters are reported: learning rates (7×10⁻⁶ to 7×10⁻⁷), weight decay (0.1), gradient clipping (1.0), batch sizes (2048), SFT epochs (2), DPO pairs (150K). Architecture details in Table 1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a model training and benchmark evaluation paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data filtering using Qwen2-Instruct as quality filters, domain rebalancing strategy, synthetic data generation and filtering. Section 4.1 describes SFT data curation including back-translation, rejection sampling, and response filtering."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section. The paper ends with a Conclusion (Section 6) and future work directions but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues with its evaluation methodology or claims."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show or what settings are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation data or training data is made available. Only aggregated benchmark scores are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Pre-training data collection is described in Section 3.1 (quality filtering, domain rebalancing, synthetic data). Post-training data collection is described in Sections 4.1-4.3 with specific procedures for each capability area."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and internally curated datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The training pipeline is documented: pre-training (4K→32K context extension), SFT (1M+ examples, 2 epochs), offline RL (150K DPO pairs, 1 epoch), online RL (GRPO with reward model). Data filtering steps described in Section 3.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section. The paper does not disclose funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are identified as 'Qwen Team' with links to Hugging Face and ModelScope pages associated with Alibaba/Qwen."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Alibaba Cloud is both the developer and evaluator of Qwen2.5. The proprietary models are offered through Alibaba Cloud Model Studio. The funder has direct financial interest in positive results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Alibaba has commercial interest in Qwen2.5 models (offered via Alibaba Cloud), but this is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff date is stated. The paper does not specify when the 18T token pre-training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes n-gram matching decontamination: training sequences are removed if LCS with test sequences satisfies |LCS| ≥ 13 and |LCS| ≥ 0.6 × min(|st|,|se|)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly addresses benchmark contamination with a concrete decontamination procedure using longest common subsequence matching between training and test data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. This is a model training and benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Inference cost is not quantified. TTFT speedups are shown (Figure 3) but absolute costs, API pricing, or per-example inference times are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. GPU hours, hardware used for training, or total training cost are not disclosed despite training 7+ models on 18T tokens."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All benchmark results appear to be single-run evaluations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs is not stated for any benchmark result."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Scaling laws for hyperparameters are discussed (Section 3.2) but the search budget (number of configurations tried, compute spent on search) is not quantified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes using scaling laws to predict optimal hyperparameters across model sizes, providing a principled selection criterion rather than arbitrary tuning."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are used at all, so multiple comparison correction is not applicable in practice, but given the many comparisons claimed, it should have been. No correction is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Qwen team evaluates their own models against competitors without acknowledging self-evaluation bias. No independent evaluation is mentioned."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is not reported as a function of compute budget. The paper claims Qwen2.5-72B matches Llama-3-405B without comparing their training compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses many benchmarks without discussing whether they measure the claimed capabilities. Section 5.2.3 does note that RM benchmarks don't predict RL performance, but this is limited to reward models."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the evaluations. Models are tested directly on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Many benchmarks used (HumanEval 2021, MMLU 2021, GSM8K 2021) predate the training data, and this is not discussed beyond the n-gram decontamination."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage or whether evaluation setups provide information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test data share structural similarities beyond the n-gram overlap check."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes a concrete decontamination method using n-gram matching with specific thresholds (LCS ≥ 13 and ≥ 0.6 × min length). This is a concrete prevention method applied to the training data."
       }
     }
   },
   "claims": [
     {
       "claim": "Qwen2.5-72B-Instruct outperforms Llama-3.1-405B-Instruct on several benchmarks including MMLU-redux, MATH, MBPP, MultiPL-E, LiveCodeBench, Arena-Hard, and MTBench.",
       "evidence": "Table 6 shows Qwen2.5-72B-Instruct scores: MMLU-redux 86.8 vs 86.2, MATH 83.1 vs 73.8, MBPP 88.2 vs 84.5, MultiPL-E 75.1 vs 73.5, LiveCodeBench 55.5 vs 41.6, Arena-Hard 81.2 vs 69.3, MTBench 9.35 vs 9.08.",
       "supported": "moderate"
     },
     {
       "claim": "Pre-training data scaling from 7T to 18T tokens significantly improves model capabilities, especially domain expertise.",
       "evidence": "Figure 1 shows improvement from Qwen2 (7T) to Qwen2.5 (18T) on Math, MBPP, BBH, MMLU. Tables 2-5 show Qwen2.5 outperforming Qwen2 across size variants.",
       "supported": "moderate"
     },
     {
       "claim": "Qwen2.5-Turbo handles up to 1M token context with 100% passkey retrieval accuracy.",
       "evidence": "Figure 2 shows 100% accuracy on passkey retrieval across all document depths and context lengths up to 1M tokens. Table 16 shows RULER scores.",
       "supported": "moderate"
     },
     {
       "claim": "Qwen2.5-Turbo and Qwen2.5-Plus perform competitively against GPT-4o-mini and GPT-4o respectively.",
       "evidence": "Tables 6-7 show Qwen2.5-Plus outperforming GPT-4o-mini on most benchmarks, and Qwen2.5-Turbo competitive with GPT-4o-mini.",
       "supported": "moderate"
     },
     {
       "claim": "Two-stage reinforcement learning (offline DPO + online GRPO) significantly enhances human preference alignment.",
       "evidence": "Arena-Hard and MTBench scores in Tables 6-8 show high alignment scores, but no ablation isolating RL contribution vs. SFT alone is provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Alibaba's Qwen team evaluates their own Qwen2.5 models. All benchmark runs, baseline reproductions, and in-house evaluations are conducted by the same team that developed the models. No independent evaluation is mentioned."
     },
     {
       "flag": "No error bars or variance across runs",
       "detail": "All benchmark results are single point estimates across 20+ benchmarks and 7+ model sizes. No variance, confidence intervals, or multi-run statistics are reported despite LLM evaluation being known to be noisy."
     },
     {
       "flag": "No negative results",
       "detail": "Every comparison shows Qwen2.5 improving over Qwen2 and being competitive with or better than all baselines. No failed experiments, unsuccessful configurations, or areas of clear weakness are discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundaries discussion. This is a significant omission for a paper making broad capability claims about a model family."
     },
     {
       "flag": "Undisclosed compute budget",
       "detail": "Training 7+ dense models and 2 MoE models on 18T tokens with 1M+ SFT examples requires massive compute, but no GPU hours, hardware specifications, or training costs are disclosed."
     },
     {
       "flag": "Selective benchmark presentation",
       "detail": "In-house benchmarks (Tables 11-12) are used alongside open benchmarks, but the in-house benchmarks are not publicly available for independent verification."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, widely used for LLM code evaluation."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper that establishes the SFT + RLHF post-training paradigm used by Qwen2.5."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov"],
       "year": 2023,
       "relevance": "DPO method used as the offline RL stage in Qwen2.5's post-training pipeline."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Primary open-weight baseline compared against Qwen2.5 across all model sizes."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws that inform Qwen2.5's hyperparameter optimization approach."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark used to evaluate Qwen2.5 instruction-tuned models."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu"],
       "year": 2023,
       "relevance": "Introduces HumanEval+ and MBPP+ for more rigorous code generation evaluation, used as Qwen2.5 benchmarks."
     },
     {
       "title": "RewardBench: Evaluating reward models for language modeling",
       "authors": ["Nathan Lambert"],
       "year": 2024,
       "arxiv_id": "2403.13787",
       "relevance": "Benchmark used to evaluate Qwen2.5's reward model, with discussion of Goodhart's law in RM evaluation."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": ["Zhihong Shao"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Introduces GRPO method used as Qwen2.5's online RL algorithm."
     },
     {
       "title": "LiveBench: A challenging, contamination-free LLM benchmark",
       "authors": ["Colin White"],
       "year": 2024,
       "arxiv_id": "2406.19314",
       "relevance": "Contamination-free benchmark used to evaluate Qwen2.5 instruction-tuned models."
     }
   ]
 }