ai-research-survey

scan.json (26467B)

---

 {
   "paper": {
     "title": "Is GPT-OSS Good? A Comprehensive Evaluation of OpenAI's Latest Open Source Models",
     "authors": ["Ziqian Bi", "Keyu Chen", "Chiung-Yi Tseng", "Danyang Zhang", "Tianyang Wang", "Hongying Luo", "Lu Chen", "Junming Huang", "Jibin Guan", "Junfeng Hao", "Xinyuan Song", "Junhao Song"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.12461",
     "doi": "10.48550/arXiv.2508.12461"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GPT-OSS models (20B and 120B MoE) show mid-tier performance among contemporary open-source LLMs. The 20B model outperforms the 120B on several benchmarks (MMLU, SciQ), contradicting scaling expectations with statistical significance (p<0.01, Cohen's d=0.73). Both models are weak on multilingual tasks (C-Eval: 20-28%). The 20B variant offers superior cost-performance, requiring 5x less GPU memory with comparable or better accuracy.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract mentions 'evaluation scripts are available at the Project Webpage' but no actual URL is provided in the paper text. No repository link found."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All ten benchmarks used (MMLU, GSM8K, HumanEval, FinQA, PIQA, SciQ, MedQA, LegalQA, DialogSum, C-Eval) are publicly available standard benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-A mentions '8 NVIDIA H100 80GB GPUs, Ubuntu 22.04 LTS, vLLM' but no software versions, dependency files, or library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the pipeline at a high level (Section IV-C) but provides no commands, scripts, or README."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Fig 3 shows error bars representing 95% bootstrap confidence intervals. Section V-H states bootstrap CIs with max width ±2.1%."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "McNemar's test with Benjamini-Hochberg correction is used for pairwise comparisons (Section III-D). Section V-H confirms p-values below 0.05 for all reported differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d effect sizes are reported: d=0.73 for the inverse scaling finding (Section V-A), and range from d=0.52 to d=1.84 across comparisons (Section V-H)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. Section III-C mentions 'power analysis guidelines from Card et al.' but does not actually report a power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations or variance across runs are reported in the results tables. Results appear to be single-run numbers. Only bootstrap CIs on the aggregated metrics are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six contemporary open-source models serve as baselines: Qwen 3 235B, DeepSeek-R1 70B, Phi-4 Reasoning, Llama 4 Scout, Llama 3.3 70B, Gemma 3 27B (Table I)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baseline models are contemporary (2024-2025): Qwen 3, DeepSeek-R1, Phi-4 Reasoning, Llama 4 Scout, Llama 3.3, Gemma 3. These represent state-of-the-art open-source models."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation comparing existing models, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Ten benchmarks across five capability domains are used: MMLU, GSM8K, HumanEval, FinQA, PIQA, SciQ, MedQA, LegalQA, DialogSum, C-Eval (Table II)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E presents a case study with multi-dimensional quality assessment including human readability, answer clarity, and conciseness ratings (Table IV). A blind evaluation protocol is described in Section III-C."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used. Section IV-B describes data quality assurance with 5% manual inspection of samples. The benchmarks have established train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-benchmark breakdowns in Table II and Fig 4. Table III provides mathematical reasoning sub-category breakdown (basic arithmetic, multi-step algebra, word problems). Section V-B discusses per-subject MMLU breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C discusses failure patterns: 'both GPT-OSS models consistently fail on problems requiring numerical precision maintenance, particularly those involving unit conversions.' Section V-D discusses code generation failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The inverse scaling finding is itself a negative result — the larger model underperforms the smaller one. GPT-OSS's severe weakness in multilingual tasks (C-Eval: 20-28%) is prominently reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about mid-tier performance, inverse scaling between 20B and 120B, strength in code generation, and weakness in multilingual tasks are all supported by Tables II and IV and Fig 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'scaling in sparse architectures may not yield proportional performance gains' and attributes inverse scaling to 'potential inefficiencies in the MoE routing mechanism or suboptimal training configuration' (Section V-K) without evidence to support these causal mechanisms. The study is observational — it compares models but cannot isolate the cause of the scaling anomaly."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (Conclusion) explicitly bounds claims: 'this study relies on established benchmarks that, though widely adopted, do not fully capture emerging capabilities or real-world robustness, and we did not exhaustively optimise prompting or decoding strategies for any single model.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-K discusses multiple explanations for inverse scaling: MoE routing inefficiencies, suboptimal training configuration for the larger model. Section V-E acknowledges that response length differences 'may represent presentation choices rather than fundamental capability differences.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements — it reports benchmark accuracy scores and frames results in terms of those specific benchmarks rather than making broader capability claims. The abstract says 'mid-tier overall performance' based on the specific benchmarks tested."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table I lists model names and parameter counts (e.g., 'GPT-OSS 120B', 'Qwen 3 235B', 'DeepSeek-R1 70B') but no specific version hashes, snapshot dates, or checkpoint identifiers are provided for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. Section III-C mentions 'prompt engineering incorporates insights from Brown et al., Wei et al., and Kojima et al.' but the actual prompts used for each benchmark are not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A reports: temperature 0.7/0.1 by task type, top-p 0.95, top-k 50, max tokens 2000, repetition penalty 1.1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on benchmarks via standard inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B documents a multi-stage validation: 5% stratified manual inspection (~2,100 examples), character encoding validation, quality filtering with 98.7% retention rate and documentation of removed examples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The conclusion contains a single sentence acknowledging benchmark limitations and prompting non-optimization, but there is no substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The conclusion mentions generic limitations ('benchmarks do not fully capture emerging capabilities') without study-specific threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI states: 'this study relies on established benchmarks that...do not fully capture emerging capabilities or real-world robustness, and we did not exhaustively optimise prompting or decoding strategies for any single model.' Section V-J specifically notes GPT-OSS weakness in non-English domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation outputs, model responses, or per-example results are made available. Only aggregated accuracy numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes the data preparation pipeline in detail: manual inspection of 5% samples, character encoding validation, quality filtering. Section III-B describes benchmark selection criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes a five-stage evaluation pipeline (initialization, data loading, inference, post-processing, aggregation). Section IV-B documents filtering with 98.7% retention rate."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. The acknowledgments section thanks the open source community and mentions 'computational resources provided by our institution' but names no specific funder or grant."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Purdue, Georgia Tech, ByteDance, LuxMuse AI, University of Minnesota, Emory, Imperial College London, plus 'AI Agent Lab, Vokram Group' for most authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funder is disclosed, making it impossible to assess independence. The paper cannot demonstrate funder independence without first disclosing the funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included. Several authors are affiliated with commercial entities (ByteDance, LuxMuse AI, Vokram Group) but no financial interest declaration is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the eight evaluated models. This is critical for assessing whether benchmark data appeared in training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section II-C mentions contamination as a general concern and cites Sainz et al. and Magar & Schwartz, but no actual analysis of train/test overlap is performed for the models tested."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-E mentions 'data contamination detection employs methods from Sainz et al. and Magar & Schwartz' but no results of contamination detection are reported. HumanEval (2021), MMLU (2021), GSM8K (2021) etc. all predate these models' training — high contamination risk is unaddressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-I reports GPU memory per device (80GB vs 16GB), throughput (128 vs 178 tokens/s), and energy consumption (2.6x difference). Token count distributions shown in Fig 6."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (8x H100 80GB GPUs) but total compute time, GPU hours, or wall-clock time for the full evaluation is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. Section IV-E mentions 'fixed random seeds' for reproducibility but no seed sensitivity analysis is shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per benchmark per model is never stated. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Temperature and sampling parameters are stated but no search process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a single configuration per task type (temperature 0.7 for creative, 0.1 for factual) without justifying why these specific values were chosen or whether alternatives were tested."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D explicitly applies Benjamini-Hochberg correction for multiple comparisons among 8 models. Section V-H confirms Bonferroni correction was also used."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. The paper does not acknowledge that running all models through their own pipeline could introduce systematic biases."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section V-G and V-I explicitly compare efficiency vs performance: GPU memory (5x difference), energy per response (2.6x), throughput. Fig 2 plots parameter count vs accuracy."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the chosen benchmarks actually measure the claimed capabilities. The paper uses standard benchmarks without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly via standard inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage despite all benchmarks (MMLU 2021, HumanEval 2021, GSM8K 2021, etc.) predating the 2025 models by years."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information through context or formatting."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities with training data or whether independence was verified."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-E claims contamination detection methods were used citing Sainz et al. but no results are reported. No canary strings, membership inference, or overlap analysis results are shown."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-OSS-20B consistently outperforms GPT-OSS-120B on several benchmarks including MMLU (69% vs 66%) and SciQ (87% vs 82%), demonstrating inverse scaling.",
       "evidence": "Table II, Fig 7, Section V-A. Statistical validation: p<0.01, Cohen's d=0.73.",
       "supported": "strong"
     },
     {
       "claim": "Both GPT-OSS models demonstrate mid-tier overall performance within the current open-source landscape.",
       "evidence": "Table II shows GPT-OSS-20B averaging 67.7% and GPT-OSS-120B averaging 64.8%, placing them 7th and 8th among 8 models. Fig 3 confirms rankings.",
       "supported": "strong"
     },
     {
       "claim": "GPT-OSS-20B achieves comparable or superior accuracy with 5x less GPU memory and 2.6x lower energy per response.",
       "evidence": "Section V-I reports 80GB vs 16GB peak GPU memory and 2.6x energy reduction. Throughput: 178 vs 128 tokens/s.",
       "supported": "moderate"
     },
     {
       "claim": "Both GPT-OSS models show severe weakness in multilingual tasks with C-Eval scores of 28% (20B) and 20% (120B).",
       "evidence": "Fig 4 C-Eval panel and Table II. Well below the 45% threshold stated in Section V-J.",
       "supported": "strong"
     },
     {
       "claim": "Scaling laws may be task-dependent rather than universal, as GSM8K shows conventional scaling (120B: 88% vs 20B: 82%) while MMLU shows inverse scaling.",
       "evidence": "Table II and Section V-C compare task-specific scaling patterns. Statistical testing confirms differences.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Inconsistent data between tables and figures",
       "detail": "Table II shows 'averaged benchmark results with published results for a more moderate view' — mixing the paper's own measurements with externally published numbers without clear separation. Fig 4 caption says 'collected from published benchmark results' while the text implies independent evaluation. It is unclear which numbers are the authors' measurements and which are borrowed from published sources."
     },
     {
       "flag": "Contamination claims without evidence",
       "detail": "Section IV-E claims 'data contamination detection employs methods from Sainz et al. and Magar & Schwartz' but no contamination analysis results are reported anywhere. All ten benchmarks predate the models by years, making contamination a serious concern that is never actually addressed."
     },
     {
       "flag": "No variance or multi-run reporting",
       "detail": "Despite claiming rigorous statistical methodology, the paper appears to report single-run results without variance across seeds. Bootstrap CIs on aggregated scores are not a substitute for multi-run variance assessment."
     },
     {
       "flag": "Qualitative quality ratings presented as objective",
       "detail": "Table IV assigns 'Excellent', 'Good', 'Medium', 'Poor' quality ratings to model responses on a single logic puzzle, but the rating methodology is subjective and the sample size is N=1 task."
     },
     {
       "flag": "Vague reference to 'Project Webpage'",
       "detail": "The abstract says 'evaluation scripts are available at the Project Webpage' but no URL is actually provided in the paper, making reproducibility claims unverifiable."
     },
     {
       "flag": "Author affiliations raise questions",
       "detail": "Most authors share affiliation with 'AI Agent Lab, Vokram Group' — an entity that does not appear to be a well-known research institution. Combined with ByteDance affiliation but no conflict-of-interest statement, this warrants scrutiny."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["J. Kaplan", "S. McCandlish", "T. Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper — this paper's inverse scaling finding directly challenges its predictions."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["J. Hoffmann", "S. Borgeaud"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling approach, relevant to understanding compute-optimal model sizing."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"],
       "year": 2024,
       "relevance": "Challenges emergent abilities narrative; relevant to understanding non-monotonic scaling."
     },
     {
       "title": "Holistic evaluation of language models",
       "authors": ["P. Liang", "R. Bommasani"],
       "year": 2023,
       "relevance": "HELM framework for LLM evaluation methodology — survey-relevant evaluation infrastructure."
     },
     {
       "title": "With little power comes great responsibility",
       "authors": ["D. Card", "P. Henderson"],
       "year": 2020,
       "relevance": "Statistical validity in NLP evaluation — directly relevant to methodology quality assessment."
     },
     {
       "title": "The hitchhiker's guide to testing statistical significance in natural language processing",
       "authors": ["R. Dror", "G. Baumer"],
       "year": 2018,
       "relevance": "Guidelines for statistical testing in NLP; relevant to evaluation methodology rigor."
     },
     {
       "title": "NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark",
       "authors": ["O. Sainz", "J. A. Campos"],
       "year": 2023,
       "arxiv_id": "2310.18018",
       "relevance": "Benchmark contamination detection methodology — directly relevant to data leakage assessment."
     },
     {
       "title": "Inverse scaling can become u-shaped",
       "authors": ["J. Wei", "N. Hou"],
       "year": 2023,
       "arxiv_id": "2211.02011",
       "relevance": "Documents inverse scaling phenomena in LLMs, directly relevant to this paper's main finding."
     },
     {
       "title": "Show your work: Improved reporting of experimental results",
       "authors": ["J. Dodge", "S. Gururangan", "D. Card"],
       "year": 2019,
       "relevance": "Best practices for reporting ML experimental results — methodology quality reference."
     },
     {
       "title": "Improving reproducibility in machine learning research",
       "authors": ["J. Pineau", "P. Vincent-Lamarre"],
       "year": 2021,
       "relevance": "NeurIPS reproducibility program report — directly relevant to our assessment of reproducibility practices."
     },
     {
       "title": "Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity",
       "authors": ["W. Fedus", "B. Zoph", "N. Shazeer"],
       "year": 2022,
       "relevance": "Foundational MoE architecture paper relevant to understanding sparse model scaling."
     }
   ]
 }