ai-research-survey

scan.json (19762B)

---

 {
   "paper": {
     "title": "Competitive Programming with Large Reasoning Models",
     "authors": ["OpenAI"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.06807"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository URL is provided in the paper. The paper includes sample IOI solutions in Appendix C but no reproducible codebase."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The CodeForces problems and IOI problems are publicly available, but the paper does not release its specific test set, evaluation scripts, or any curated dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details for reproduction are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The models (o1, o1-ioi, o3) are proprietary and not publicly available."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (ratings, percentiles, scores). No confidence intervals or error bars are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims o3 outperforms o1-ioi and other models but provides no statistical significance tests. Comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context, e.g., 'rating increase from 2214 (98th percentile) to 2724 (99.8th percentile)' and specific point differences (e.g., 213 vs 362.14 vs 395.64 on IOI)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set uses 12 Division 1 contests but no justification is given for why this number is sufficient. One contest was excluded for convenience."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench results are 'averaged over 3 trials' but no standard deviation or variance is reported. CodeForces and IOI results appear to be single-run evaluations with pass@10 but no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares gpt-4o, o1-preview, o1, o1-ioi, and o3 across benchmarks, providing a progression of baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "All baselines are OpenAI's own models. No comparison with external systems like DeepSeek-R1, Kimi k1.5, or AlphaCode2 on the same benchmarks despite mentioning them in the introduction."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper effectively ablates the test-time strategy by comparing o1-ioi with random submissions vs. hand-crafted strategy vs. relaxed limits (Figure 4), and compares o3's simpler strategy against o1-ioi's complex one."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: CodeForces rating/percentile, IOI score, pass@1, pass@10, average score on HackerRank Astra, and SWE-bench verified percentage."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "All evaluation is through automated competitive programming judges and test suites. Human evaluation is not relevant for this type of benchmark evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 states Division 1 contests from late 2023 and 2024 were used, 'all of which occurred after the o3 training set data cut-off.' A separate Division 2 validation set was used during training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 in Appendix B provides detailed per-problem breakdowns across all 12 contests with pass@1, pass@10, and ranking information."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes that 'the very best humans still solve more problems than AI' but does not analyze specific failure cases or why certain problems remain unsolved."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement over the previous model. No failed approaches, configurations that didn't work, or negative results are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about o1-ioi placing 49th percentile at IOI, achieving gold under relaxed constraints, and o3 achieving gold without hand-crafted strategies are all supported by the results in Sections 3.4 and 4.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims that 'reinforcement learning... significantly boosts performance' and 'scaling general-purpose reinforcement learning... offers a robust path.' However, multiple variables change between models (architecture, data, RL compute, training data) without controlled isolation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and conclusion claim results about 'reasoning domains' broadly, but evaluation is limited to competitive programming (CodeForces, IOI) and two software engineering benchmarks. The conclusion states results 'will unlock many new use cases for AI in science, coding, math, and many other fields.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The improvements could be due to larger training data, more compute, architecture changes, or other factors besides RL, but these are not addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to as 'o1', 'o1-ioi', 'o3', 'gpt-4o', and 'o1-preview' without specific version identifiers, snapshot dates, or API versions. o3 is described as 'early checkpoints' without further specification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 mentions prompts include 'the problem description, constraints, and any available sample test cases' but the actual prompt text or template is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — no temperature, top-p, max tokens, or sampling parameters. The number of samples (10K for o1-ioi, 1K for o3) is stated but inference parameters are not."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The o1-ioi test-time strategy is described in detail in Section 3.2: subtask decomposition, 10K sampling, clustering via model-generated test inputs, reranking with learned scoring function, and round-robin submission. o3's simpler approach is also described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 describes the test set construction: Division 1 contests from late 2023 and 2024, post training cutoff, one contest excluded for interactive problem, validation set of Division 2 problems used during training."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations or threats-to-validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show or what settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (model outputs, individual trial results, generated solutions beyond Appendix C samples) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 describes the test set selection: Division 1 contests from late 2023 and 2024, post-cutoff, with contamination checks via embedding search."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data sources are standard competitive programming benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B describes the evaluation pipeline: problem selection, grading methodology (full test suite, matching official CodeForces grading), rating calculation methodology, and thinking time adjustments."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The paper is authored by 'OpenAI' but no specific funding or grant information is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clearly authored by OpenAI, evaluating OpenAI's own models. The affiliation is stated in the author line."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI has a direct financial interest in demonstrating the capabilities of o1 and o3. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 states all test contests 'occurred after the o3 training set data cut-off.' Section 4.2 states 'IOI 2024 occurred after the training cut-off for this model.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1: 'we used embedding search to confirm that the test problems have not been seen by the model during training.' Section 4.2: 'we additionally confirmed with search that the IOI test problems are not contaminated with the training set.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is addressed via temporal separation (post-cutoff contests) and embedding-based search verification. Section 2.1 also mentions 'a contamination check as a sanity measure.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or latency is reported despite sampling 10,000 solutions per problem for o1-ioi and 1,024 for o3."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated for training or inference. The paper mentions 'significantly greater compute resources' for o3 without quantifying."
       }
     }
   },
   "claims": [
     {
       "claim": "o3 achieves a CodeForces rating of 2724 (99.8th percentile), placing it among the top 200 active competitors worldwide.",
       "evidence": "Figure 5 and Section 4.1 report this rating. Figure 10 shows o3 vs top active competitors. Appendix B provides per-problem breakdown.",
       "supported": "moderate"
     },
     {
       "claim": "o3 achieves a gold medal on IOI 2024 (395.64 points) under the 50-submission limit without hand-crafted test-time strategies.",
       "evidence": "Figure 7 and Section 4.2 report this result. Gold threshold was approximately 360 points.",
       "supported": "moderate"
     },
     {
       "claim": "Scaling general-purpose reinforcement learning, rather than domain-specific techniques, offers a robust path toward state-of-the-art AI in reasoning domains.",
       "evidence": "Comparison of o1-ioi (domain-specific) vs o3 (general-purpose) across CodeForces and IOI benchmarks in Sections 3-4.",
       "supported": "weak"
     },
     {
       "claim": "o3 achieves 71.7% on SWE-bench Verified, a 22.8% improvement over o1.",
       "evidence": "Figure 9 shows these results, referenced from o1 system card and early o3 checkpoint results.",
       "supported": "moderate"
     },
     {
       "claim": "o1-ioi's hand-crafted test-time selection strategy contributed nearly 60 additional points over random submission at IOI 2024.",
       "evidence": "Figure 4 shows 213 points with hand-crafted strategy vs 156 points with random 50 submissions.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OpenAI's o3 reasoning model achieves a CodeForces rating of 2724 (99.8th percentile) and an IOI 2024 gold medal score of 395.64 under standard competition constraints, surpassing the domain-specific o1-ioi system without hand-crafted test-time strategies. The paper demonstrates a progression from gpt-4o (11th percentile on CodeForces) through o1-preview, o1, and o1-ioi to o3, attributing gains to reinforcement learning at scale. o3 also achieves 71.7% on SWE-bench Verified, suggesting reasoning improvements transfer beyond competitive programming.",
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "OpenAI is evaluating its own proprietary models (o1, o1-ioi, o3) with no independent verification. All baselines are also OpenAI models — no comparison with competitors like DeepSeek-R1 or AlphaCode2 on the same benchmarks."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations section, no threats to validity, and no discussion of what the results do not show. This is unusual for a paper making broad claims about reasoning capabilities."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All results are point estimates with no confidence intervals, error bars, significance tests, or variance reporting. SWE-bench is averaged over 3 trials with no spread reported."
     },
     {
       "flag": "Unreproducible",
       "detail": "The models are proprietary and not publicly available. No code, prompts, hyperparameters, or environment details are provided. Results cannot be independently verified or reproduced."
     },
     {
       "flag": "Overclaimed generalization",
       "detail": "The conclusion claims results 'will unlock many new use cases for AI in science, coding, math, and many other fields' based on competitive programming and two software engineering benchmarks."
     },
     {
       "flag": "Confounded causal claims",
       "detail": "The paper attributes improvements to RL scaling, but multiple variables change between models (architecture, training data volume, compute budget, data quality) without controlled isolation."
     }
   ],
   "cited_papers": [
     {
       "title": "Competition-Level Code Generation with AlphaCode",
       "authors": ["Li et al."],
       "year": 2022,
       "relevance": "Foundational work on AI competitive programming using large-scale sampling and test-time heuristics."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Chen et al."],
       "year": 2021,
       "relevance": "Introduced Codex and the HumanEval benchmark for code generation evaluation."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "relevance": "Independent demonstration of RL-based chain-of-thought improving programming and math performance."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Jimenez et al."],
       "year": 2024,
       "relevance": "Benchmark for evaluating AI on real-world software engineering tasks, used in this paper's evaluation."
     },
     {
       "title": "AlphaCode 2 Technical Report",
       "authors": ["Google DeepMind"],
       "year": 2023,
       "relevance": "Prior state-of-the-art in competitive programming AI, achieving 85th percentile on CodeForces."
     },
     {
       "title": "Learning to Reason with LLMs",
       "authors": ["OpenAI"],
       "year": 2024,
       "relevance": "Technical report on o1 reasoning model, foundational to this paper's systems."
     },
     {
       "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs",
       "authors": ["Kimi Team"],
       "year": 2025,
       "relevance": "Independent work on RL-based reasoning for mathematical and programming challenges."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Austin et al."],
       "year": 2021,
       "relevance": "Early work showing LLM code generation scaling with model size, foundational to the field."
     }
   ]
 }