ai-research-survey

scan.json (30966B)

---

 {
   "paper": {
     "title": "OJBench: A Competition Level Code Benchmark For Large Language Models",
     "authors": [
       "Zhexu Wang",
       "Yiping Liu",
       "Yejie Wang",
       "Wenyang He",
       "Bofei Gao",
       "Muxi Diao",
       "Yanxu Chen",
       "Kelin Fu",
       "Flood Sung",
       "Zhilin Yang",
       "Tianyu Liu",
       "Weiran Xu"
     ],
     "year": 2025,
     "venue": "arXiv preprint (under review)",
     "arxiv_id": "2506.16395",
     "doi": "10.48550/arXiv.2506.16395"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OJBench is a 232-problem competitive programming benchmark sourced from NOI and ICPC that is substantially harder than LiveCodeBench. Evaluation of 37 models shows the best model (Gemini-2.5-pro-exp-03-25) achieves only 38.91% Pass@1 overall and 9.48% on hard problems. Reasoning-oriented models significantly outperform non-reasoning models, and CPP outperforms Python for most reasoning models on these competition-level tasks. Iterative refinement with execution feedback helps fix compile errors but struggles with time-limit-exceeded errors requiring more efficient algorithms.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "A GitHub URL (https://github.com/He-Ren/OJBench) is listed, but footnote 4 states 'We are finalizing our code and website. The article will be updated once everything is ready.' This is a promise of future release, not a currently available artifact."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark dataset of 232 problems with test cases is described but not yet released. The paper states the code and website are being finalized."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E mentions 'two computing clusters equipped with 8 NVIDIA A100-80GB GPUs' and use of VLLM, but no requirements.txt, Dockerfile, or detailed dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The code and evaluation framework are stated to be in preparation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 2 are point estimates (e.g., '33.30' Pass@1 for o4-mini). No confidence intervals or error bars are reported anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'reasoning-oriented models significantly outperformed non-reasoning-oriented models') based solely on comparing raw numbers in Table 2 without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context, e.g., 'Qwen3-235B-A22B and DeepSeek-R1 exhibit a pass@8 improvement of 14.55 and 11.05 in python, respectively, compared to pass@1' (Section 3.2). Table 3 compares OJBench vs LiveCodeBench scores for the same models."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 232 problems (36 Easy, 79 Medium, 117 Hard) but no justification is given for why this number is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Pass@1 is computed from 8 candidate solutions per problem, but no variance, standard deviation, or confidence intervals across runs are reported. All results are single point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 37 models spanning non-reasoning open-source (CodeLlama, DeepSeek-Coder, Qwen2.5-Coder), reasoning open-source (DeepSeek-R1, QWQ, Qwen3), and closed-source (GPT series, Claude, Gemini, o-series) categories, providing extensive cross-model comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes state-of-the-art models released in 2025 such as o4-mini, Gemini-2.5-pro-exp-03-25, Qwen3-235B-A22B, and DeepSeek-R1."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "OJBench is a benchmark dataset, not a multi-component system. There are no components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Pass@1 and Pass@8 for both Python and CPP, plus per-difficulty-level breakdowns (Easy, Medium, Hard pass rates). Table 2 presents all these metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via test case pass/fail. No human evaluation of model outputs is conducted. Manual verification was performed only on the GPT-4o translations of problem descriptions, not on model solutions."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "OJBench serves entirely as a test set. Models are evaluated as-is without any fine-tuning or development on the benchmark problems."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides breakdowns by difficulty level (Easy, Medium, Hard). Results are also categorized by model type (reasoning vs non-reasoning, open vs closed-source) and programming language (Python vs CPP)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 analyzes error types (CE, TLE, WA, MLE, RTE) and their distribution. Section 4.4 and Figure 5 provide a detailed case analysis of Qwen3-235B-A22B's reasoning process on a failed problem, showing repetitive restatements and incorrect strategy selection."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that models struggle to resolve TLE errors through refinement (Section 4.3), that almost all non-reasoning models score 0% on hard problems (Section 3.2), and that the Qwen3 distilled models perform worse in CPP than Python counter to the general trend (Section 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'even state-of-the-art reasoning-oriented models, such as o4-mini and Gemini-2.5-pro-exp, struggle with highly challenging competition-level problems.' Table 2 confirms this: the best model achieves only 38.91% Pass@1 overall and 9.48% on hard problems."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims without adequate study design. Section 3.2 states 'reinforcement learning and distillation from powerful reasoning-oriented models hold significant potential for improving models' code reasoning capabilities,' attributing performance differences to training methodology without controlling for confounds like model size, training data volume, or compute budget."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title specifies 'Competition Level' and the paper explicitly bounds scope in Appendix A: 'The problem types of OJBench are mainly focused on the field of algorithm competitions' and acknowledges insufficient coverage of IoT, blockchain, and AI-code application scenarios."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers single explanations for findings without considering alternatives. For example, CPP's advantage is attributed to 'CPP is inherently a high-performance programming language' (Section 4.2) without considering training data composition, tokenization differences, or prompt formatting effects."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Pass@n on 232 NOI/ICPC algorithmic competition problems but broadly frames this as assessing 'code reasoning abilities' and 'competitive-level code reasoning abilities of LLMs' throughout. Competitive programming is a narrow subset of code reasoning; this proxy gap is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models have version dates (Claude-3.5-sonnet-20241022, GPT-4o-20241120, o1-20241217, Gemini-2.5-pro-exp-03-25) but several lack specific versions: 'GPT3.5-Turbo', 'GPT4-Turbo', and 'o4-mini' are listed without snapshot dates or API versions. Open-source models are specified by name and size."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts or system instructions are shown anywhere in the paper or appendix. Section 3.1 describes the evaluation setup but does not include the actual prompt text given to models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 states 'we assess the models using the hyper parameters officially recommended by each model, including temperature, top_k, and top_p' and 'max_tokens to 64k' for reasoning models, but the actual per-model hyperparameter values are not listed."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The main evaluation involves direct code generation without agentic scaffolding. The refinement experiment (Section 4.3) uses simple error feedback but is not an agentic scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 and Figure 1 document the pipeline: collection from Logu and ICPC official website, validation with contestant code submissions, filtering out special-judge problems, GPT-4o translation of NOI problems with manual verification. Table 1 provides final statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A contains a dedicated 'Limitations and Broader Impact' section with substantive discussion of two specific limitations: limited coverage and insufficient diversity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A identifies specific threats: '(1) Limited coverage: The data of OJBench mainly comes from open-source programming competition platforms' and '(2) Insufficient diversity: The problem types of OJBench are mainly focused on the field of algorithm competitions' with specific examples of uncovered areas."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A explicitly states 'our evaluation cannot cover the entire scope of programming competitions' and lists specific excluded scenarios: 'development combining artificial intelligence and code, Internet of Things programming, and blockchain smart contract development.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark problems and test cases are not yet publicly available. The paper states the code and website are being finalized."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes data sources (Logu platform for NOI, ICPC official website), what was collected (problem descriptions in Markdown, comprehensive test cases from competition organizers), and the filtering process (validation with contestant code, removal of special-judge problems)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data consists of programming competition problems sourced from well-known competition platforms (NOI, ICPC)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the full pipeline. Section 2.1 documents each step: collection from two sources, validation against contestant solutions, filtering special-judge problems, translation with GPT-4o, and manual verification. Final count of 232 problems is given in Table 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. No grants, sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Beijing University of Posts and Telecommunications, Tsinghua University, University of Chinese Academy of Sciences, Peking University, and Moonshot AI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Some authors are affiliated with Moonshot AI (an AI company that develops LLMs), though Moonshot AI models are not directly evaluated in the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 37 evaluated models. This is critical since NOI and ICPC problems are publicly available and could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions selecting NOI/ICPC 'to minimize overlap with existing benchmarks' (Section 2.1) but does not discuss whether these publicly available competition problems appear in model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NOI and ICPC problems are publicly available on competition platforms (Logu, ICPC official website). Models trained on internet-scale data could have seen these problems and their solutions. No contamination analysis or mitigation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLMs on programming competition problems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The ethical statement (Appendix B) addresses data collection ethics and crowd worker compensation but no human subjects were studied."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E mentions hardware ('8 NVIDIA A100-80GB GPUs') and time ('about two hours' per model) for open-source models under 72B parameters, but no API costs for closed-source models are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Partial compute information is given in Appendix E for open-source models only. No total compute budget across all 37 models is stated, and closed-source API costs are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Eight candidate solutions are generated per problem but no seed sensitivity analysis is performed. Results are not reported across multiple random seeds or independent runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'we use API or VLLM to generate eight candidate solutions for each problem' for computing Pass@n metrics."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'we assess the models using the hyper parameters officially recommended by each model,' which implicitly indicates no hyperparameter search was conducted — defaults were used for all models."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses officially recommended hyperparameters for each model (Section 3.1), which is a principled and justified approach for a benchmark evaluation paper."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed in the paper, so correction for multiple comparisons is moot. The absence of tests is captured by the statistical_methodology category."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors are not evaluating their own model or system. They evaluate third-party models on their benchmark. Self-comparison bias is not applicable."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models with vastly different compute requirements (1.5B to 671B parameters, open-source vs API) are compared without any discussion of performance relative to compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to measure 'code reasoning abilities' but only tests algorithmic competition problems. No discussion of whether competitive programming success equates to general code reasoning capability. The paper compares difficulty against LiveCodeBench (Table 3) but does not question construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The main evaluation involves direct code generation without scaffolding. Models generate solutions directly from problem descriptions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NOI and ICPC problems were created for past competitions and published on public platforms. Models trained on internet data could have seen these problems and their solutions. No temporal leakage analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the problem descriptions or input-output examples provide information that could leak answer structure. The use of GPT-4o for translating NOI problems introduces a potential channel of familiarity for GPT-family models."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether model training data contains NOI/ICPC problem solutions, related competition problems, or editorial discussions that could create non-independence between train and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are used. No canary strings, membership inference tests, n-gram overlap analysis, or temporal splits are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "OJBench is substantially harder than LiveCodeBench for state-of-the-art models.",
       "evidence": "Table 3 compares the same models on both benchmarks: o4-mini scores 63.7% on LCB vs 33.3% on OJBench; Gemini-2.5-pro scores 65.9% on LCB vs 38.91% on OJBench; Qwen3-235B scores 56.6% on LCB vs 25.97% on OJBench (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Reasoning-oriented models significantly outperform non-reasoning-oriented models on competitive coding tasks.",
       "evidence": "Table 2 shows clear separation: best non-reasoning model (DeepSeek-V3-0324) achieves 25.54% Pass@1 in Python, while multiple reasoning models exceed 25% (DeepSeek-R1 at 26.02%, Qwen3-235B at 25.97%, o4-mini at 33.30%, Gemini-2.5-pro at 38.91%).",
       "supported": "strong"
     },
     {
       "claim": "For most reasoning-oriented models, CPP outperforms Python on competition-level problems.",
       "evidence": "Table 2 and Figure 3 show CPP advantages for o4-mini (46.12% vs 33.30% Pass@1), o1 (33.24% vs 26.45%), Gemini-2.5-pro (44.26% vs 38.91%). However, DeepSeek-R1-Distill-Qwen models show the opposite pattern (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "Models can improve pass rates through iterative refinement with execution feedback, but struggle with time-limit-exceeded errors.",
       "evidence": "Figure 4 shows CE errors decrease significantly through refinement rounds, but TLE errors persist. The paper explains TLE requires designing more efficient algorithms, which current models struggle with (Section 4.3).",
       "supported": "moderate"
     },
     {
       "claim": "Even the most powerful reasoning-oriented models achieve only single-digit pass rates on hard competition problems.",
       "evidence": "Table 2 shows Hard pass rates: Gemini-2.5-pro 9.48%, o3-mini 6.84%, o4-mini 5.77%, Qwen3-235B 4.27%, DeepSeek-R1 3.53%. All non-reasoning models score below 4%.",
       "supported": "strong"
     },
     {
       "claim": "Using a limited number of test cases for evaluation produces false-positive results.",
       "evidence": "Figure 2 shows performance of all models decreases with increasing number of test cases, demonstrating that partial test suites inflate accuracy numbers. The paper uses the full test suite (avg 31.81 tests per problem) to avoid this (Section 2.3).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "NOI and ICPC problems are publicly available on competition platforms (Logu, ICPC official website) with solutions widely shared online. Models trained on internet-scale data could have memorized these problems. No contamination detection, temporal analysis, or mitigation is performed despite this being a benchmark evaluation paper."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims across 37 models are based on raw number comparisons without any statistical tests, confidence intervals, or uncertainty quantification. Claims like 'significantly outperformed' are not backed by statistical significance."
     },
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "Results are reported as single point estimates from 8 samples per problem. No standard deviations, confidence intervals, or measures of result stability across runs are provided."
     },
     {
       "flag": "GPT-4o used for problem translation",
       "detail": "NOI problems were translated from Chinese to English using GPT-4o (Section 2.1). This could introduce a subtle bias favoring GPT-family models, since they may find GPT-generated English text more natural to parse. The paper does not discuss this potential confound."
     },
     {
       "flag": "Benchmark and code not yet available",
       "detail": "Footnote 4 states 'We are finalizing our code and website.' The benchmark cannot be independently verified or reproduced until release."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and the Pass@k metric used as the primary evaluation metric in this paper."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Key comparison benchmark; paper argues OJBench is substantially harder than LiveCodeBench for evaluating code reasoning."
     },
     {
       "title": "CodeELO: Benchmarking competition-level code generation of LLMs with human-comparable Elo ratings",
       "authors": ["Shanghaoran Quan", "Jiaxi Yang", "Bowen Yu"],
       "year": 2025,
       "arxiv_id": "2501.01257",
       "relevance": "Closely related competition-level code benchmark using CodeForces and Elo ratings."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Key reasoning-oriented model evaluated in the benchmark; demonstrates RL-based reasoning training for code."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Foundational work on LLM-based competition-level code generation using large-scale sampling."
     },
     {
       "title": "Can language models solve olympiad programming?",
       "authors": ["Quan Shi", "Michael Tang", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "arxiv_id": "2404.10952",
       "relevance": "USACO benchmark for olympiad-level programming evaluation of LLMs."
     },
     {
       "title": "Probench: Benchmarking large language models in competitive programming",
       "authors": ["Lei Yang", "Renren Jin", "Ling Shi"],
       "year": 2025,
       "arxiv_id": "2502.20868",
       "relevance": "Closely related competitive programming benchmark; discusses false-positive risk from limited test cases."
     },
     {
       "title": "Qwen2.5-Coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for the Qwen2.5-Coder model family evaluated in the benchmark."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Early code benchmark for evaluating LLMs on programming challenges of varying difficulty."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the barrier of closed-source models in code intelligence",
       "authors": ["Qihao Zhu", "Daya Guo", "Zhihong Shao"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "Code-specialized LLM evaluated in the benchmark; represents strong open-source code models."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for program synthesis evaluation."
     },
     {
       "title": "TACO: Topics in algorithmic code generation dataset",
       "authors": ["Rongao Li", "Jie Fu", "Bo-Wen Zhang"],
       "year": 2023,
       "arxiv_id": "2312.14852",
       "relevance": "Algorithmic code generation benchmark related to competition-level evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Useful as a benchmark for researchers evaluating code reasoning models, but not directly applicable for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expected finding that current models struggle on hard competition problems; the CPP vs Python finding adds some novelty."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk implications; purely evaluates code generation capability on competitive programming."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Provides a leaderboard ranking 37 models, which generates some competitive interest, but no controversial claims."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Code and website are explicitly stated to be 'being finalized' and not yet available for use."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates high-profile models from OpenAI (o4-mini), Google (Gemini), Anthropic (Claude), and DeepSeek, though the authoring institutions are less prominent internationally."
     }
   }
 }