ai-research-survey

scan.json (26229B)

---

 {
   "paper": {
     "title": "CodeContests+: High-Quality Test Case Generation for Competitive Programming",
     "authors": ["Zihan Wang", "Siyao Liu", "Yang Sun", "Hongyan Li", "Kai Shen"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2506.05817",
     "doi": "10.48550/arXiv.2506.05817"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper open-sources SandboxFusion (https://github.com/bytedance/SandboxFusion) and publishes the dataset on HuggingFace (https://huggingface.co/datasets/ByteDance-Seed/Code-Contests-Plus). Both URLs are provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset CodeContests+ is released on HuggingFace at https://huggingface.co/datasets/ByteDance-Seed/Code-Contests-Plus, including pre-processed test cases in 1x-5x versions, generators, validators, and checkers."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions using C++ with testlib and SandboxFusion supporting 20+ languages but does not specify library versions or environment details needed for reproduction."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the dataset and SandboxFusion tool are released, the paper does not include a README with commands to replicate the main experiments (e.g., RL training, large-scale evaluation)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Table 3 reports Pass@1 on LiveCodeBench as point estimates (e.g., 0.622 vs 0.637) without any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper claims CodeContests+ achieves 'consistent gains' and 'significantly higher accuracy' over CodeContests based solely on comparing raw numbers in Table 3 and Figure 3 without any formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are contextualized. For example, the paper reports that CodeContests+ 5x yields 'almost twice' the number of qualified problems compared to CodeContests, and Table 3 shows absolute Pass@1 values with baselines (e.g., 0.622 vs 0.637), providing magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes used. The evaluation uses 100 positive and 100 negative samples per problem (1.72M total submissions), but no justification is provided for why 100 per class is sufficient for TPR/TNR estimation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across runs. The RL training results in Table 3 appear to be from a single run. The paper uses 'avg@15' (average of pass@1 from 15 independent responses) as a metric, but does not report variance across training runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares CodeContests+ against the original CodeContests dataset throughout Section 4.4 and 4.5, using identical problem sets. Table 1 also compares against MBPP, HumanEval, USACO, LiveCodeBench, APPS, and TACO."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeContests (2022) is the direct predecessor and the most relevant baseline. LiveCodeBench (2024) is used as the evaluation benchmark for RL training. These are contemporary and appropriate baselines for the task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates test case quantity via the 1x-5x versions (Section 4.4, Figure 3c), showing that increasing test case count improves evaluation accuracy. The comparison of CodeContests+ vs CodeContests+HQ (TPR&TNR >= 0.9 filtered) in the RL experiment also serves as an ablation on test case quality filtering."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: True Positive Rate (TPR), True Negative Rate (TNR) for test case quality evaluation, and Pass@1 across Easy/Medium/Hard difficulty levels for RL training evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated test cases, validators, or checkers is reported. The paper relies entirely on automated metrics (TPR/TNR via submission labels, Pass@1 on LiveCodeBench). Given claims about test case 'quality' and 'correctness,' human expert review of a sample would strengthen the evidence."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench (Aug 2024 - Feb 2025) serves as a held-out evaluation benchmark for the RL training experiment. The test case quality verification uses independently labeled submissions from CodeForces."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-difficulty breakdowns (Easy, Medium, Hard) for LiveCodeBench results. Figure 3 shows TPR/TNR distributions across all problems. The 1x-5x versions provide a breakdown by test case quantity."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 discusses validator limitations including two error categories. Section 4.4 notes that 67.1% of CodeContests test cases fail validation. Case studies in Appendix D.3 show specific failure examples. The paper acknowledges lacking 'an automatic supervision method' for the second type of validator error."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the validator agent still has unresolved error modes (Section 3.2: 'we still lack an automatic supervision method to address the second type of error'), and that 1,920 problems were removed during data cleaning (Section 4.1), acknowledging limitations of the approach."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that CodeContests+ achieves 'significantly higher accuracy than CodeContests, particularly with a notably higher True Positive Rate' and that 'improvements in test case quality yield considerable advantages for RL.' Both are supported by Figure 3 and Table 3 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that higher-quality test cases improve RL training outcomes. The controlled experiment in Section 4.5 uses the same model (Qwen2.5-32B), same training procedure (DAPO), and same problem set, changing only the test cases. This single-variable manipulation provides adequate causal evidence for the claim, though only one training run is shown."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title and conclusions speak broadly about 'competitive programming' and 'further enhancing the reasoning and coding capabilities of LLMs,' but results are only on one dataset (CodeContests/CodeForces), one model (Qwen2.5-32B), and one RL algorithm (DAPO). No explicit scoping of generalization boundaries is provided."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed improvements. For example, the RL improvement could be partly due to the data cleaning (removal of 1,920 problems) rather than test case quality alone. The paper does not consider confounds between data cleaning and test case improvement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not specify which LLM is used for the Generator, Validator, or Checker agents. Section 3 describes the agent system architecture but never names the model or version powering these agents. The RL training uses 'Qwen2.5-32B' which is a model name but not a specific version/checkpoint."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The prompts used for the Generator, Validator, and Checker agents are not provided. The paper describes the agent workflows in natural language (e.g., 'the Generator Agent is given a problem statement. It is then instructed to read the statement carefully') but does not provide the actual prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports key hyperparameters: clipping ratios epsilon_low=0.2 and epsilon_high=0.28 for DAPO, the use of avg@15 metric, and the LiveCodeBench time window (Aug 2024 - Feb 2025). However, LLM agent temperature/sampling settings are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The Generator-Validator agent system is described in detail in Section 3. The Generator agent workflow (problem analysis, constraint identification, corner case design, program generation), the Validator supervision loop (feedback on validation failures, compilation errors, timeouts), and the Checker agent workflow are all described. Figure 2 provides a visual overview."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the data cleaning procedure with specific filtering criteria: problems without statements, interactive problems, problems without correct submissions, file I/O problems, special problems, image-dependent problems, crawling errors, and low-quality problems. The count is reduced from 13,610 to 11,690."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section is present. The paper has a 'Conclusion and Future Work' section (Section 5) that mentions future directions but does not substantively discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. While Section 3.2 acknowledges the validator cannot catch all errors, there is no discussion of broader threats such as the generalizability of results beyond CodeForces problems, the single RL run, or potential biases in the submission labels used for evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what types of competitive programming problems the approach may not work for, or which LLMs/RL algorithms the benefits might not transfer to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace with generators, validators, checkers, and pre-processed test cases. TPR/TNR values per problem are provided. The original CodeContests data is also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data collection process: starting from CodeContests (which collects problems from CodeForces), applying data cleaning (Section 4.1), generating test cases via the G-V agent system (Section 4.2), and sampling 100 positive/100 negative submissions per problem for evaluation (Section 4.4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from publicly available competitive programming problems and automated contestant submissions on CodeForces."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw CodeContests data to CodeContests+ is documented: cleaning (13,610 to 11,690 problems), G-V agent system generation, validation (100% pass rate vs 67.1% for original), and filtering for high-quality subset (TPR&TNR >= 0.9 for CodeContests+HQ). Counts at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement is present. The Acknowledgements section thanks ByteDance colleagues but does not disclose specific funding sources or grants."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: ByteDance Seed and Peking University. The first author has dual affiliation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of five authors are affiliated with ByteDance Seed, and the dataset is released under the ByteDance brand. ByteDance has a commercial interest in LLM training data and infrastructure. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. ByteDance employees evaluating and releasing a ByteDance-branded dataset represents a potential conflict that is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training cutoff for Qwen2.5-32B (used for RL) is not stated. The LLM used for the Generator/Validator/Checker agents is not even named, let alone its training cutoff. LiveCodeBench is used with problems from Aug 2024-Feb 2025, but whether the base model was trained before this window is not explicitly stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Qwen2.5-32B or the agent LLM may have seen CodeForces problems or LiveCodeBench problems during pre-training. Since CodeForces problems are publicly available and widely used in training corpora, this is a relevant concern."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LiveCodeBench is designed to mitigate contamination via temporal windowing, and the paper uses problems from Aug 2024-Feb 2025, but this contamination mitigation is not discussed by the authors. No explicit mention of contamination risk for any benchmark used."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs are reported for the LLM agent system. The paper mentions a cluster with 25,000 CPU cores and 70 TB memory for evaluation (Section 4.4), but does not report the cost of running the Generator-Validator agents or the total API/compute spend."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 describes the evaluation infrastructure: 'running on a cluster with 25,000 CPU cores and 70 TB memory' for 300+ million program executions. Appendix C provides further details about the cloud architecture (8,000 cores for judging pods, 17,000 for execution pods)."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeContests+ achieves significantly higher accuracy than CodeContests in classifying correct/incorrect solutions, particularly with a notably higher True Positive Rate.",
       "evidence": "Figure 3 shows that CodeContests has over 4000 problems with TPR <= 0.1, while CodeContests+ 5x has far fewer such problems. Figure 3c shows CodeContests+ 5x has almost twice the number of qualified problems (TPR&TNR >= threshold) compared to CodeContests at various thresholds.",
       "supported": "strong"
     },
     {
       "claim": "Only 67.1% of test cases in the original CodeContests dataset pass validation.",
       "evidence": "Section 4.2: 'We used validators to check the correctness of all 1.18 million generated test cases in CodeContests. Only 0.79 million of these passed validation, which is 67.1%.'",
       "supported": "strong"
     },
     {
       "claim": "Even with one-quarter of the test cases, CodeContests+ 1x has significantly higher evaluation accuracy and yields over 80% more qualified problems than CodeContests.",
       "evidence": "Section 4.4 and Figure 3c show that CodeContests+ 1x (avg 25 tests per problem) outperforms CodeContests (avg 101 tests per problem) in number of qualified problems at various thresholds.",
       "supported": "strong"
     },
     {
       "claim": "Higher-quality test cases yield consistent improvements in RL training across all difficulty levels.",
       "evidence": "Table 3 shows CodeContests+HQ improves over CodeContests: Easy 0.965 vs 0.958, Medium 0.812 vs 0.786, Hard 0.340 vs 0.329, All 0.637 vs 0.622 on LiveCodeBench Pass@1.",
       "supported": "moderate"
     },
     {
       "claim": "The Generator-Validator agent system is the first LLM agent system designed for constructing high-quality test cases for competitive programming problems.",
       "evidence": "Section 1, contribution 1. Prior work described in Section 2 covers manual, mutation-based, and direct LLM output methods, none of which use a generator-validator agent architecture.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces a Generator-Validator (G-V) LLM agent system that generates validated test cases for competitive programming problems, creating the CodeContests+ dataset. The system dramatically improves test case quality: 100% of generated test cases pass validation compared to only 67.1% in the original CodeContests. Large-scale evaluation with 1.72 million submissions shows CodeContests+ yields nearly twice the number of high-quality problems (TPR&TNR >= 0.9) compared to CodeContests. RL training with CodeContests+HQ on Qwen2.5-32B shows consistent improvements on LiveCodeBench, with overall Pass@1 increasing from 0.622 to 0.637.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Three of five authors are from ByteDance Seed, and the released dataset carries the ByteDance brand. While the dataset improvements over CodeContests appear genuine (backed by 1.72M submission evaluation), no external validation or independent evaluation is reported."
     },
     {
       "flag": "Single RL training run",
       "detail": "The RL training comparison (Table 3) appears to be from a single training run with no variance or confidence intervals reported. The improvements (e.g., 0.622 to 0.637 overall) are small enough that they could plausibly be within run-to-run variance."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Claims of 'significantly higher accuracy' and 'consistent gains' are made based on raw number comparisons without any formal significance tests, especially concerning for the modest RL training improvements."
     },
     {
       "flag": "Agent model not disclosed",
       "detail": "The LLM used for the Generator, Validator, and Checker agents is never identified. This is a critical reproducibility gap since the quality of the generated test cases depends entirely on the agent model's capabilities."
     },
     {
       "flag": "Confound between data cleaning and test case quality",
       "detail": "CodeContests+ uses 11,690 problems (after cleaning) while CodeContests has 13,610. Although the paper states it uses the same subset for comparison experiments, the RL training with CodeContests+HQ further filters problems by TPR/TNR thresholds, potentially confounding test case quality with problem selection."
     }
   ],
   "cited_papers": [
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces the MBPP benchmark for code generation, which CodeContests+ compares against in Table 1 for test case construction methodology."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation; foundational work for evaluating LLM coding capabilities."
     },
     {
       "title": "Competitive programming with large reasoning models",
       "authors": ["Ahmed El-Kishky", "Alexander Wei"],
       "year": 2025,
       "arxiv_id": "2502.06807",
       "relevance": "Demonstrates reasoning models on competitive programming tasks, directly relevant to evaluating coding capabilities of LLMs."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major RL-based reasoning model trained on code data; CodeContests+ is designed to improve the data foundation for such training."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Introduces the original CodeContests dataset which this paper improves upon; landmark work in competitive programming code generation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "Identifies high False Positive Rates in MBPP and HumanEval due to insufficient test cases; directly motivates the test case quality problem addressed by CodeContests+."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2025,
       "relevance": "Used as the held-out evaluation benchmark for the RL training experiments; designed to mitigate contamination in code evaluation."
     },
     {
       "title": "TACO: topics in algorithmic code generation dataset",
       "authors": ["Rongao Li", "Jie Fu"],
       "year": 2023,
       "arxiv_id": "2312.14852",
       "relevance": "Competitive programming dataset that uses LLMs to directly output test cases; compared against in Table 1 as a baseline approach."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "relevance": "Large-scale coding challenge dataset compared in Table 1; relevant to benchmarking code generation capabilities."
     },
     {
       "title": "DAPO: An open-source LLM reinforcement learning system at scale",
       "authors": ["Qiying Yu", "Zheng Zhang"],
       "year": 2025,
       "arxiv_id": "2503.14476",
       "relevance": "The RL training algorithm used in the paper's experiments; an open-source RL system for training code LLMs."
     },
     {
       "title": "Qwen2.5 technical report",
       "authors": ["An Yang", "Baosong Yang"],
       "year": 2025,
       "arxiv_id": "2412.15115",
       "relevance": "The base model (Qwen2.5-32B) used for RL training experiments in the paper."
     }
   ]
 }