ai-research-survey

scan.json (35047B)

---

 {
   "paper": {
     "title": "HardTests: Synthesizing High-Quality Test Cases for LLM Coding",
     "authors": [
       "Zhongmou He",
       "Yee Man Choi",
       "Kexun Zhang",
       "Jiabao Ji",
       "Junting Zhou",
       "Dejia Xu",
       "Ivan Bercovich",
       "Aidan Zhang",
       "Lei Li"
     ],
     "year": 2025,
     "venue": "arXiv (Preprint, Under review)",
     "arxiv_id": "2505.24098",
     "doi": "10.48550/arXiv.2505.24098"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HardTestGen, an LLM-based test synthesis pipeline using three input types (direct, regular, hacking) and oracle program verification, produces test cases with 11.3pp higher precision and 17.5pp higher recall than existing datasets (TACO, CodeContests) for evaluating LLM-generated code on competitive programming. The precision advantage reaches ~40pp on harder problems. Post-training experiments show test quality matters substantially for reinforcement learning and self-distillation (RL with HardTests improves pass@1 while TACO RL hurts it), but teacher-distillation benefits more from question scaling than test quality.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We will open-source our dataset and synthesis pipeline at https://leililab.github.io/HardTests/' — this is a promise of future release, not an actual released codebase. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as code: 'We will open-source our dataset' is a promise, not a current release. The 47k problem dataset is not yet publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. Appendix A.6 mentions 'Llama-factory' and 'veRL' for training and 'firejail' for sandboxing but does not provide a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the pipeline conceptually but does not include scripts or commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1–5 report only point estimates for precision, recall, and pass@k. No confidence intervals, error bars, or ± notation appears anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims HardTests 'significantly outperforms' baselines (Section 4.4) but provides no statistical significance tests (no p-values, t-tests, or bootstrap tests). All comparisons are raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point differences with baseline context throughout: '11.3 percentage points higher precision and 17.5 percentage points higher recall' (Abstract), and tables show both baseline and HardTests numbers side-by-side, allowing direct comparison of magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification given for sample sizes. 653 AtCoder problems and 600 Codeforces problems are used for direct evaluation (Section 4.3), 105 problems for LiveCodeBench evaluation, and ~5k problems for RL/distillation. No power analysis or sample size rationale provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported across experimental runs. Tables show single-run point estimates. Figure 3 shows single RL training curves per condition without confidence bands."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "TACO (Li et al., 2023) and CodeContests (Li et al., 2022) are used as baselines for direct test quality evaluation (Tables 1–2). Ablative baselines (HT-TYPE1, HT-TYPE1+2) are included in Table 1. OlympicCoder-7B serves as a baseline for teacher-distillation (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "TACO (2023) and CodeContests (2022) are the most relevant existing test synthesis datasets. The paper also compares with concurrent work (rStar-Coder, HF-Codeforces) in the related work section. OlympicCoder-7B (2025) is a contemporary training baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows ablation results: HT-TYPE1 (only directly generated inputs), HT-TYPE1+2 (adding regular inputs), and full HardTests (all three types). This demonstrates the contribution of each input type, showing precision improvements of 2–48pp from Type 2 and Type 3."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Direct evaluation uses both precision and recall (Section 4.1). Downstream evaluation uses pass@1, pass@5, and pass@10 (Tables 3–5). RL training also reports validation reward curves (Figure 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of test case quality is included. All evaluation is automated: precision/recall computed against oracle test suites and Codeforces submission verdicts."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench-105 is used as a held-out evaluation set for all post-training experiments (Section 5.1). The paper explicitly decontaminates training data against LiveCodeBench: 'We conduct decontamination by removing the problems that are in LiveCodeBench from our dataset' (Section 3.4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1–5 break down results by difficulty level (difficulty 1–4+). Table 3 breaks down by LiveCodeBench difficulty (Easy/Medium/Hard). Results per model (Qwen-7B, Qwen-14B, GPT-4o, human) are also shown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.5 provides three detailed qualitative examples of false positives and false negatives, showing how specific test types catch or miss specific types of incorrect programs. Figure 5 documents pipeline failure rates and reasons (6.62% no valid oracle, 5.85% output verification failed, 3.72% input generation failed)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows that filtering to 13k high-quality trajectories underperforms using all 46.6k trajectories for teacher-distillation, contradicting the hypothesis that test quality always helps. Table 5 shows RL with TACO tests actually hurts performance (38.48→36.95 pass@1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '11.3 percentage points higher precision and 17.5 percentage points higher recall' are supported by Tables 1–2 showing consistent improvements across settings. The '40 points' precision claim is supported by Table 1 (difficulty 4+, Qwen-7B: 21.67→60.00, ~38pp). Post-training effectiveness claims are supported by Tables 3–5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('test quality matters significantly for RL') are based on controlled single-variable manipulation: same model (Qwen3-4B), same training setup, same problem set (~5k), different test cases (HardTests vs TACO). The ablation study (Table 1) is also controlled single-variable manipulation of test types. These designs are adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'High-Quality Test Cases for LLM Coding' broadly, but the paper only tests competitive programming problems with standard I/O. The Limitation section acknowledges the I/O constraint but the title and abstract do not bound claims to competitive programming. SWE-bench-style coding (file I/O, web I/O, multi-file) is explicitly noted as out of scope but only in the Limitation section."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why HardTestGen works better. For instance, the improvement could partly stem from having more test cases rather than specifically harder ones, or from the oracle program quality rather than the generation strategy. The teacher-distillation finding (quantity > quality) is noted but not explored for alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines its proxy (precision and recall of test suites as binary classifiers of program correctness, Section 4.1) and maps it to the claimed outcome (test quality for verification). The downstream evaluation directly measures code generation performance (pass@k on LiveCodeBench). No unacknowledged proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' for all test generation (Section 3.3) without a version or snapshot date. 'Qwen2.5-Coder-7B-Instruct' and 'Qwen2.5-Coder-14B-Instruct' are reasonably specific, but 'DeepSeek-R1', 'Qwen3-4B', and 'GPT-4o' lack version/snapshot identifiers. Model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2.1 provides the complete prompt text for both the input validator generation and the input generator (all three types), including multiple detailed examples for in-context learning. These are the actual prompts used, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Sampling parameters are stated: temperature=0.7, top_p=0.95 for candidate program generation (Section 4.3). Appendix A.6 provides detailed training hyperparameters for all three post-training scenarios: epochs, learning rate, batch size, max length, and evaluation sampling parameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. HardTestGen is a multi-step pipeline with separate LLM calls for validation, generation, and output checking, but has no agent loop, tools, retry logic, or memory. The pipeline architecture is well-described in Section 3 and Figure 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes cleaning, deduplication (using problem IDs and n-gram overlaps), and decontamination (removing LiveCodeBench problems via URL comparison). Appendix A.2.1 documents the problem filtering pipeline: removing problems without oracle programs, excluding 'core logic' (non-I/O) problems, leaving 32.5k problems."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitation' section follows the Conclusion, discussing three specific limitations: tests still not as good as human-written, reliance on oracle solutions, and constraint to single-file standard I/O."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) synthetic tests underperform human-written ones, (2) oracle solutions may not exist for all coding domains, (3) code must be single-file with standard I/O. These are concrete, not boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'the code being tested is constrained to a single file that uses Standard I/O for input and output. However, many real-world coding problems are more complicated, e.g. coding problems in SWE-bench that may involve file I/O or web I/O' (Limitation section). This clearly bounds what was NOT tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is not yet available. 'We will open-source our dataset' is a promise, not a current release. No supplementary data files are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 describes data collection in detail: 5 direct data sources (Codeforces, AtCoder, Luogu, CodeContests, TACO) covering 13 OJs. Table 6 lists all OJ URLs and sources. Collection methods include scraping problem specifications and user submissions from each platform with specific time boundaries (up to September 2024)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public competitive programming platforms (online judges). Human submissions used for evaluation (MatrixStudio/Codeforces-Python-Submissions) are pre-existing public data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: problem collection from 13 OJs → filtering (removing non-I/O problems, those without oracle programs) → deduplication (IDs + n-gram overlap) → decontamination (removing LiveCodeBench problems) → test generation → output verification. Figure 5 shows the status distribution of test generation (81.9% success rate with specific failure reasons and percentages)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'The OpenAI API credits used in this paper were partially supported by the OpenAI Research Access Program. The training compute used was partially supported by National Center for Supercomputing Applications and ScOp Venture Capital. KZ was partially supported by ChipAgents.ai.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Carnegie Mellon University, UC Santa Barbara, UT Austin. No authors are affiliated with OpenAI or the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI provided API credits and the paper uses GPT-4o exclusively for all test synthesis. A positive result for HardTestGen implicitly demonstrates GPT-4o's utility for test generation. OpenAI has a commercial interest in showing GPT-4o is useful. ScOp Venture Capital's interests are unclear."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The relationship with ScOp Venture Capital and ChipAgents.ai is mentioned in funding but no declaration of whether authors hold equity or other interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (GPT-4o, Qwen2.5-Coder, Qwen3-4B, DeepSeek-R1). Without cutoff dates, it is impossible to assess whether test problems were in the models' pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses train/test overlap for post-training: 'We conduct decontamination by removing the problems that are in LiveCodeBench from our dataset. Since most of its problems are from Codeforces and AtCoder, we directly compare the URLs to the problems' (Section 3.4)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench is specifically designed for contamination-free evaluation (Jain et al., 2025b, described as 'Holistic and contamination free evaluation'). The paper decontaminates training data against it via URL comparison. However, contamination risk for the direct evaluation on Codeforces/AtCoder (Section 4) is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluations are automated or use pre-existing public submission data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses public competitive programming problems and automated code evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Human submissions are from public Codeforces data but participants are not subjects of study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or per-example costs are reported for generating test cases with GPT-4o across 32.5k problems. No cost comparison between HardTestGen and baselines is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.6 states 'All our experiments are run on 8 NVIDIA H100 GPUs' but does not report total GPU hours, training time, or API spend. Hardware is identified but total compute budget is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results reported. Tables show single-run point estimates. Figure 3 shows single RL training curves per condition without confidence bands or seed variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "For candidate program generation, '10 candidate programs from each LLM' is stated (Section 4.3). However, for the RL and distillation training experiments (the key downstream claims), the number of independent training runs is not stated. Figure 3 implies single runs per condition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Training hyperparameters appear chosen without systematic search, and no search budget (configurations tried, compute spent) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "For RL, 'we run the best checkpoints (according to valid reward) of both training jobs within 100 steps' (Section 5.2). Checkpoint selection is based on validation reward, not test performance, which is appropriate."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is moot."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their own system (HardTestGen) against baselines (TACO, CodeContests) without acknowledging potential bias from evaluating their own system. No independent evaluation or discussion of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "HardTestGen generates three types of tests using multiple LLM calls per problem (input validators, generators, oracle output computation), likely requiring significantly more compute than TACO's simpler approach. This compute difference is never discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "LiveCodeBench is used for downstream evaluation without discussing its construct validity — whether performance on 105 competitive programming problems measures the claimed 'code generation performance' broadly. The direct evaluation metric (precision/recall) is well-defined but no discussion of whether binary pass/fail captures true code quality."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used in model comparisons. All comparisons are between the same model trained with different data, so scaffold confound does not arise."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of temporal leakage. The models (Qwen, GPT-4o) may have been trained on Codeforces/AtCoder solutions used in Section 4 evaluation. While LiveCodeBench uses recent problems, the temporal relationship between model training and benchmark creation is not explicitly discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The evaluation setup provides problem descriptions to models for code generation — whether these descriptions could leak answer information (e.g., through editorial-style hints) is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of non-independence. Problems from the same OJs (Codeforces, AtCoder) may share structural similarities or be from the same contest series. Training and evaluation problems could share authors or problem templates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "A concrete decontamination method is applied: 'we directly compare the URLs to the problems' (Section 3.4) to remove LiveCodeBench problems from training data. For deduplication, n-gram overlap analysis and problem ID matching are used."
       }
     }
   },
   "claims": [
     {
       "claim": "HardTestGen tests demonstrate 11.3 percentage points higher precision and 17.5 percentage points higher recall on average compared to existing tests when evaluating LLM-generated code.",
       "evidence": "Tables 1 and 2 (Section 4.4) show precision and recall across 4 candidate program sources (3 LLMs + human) on AtCoder (653 problems) and Codeforces (600 problems), with consistent improvements across most settings.",
       "supported": "moderate"
     },
     {
       "claim": "For harder problems, the precision improvement can be as large as 40 percentage points.",
       "evidence": "Table 1: For Qwen2.5-Coder-7B on AtCoder difficulty 4+, TACO precision is 21.67 vs HardTests 60.00 (~38pp). Table 2: For Qwen-7B on Codeforces difficulty 4, TACO 9.82 vs HardTests 50.00 (~40pp).",
       "supported": "moderate"
     },
     {
       "claim": "Test quality matters significantly for reinforcement learning: RL with HardTests improves downstream performance while RL with TACO hurts it.",
       "evidence": "Table 5 and Figure 3 (Section 5.2): Qwen3-4B base pass@1=38.48; RL with TACO=36.95 (decrease); RL with HardTests=39.42 (increase). pass@10 improves from 56.19 to 64.76 with HardTests. Validation reward curve (Figure 3) is consistently higher for HardTests.",
       "supported": "moderate"
     },
     {
       "claim": "Teacher-distillation benefits more from question scaling than test quality or sample scaling.",
       "evidence": "Table 3: Full 46.6k trajectories (unfiltered) achieve pass@1=32.86, outperforming both the 13k filtered subset (25.24) and OlympicCoder-7B trained with 100k trajectories (25.81).",
       "supported": "moderate"
     },
     {
       "claim": "Self-distillation performance is highly dependent on sample quality, requiring a good verifier to select correct trajectories.",
       "evidence": "Table 4: Qwen3-4B self-distilled with 'good 5k' (verified correct) achieves pass@1=36.00, pass@10=60.00, vs 'bad 5k' (verified incorrect) at 34.00/54.92 and 'random 5k' at 32.75/57.14.",
       "supported": "moderate"
     },
     {
       "claim": "Type 2 (Regular) and Type 3 (Hacking) test inputs are necessary — precision improves 2–48pp over Type 1 alone with at most 2.5pp recall decrease.",
       "evidence": "Table 1 ablation: HT-TYPE1 precision ranges from 10.40 to 99.42 across settings, while full HardTests ranges from 60.00 to 100.0. HT-TYPE1 recall ranges from 89.02 to 100.0, while full HardTests ranges from 87.80 to 99.18.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests on any result",
       "detail": "All tables (1–5) report point estimates without confidence intervals, standard deviations, or significance tests. The word 'significantly' is used (Section 4.4) without any statistical significance test. RL training curves in Figure 3 appear to be single runs, which are known to have high variance (Henderson et al., 2018, which the paper cites for related work)."
     },
     {
       "flag": "Single-run RL experiments for a key claim",
       "detail": "The claim that 'test quality matters significantly for reinforcement learning' appears based on a single RL training run per condition. RL training is notoriously noisy, and the pass@1 difference (36.95 vs 39.42) could plausibly be within single-run variance. Without multiple seeds, this claim is fragile."
     },
     {
       "flag": "Small evaluation set for downstream claims",
       "detail": "LiveCodeBench-105 contains only 105 problems (selected as the subset with 'stdin' type test cases). This is small for evaluating post-training effects and limits statistical power for detecting real differences, especially when broken down by difficulty."
     },
     {
       "flag": "OpenAI funding with exclusive GPT-4o usage",
       "detail": "The paper received OpenAI API credits and exclusively uses GPT-4o for all test synthesis. A positive result for HardTestGen implicitly demonstrates GPT-4o's utility. No alternative LLMs were tested as the test generator, so the pipeline's generalizability beyond GPT-4o is unknown."
     },
     {
       "flag": "Compute cost of pipeline not reported or compared",
       "detail": "HardTestGen uses multiple LLM calls per problem (input validator, 3 types of generators, oracle execution, special judge generation for 25.4% of problems). The total API cost for processing 32.5k problems with GPT-4o is likely substantial but never reported or compared to baselines, making cost-effectiveness impossible to assess."
     }
   ],
   "cited_papers": [
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Introduced the CodeContests dataset for competitive programming, a key baseline in this paper."
     },
     {
       "title": "TACO: Topics in Algorithmic Code Generation Dataset",
       "authors": ["Rongao Li", "Jie Fu", "Bo-Wen Zhang"],
       "year": 2023,
       "arxiv_id": "2312.14852",
       "relevance": "Major baseline dataset integrating multiple OJs with LLM-generated test cases; primary comparison target."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (EvalPlus)",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Extended HumanEval with additional tests using LLM synthesis, a closely related test augmentation approach."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major RL-trained reasoning model; used as teacher for distillation experiments and exemplifies RLVR approach."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational LLM code generation benchmark with hand-written test cases."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Early coding challenge benchmark; paper notes 60% of programs passing APPS tests are actually wrong."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2025,
       "relevance": "Contamination-free code evaluation benchmark used as the held-out evaluation set for all post-training experiments."
     },
     {
       "title": "OpenCodeReasoning: Advancing Data Distillation for Competitive Coding",
       "authors": ["Wasi Uddin Ahmad", "Sean Narenthiran", "Somshubra Majumdar"],
       "year": 2025,
       "arxiv_id": "2504.01943",
       "relevance": "Concurrent dataset of 28k problems with 735k reasoning traces; paper confirms that data scaling dominates trajectory correctness in distillation."
     },
     {
       "title": "ALGO: Synthesizing Algorithmic Programs with LLM-Generated Oracle Verifiers",
       "authors": ["Kexun Zhang", "Danqing Wang", "Jingtao Xia"],
       "year": 2023,
       "arxiv_id": "2305.14591",
       "relevance": "Prior work by same group on LLM-based test synthesis with oracle verifiers; basis for the oracle-free variant in Appendix A.7."
     },
     {
       "title": "AceCoder: Acing Coder RL via Automated Test-Case Synthesis",
       "authors": ["Huaye Zeng", "Dongfu Jiang", "Haozhe Wang"],
       "year": 2025,
       "arxiv_id": "2502.01718",
       "relevance": "Concurrent work on automated test synthesis for RL training of code models; used as baseline in oracle-free experiments."
     },
     {
       "title": "KodCode: A Diverse, Challenging, and Verifiable Synthetic Dataset for Coding",
       "authors": ["Zhangchen Xu", "Yang Liu", "Yueqin Yin"],
       "year": 2025,
       "arxiv_id": "2503.02951",
       "relevance": "Synthetic coding dataset that generates questions, solutions, and tests with LLMs; represents the Type-1-only approach."
     },
     {
       "title": "rStar-Coder: Scaling Competitive Code Reasoning with a Large-Scale Verified Dataset",
       "authors": ["Yifei Liu", "Li Lyna Zhang", "Yi Zhu"],
       "year": 2025,
       "arxiv_id": "2505.21297",
       "relevance": "Concurrent work on reliable test synthesis for competitive programming; compared in related work."
     },
     {
       "title": "Code-R1: Reproducing R1 for Code with Reliable Rewards",
       "authors": ["Jiawei Liu", "Lingming Zhang"],
       "year": 2025,
       "relevance": "RL training approach for code that emphasizes reliable rewards; the paper's RL setup is inspired by this work."
     },
     {
       "title": "Scattered Forest Search: Smarter Code Space Exploration with LLMs",
       "authors": ["Jonathan Light", "Yue Wu", "Yiyou Sun"],
       "year": 2025,
       "arxiv_id": "2411.05010",
       "relevance": "Demonstrates that weak verifiers can harm downstream code generation and search performance."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The pipeline and 47k-problem dataset are directly useful for researchers doing LLM post-training for coding, but code/data are not yet released and the approach is limited to competitive programming."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that teacher-distillation doesn't benefit from test quality is mildly surprising, but the main finding (better tests help) is expected."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or security concerns raised; the paper focuses on improving test quality for training."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly criticizes existing datasets (TACO has '90% false positive rate for difficult problems') but without dramatic framing."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A project page exists at leililab.github.io/HardTests/ but dataset and code are not yet released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "CMU authors and uses well-known models (GPT-4o, DeepSeek-R1), but not from a major AI lab."
     }
   }
 }