ai-research-survey

scan.json (23996B)

---

 {
   "paper": {
     "title": "RLTF: Reinforcement Learning from Unit Test Feedback",
     "authors": ["Jiate Liu", "Yiqin Zhu", "Kaiwen Xiao", "Qiang Fu", "Xiao Han", "Wei Yang", "Deheng Ye"],
     "year": 2023,
     "venue": "Transactions on Machine Learning Research",
     "arxiv_id": "2307.04349",
     "doi": "10.48550/arXiv.2307.04349"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RLTF proposes an online RL framework with multi-granularity unit test feedback (coarse, fine-grained, and adaptive) for improving code LLMs on program synthesis. On APPS and MBPP benchmarks using CodeT5 770M, RLTF achieves state-of-the-art results among CodeT5-based methods. Ablation studies show each feedback type contributes incrementally, with fine-grained feedback providing the largest boost. The approach generalizes across base models (CodeT5 770M and CodeGen 2.7B).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is available at: https://github.com/Zyq-scut/RLTF' providing a GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks APPS (Hendrycks et al., 2021) and MBPP (Austin et al., 2021), which are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8 NVIDIA V100 GPUs, each with 32GB of memory' but does not provide a requirements.txt, Dockerfile, or detailed dependency/library version listing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Key details like batch size and learning rate are mentioned but no structured reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-9 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RLTF outperforms CodeRL and PPOCoder but provides no statistical significance tests. Comparisons are based solely on raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context, e.g., pass@1 improves from 1.30% (SL only) to 1.45% (full RLTF) in Table 5, and from 1.30 to 1.45 vs CodeRL's 1.30 in Table 3, providing enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of test problems or generated samples. APPS and MBPP sizes are inherited from prior work without discussion of statistical adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across runs. It is unclear whether results represent single runs or averages."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares against Codex, AlphaCode, GPT variants, CodeRL, and PPOCoder. Table 5 includes SL-only baseline and incremental ablations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeRL (2022) and PPOCoder (2023) are the most recent RL-based code generation methods at time of submission, representing state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablations in Tables 4-8: impact of framework (online vs offline), feedback combinations, Rfine values, temperature, and different base models."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results reported across pass@1, pass@5, pass@10, pass@100, and pass@1000 metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here; code generation correctness is objectively measured via unit test pass/fail."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "APPS has explicit train/test splits (Section 4.1). MBPP evaluation is zero-shot on a separate test set of 500 instances."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down APPS results by difficulty level: Introductory, Interview, and Competition. Figure 2 shows per-error-type analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides qualitative analysis of error types before and after RLTF. Section 4.5 discusses that RLTF is more effective at runtime/compiler errors than semantic errors."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that timeout errors slightly increase after applying RLTF (Section 4.5), and pass@1000 with Critic Sampling is slightly lower than CodeRL (Table 3, 20.32 vs 20.98)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'state-of-the-art performance on APPS and MBPP benchmarks,' which is supported by Tables 3 and 9 showing improvements over CodeRL and PPOCoder."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'RLTF improves performance' are supported by controlled ablation studies (Tables 4-5) that isolate individual components via single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Python benchmarks (APPS and MBPP) with two base models, but Section 5 acknowledges the limitation that 'manual categorization of sub-error types makes it challenging to transfer RLTF to other programming languages.' However, the title and abstract make broad claims about 'program synthesis' without bounding to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, whether the online framework's improvement comes from simply training longer or seeing more data rather than the feedback mechanism specifically."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k on unit tests and frames results in terms of pass@k. No proxy gap — the claims match the granularity of the measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions with sizes are stated: 'CodeT5 770M' and 'CodeGen 2.7B'. These are specific open-source model checkpoints with known architectures."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a fine-tuning/RL training paper, not a prompting paper. The models are trained end-to-end on input-output pairs, not prompted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: batch size 32, learning rate 2e-6, nucleus sampling top-p 0.95, temperature 0.6 (APPS) and 1.2 (MBPP), online buffer length 6400, buffer update every 50 steps."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RLTF is an RL training framework, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'We adhere to the same preprocessing steps as those in (Hendrycks et al., 2021)' and documents the subprocess modification for segfault handling. MBPP prompt format is explicitly described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Some limitations are briefly mentioned in Section 5 (Conclusions and Future Work) but as future directions rather than as a structured limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity discussed. The conclusion mentions transferability to other languages as a limitation, but does not discuss threats like evaluation methodology, baseline fairness, or reproducibility concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states: 'The manual categorization of sub-error types we employed makes it challenging to transfer RLTF to other programming languages, which should be considered as another limitation.' Also acknowledges benchmark test diversity limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated programs, individual test outcomes, training logs) is provided for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes both benchmarks in detail: APPS (10,000 problems, train/test splits, difficulty levels, unit test counts) and MBPP (974 instances, splits described)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (APPS and MBPP)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The online buffer workflow is documented in Figure 1b and Section 3.2, including buffer size (6400), update frequency (every 50 steps), and the queue mechanism for data flow."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section. All authors are from Tencent but no funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All seven authors are clearly listed as affiliated with Tencent, with their Tencent email addresses prominently displayed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Tencent employees. Tencent has commercial interests in code generation tools, so the funder/employer is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. All authors work at Tencent, which has potential commercial interests in code generation technology."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of training data cutoff for the base CodeT5 or CodeGen models. The models are fine-tuned on APPS training data, but pre-training data temporal scope is not addressed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether APPS or MBPP test problems could have appeared in the CodeT5 or CodeGen pre-training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "APPS (2021) and MBPP (2021) were published before both CodeT5 and CodeGen training. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency reported for generating solutions."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: '8 NVIDIA V100 GPUs... training process took approximately 24 hours. Concurrently, three additional machines with similar 8-card V100 GPU configurations were used to generate the latest samples.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single runs with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It is unclear whether results are from a single training run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While ablations over Rfine and temperature are shown, the total hyperparameter search budget (number of configurations tried) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents results for Rfine=-0.3 and temperature=1.0 as the final configuration but does not explain whether selection was based on validation set or test set performance."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare RLTF against their own reproduction of CodeRL and scaled PPOCoder results without acknowledging potential bias in re-implementation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RLTF uses 4 machines (32 V100 GPUs total) while the compute used by baselines CodeRL and PPOCoder is not discussed. The online framework requires substantially more compute for sample generation, but no matched-budget comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether APPS/MBPP adequately measure code generation capability or their construct validity limitations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. This is a training method comparison, not an agentic system evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether APPS/MBPP problems or their solutions appeared in CodeT5/CodeGen pre-training data, despite both benchmarks predating the models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP includes unit test assert statements in the input sequence, which could leak answer information. The paper acknowledges this 'occasionally encourages models to overfit' but does not treat it as a leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether APPS training and test problems share structural similarities or come from the same coding websites."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "RLTF achieves state-of-the-art performance on the APPS benchmark among CodeT5-based methods",
       "evidence": "Table 3 shows RLTF achieves pass@1=1.45%, pass@5=3.78%, pass@1000=19.92% without Critic Sampling, outperforming CodeRL (1.30/3.32/17.78) and PPOCoder-scaled (1.32/3.37/17.84).",
       "supported": "moderate"
     },
     {
       "claim": "The online framework improves performance over offline training",
       "evidence": "Table 4 ablation shows online+RLTF (pass@1=1.45%) > offline+RLTF (1.34%) > online-only (1.37%) > offline-only (1.29%).",
       "supported": "moderate"
     },
     {
       "claim": "Fine-grained feedback contributes the most significant performance boost among the feedback types",
       "evidence": "Table 5 shows adding fine-grained feedback improves pass@1 from 1.37% to 1.41% (largest single-feedback jump), and Section 4.3 states this explicitly.",
       "supported": "moderate"
     },
     {
       "claim": "RLTF generalizes across base models (CodeT5 and CodeGen)",
       "evidence": "Table 8 shows improvements on both CodeT5 770M (1.30→1.45 pass@1) and CodeGen 2.7B (1.64→2.04 pass@1).",
       "supported": "moderate"
     },
     {
       "claim": "RLTF achieves state-of-the-art zero-shot performance on MBPP",
       "evidence": "Table 9 shows CodeT5+RLTF achieves pass@1=30.4%, pass@80=71.3% vs CodeRL (25.7/68.1) and PPOCoder (26.1/68.2).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or statistical tests",
       "detail": "All results are point estimates without error bars, standard deviations, or significance tests. Improvements are often small (e.g., 0.15% pass@1) and could be within noise."
     },
     {
       "flag": "Unfair baseline comparison methodology",
       "detail": "PPOCoder results are 'proportionally scaled' from CodeRL's open-source model rather than independently reproduced. The paper acknowledges discrepancies between reported and reproduced CodeRL results, raising concerns about evaluation consistency."
     },
     {
       "flag": "Substantially higher compute budget than baselines",
       "detail": "RLTF uses 4 machines with 32 V100 GPUs total (1 for training + 3 for sample generation) for 24 hours. The online framework inherently requires more compute than offline methods, but no matched-budget comparison is provided."
     },
     {
       "flag": "All authors from same company",
       "detail": "All seven authors are from Tencent with no conflict of interest declaration. Tencent has commercial interests in code generation technology."
     },
     {
       "flag": "No contamination analysis",
       "detail": "APPS and MBPP were published in 2021, before CodeT5 and CodeGen training data collection. No analysis of whether benchmark solutions leaked into pre-training data."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning",
       "authors": ["Hung Le", "Yue Wang", "Akhilesh Deepak Gotmare", "Silvio Savarese", "Steven Chu Hong Hoi"],
       "year": 2022,
       "relevance": "Primary baseline; introduces RL with unit test feedback and Critic Sampling for code generation."
     },
     {
       "title": "Execution-based Code Generation using Deep Reinforcement Learning",
       "authors": ["Parshin Shojaee", "Aneesh Jain", "Sindhu Tipirneni", "Chandan K Reddy"],
       "year": 2023,
       "arxiv_id": "2301.13816",
       "relevance": "Primary baseline; applies PPO to CodeRL framework for code generation improvement."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark for evaluating LLM code generation."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "authors": ["Dan Hendrycks"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Introduces the APPS benchmark used as primary evaluation in this paper."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces the MBPP benchmark used as secondary evaluation in this paper."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven CH Hoi"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "Base model used in RLTF experiments; encoder-decoder architecture for code tasks."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Second base model used in RLTF experiments to demonstrate generalization."
     },
     {
       "title": "Competition-Level Code Generation with AlphaCode",
       "authors": ["Yujia Li"],
       "year": 2022,
       "relevance": "Competitive code generation system used as baseline comparison in APPS evaluation."
     },
     {
       "title": "CodeBERT: A Pre-trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Early pre-trained model for code; encoder-only architecture for code understanding."
     },
     {
       "title": "Self-critiquing Models for Assisting Human Evaluators",
       "authors": ["William Saunders"],
       "year": 2022,
       "arxiv_id": "2206.05802",
       "relevance": "Explores LLM self-critique capabilities relevant to AI-assisted code review."
     }
   ]
 }