ai-research-survey

scan.json (23300B)

---

 {
   "paper": {
     "title": "Applying RLAIF for Code Generation with API-usage in Lightweight LLMs",
     "authors": ["Sujan Dutta", "Sayantan Mahinder", "Raviteja Anantha", "Bortik Bandyopadhyay"],
     "year": 2024,
     "venue": "NLRSE",
     "arxiv_id": "2406.20060",
     "doi": "10.48550/arXiv.2406.20060"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or any code repository URL is provided in the paper. The authors mention using the transformers and TRL libraries but do not release their own training code or pipeline."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available Gorilla dataset published by Patil et al. (2023). The dataset is a standard public benchmark that they did not modify."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'transformers' and 'TRL' Python libraries (Appendix B.2) and NVIDIA A100 40GB GPUs (Appendix B.3), but does not provide a requirements.txt, Dockerfile, or specific library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating experiments are provided. The methodology section describes the approach conceptually but lacks concrete reproduction details."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates (e.g., 27.9% executability rate) with no confidence intervals, error bars, or ± notation despite reporting results as the mean of three inference runs (Appendix B.2)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MRL 'boosts performance' and 'outperforms' MGorilla based solely on comparing raw numbers (e.g., 27.9% vs 26.9%) without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context: 'CodeBLEU (1.6 points abs), AST (0.66% abs) and Executability Rate (4.5% abs)' (Section 4), and the abstract states '4.5% improvement in executability rate' and '1.0% higher code executability rate' over the 7B baseline."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the dataset split (90/10 train/test from ~9k examples) or for using only three inference runs. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B.2 states 'results are reported by taking the mean of three inference runs' but no standard deviation, variance, or spread measure is reported in Table 1 or elsewhere."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against two baselines: their own supervised fine-tuned model (MSFT, 780M) and the Gorilla fine-tuned LLaMA-7B (MGorilla) from Patil et al. (2023), shown in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MGorilla (Patil et al., 2023) is a contemporary and directly relevant baseline for API-call code generation on the Gorilla dataset. The paper was submitted in 2024 and compares against a 2023 baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The framework has multiple components (8 evaluation prompts, reward model, PPO training) but no experiments isolate the contribution of individual components (e.g., varying the number of evaluation questions, comparing different reward models)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports four metrics: Executability Rate, ROUGE (average of ROUGE-1/2/L/sum), CodeBLEU, and AST sub-tree matching (Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is included. All evaluation is automated via metrics (ROUGE, CodeBLEU, AST, executability). Given that the paper claims quality improvements in code generation, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 states 'We trained our model on 90% of the data and kept the rest for evaluation,' indicating a separate evaluation split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The Gorilla dataset spans 37 different API domains, but the paper only reports aggregate metrics in Table 1. No per-domain or per-category breakdown is provided. The paper even acknowledges in the Limitations section that they 'have not analyzed the performance between more frequent APIs (head) and infrequent APIs (tail).'"
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Figure 2 shows one positive example where MRL fixes an undefined variable error, but no systematic failure analysis or examples of where MRL fails are presented."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every metric in Table 1 shows improvement for MRL over MSFT. No negative results, failed approaches, or configurations that did not work are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 4.5% improvement in executability rate (supported by Table 1: 27.9% vs 23.4%) and that the 780M model surpasses the 7B model by 1.0% (supported by Table 1: 27.9% vs 26.9%). These claims match the reported results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RLAIF 'boosts the performance' and 'significantly enhances' the baseline, implying RLAIF causes the improvement. However, there is no ablation or controlled experiment isolating the effect of RLAIF from other factors (e.g., additional training steps, the reward model architecture)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract broadly claim improvement in 'code generation' and 'API-usage' but the evaluation is limited to only the HuggingFace portion of the Gorilla dataset (Python only, ML API calls only). The Limitations section acknowledges Python-only but the title and abstract do not bound the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the performance gains. For example, the additional training steps in RL could contribute independently of the AI feedback quality, or the improvement could come from the reward model regularization rather than the feedback signal."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-3.5' without specifying a version (e.g., gpt-3.5-turbo-0613) and 'GPT-2-large' without a specific checkpoint identifier. No API version or snapshot date is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 in Appendix A lists all 8 prompts used for AI feedback. Each prompt contains the template with [instruction] and [code] placeholders, and while the actual fill values are from the dataset (which is public), the prompt templates are the complete prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 reports learning rates for all three training stages: MSFT (5×10^-4), Mreward (5×10^-5), and PPO (6×10^-6). The paper also states that no hyperparameter search was performed."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a standard training pipeline (SFT → reward model → PPO), not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states it uses the HuggingFace part of the Gorilla dataset with a 90/10 split but does not describe any preprocessing steps, filtering, or how the split was created (random vs. stratified by domain)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitations' section discussing inherited biases from GPT-2-large, lack of programming language diversity (Python only), missing head/tail API analysis, and the offline learning limitation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: biases from GPT-2-large affecting generated code comments, Python-only evaluation limiting generalizability, lack of head vs. tail API analysis, and the offline learning problem given rapidly evolving APIs."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section mentions Python-only and missing head/tail analysis, it does not explicitly state what the results do NOT show. The paper does not bound claims to the specific tested setting (HuggingFace APIs only, single dataset, single base model)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The Gorilla dataset is publicly available, but the generated preference data used to train the reward model and the intermediate outputs (GPT-3.5 feedback scores, generated code samples) are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the Gorilla dataset origin (from Patil et al. 2023), its composition (925+ unique APIs across 37 domains, 10 instructions per API), and Section 3 describes how preference data is generated via GPT-3.5 scoring."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from a standard benchmark dataset (Gorilla)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-step pipeline is documented: (1) SFT on Gorilla data, (2) generate two outputs per instruction, score via GPT-3.5 with 8 binary questions, create preference pairs, train reward model, (3) PPO fine-tuning with reward model. The flow is shown in Figure 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section listing grants or sponsors is provided. The paper notes the first author's work was done during an Apple internship, but no formal funding disclosure exists."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Rochester Institute of Technology (first author) and Apple (three co-authors). The footnote states 'Work done as a part of an internship at Apple.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of four authors are Apple employees, and Apple has commercial interests in AI/ML model capabilities. The research was conducted during an Apple internship. The funder (Apple, implicitly) has a stake in demonstrating that lightweight models can perform well, which aligns with Apple's on-device AI strategy."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations are provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-2-large and GPT-3.5 without stating training data cutoff dates for either model. GPT-2 may have seen code from HuggingFace repositories, and GPT-3.5 may have seen the Gorilla dataset or related content."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-2-large's pre-training data or GPT-3.5's training data overlaps with the Gorilla dataset or HuggingFace API documentation used to construct the benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Gorilla dataset was published in 2023 (arXiv:2305.15334). GPT-2 was trained before this, but GPT-3.5 may have been trained on data overlapping with the API documentation used to construct Gorilla. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API cost for the GPT-3.5 feedback queries is reported. The paper does not quantify the cost of running the AI feedback pipeline per example."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 states 'We used a cluster of NVIDIA A100 40GB GPUs for our experiments. We spent in total ~60 GPU hours for all of the experiments.'"
       }
     }
   },
   "claims": [
     {
       "claim": "RLAIF achieves a 4.5% absolute improvement in executability rate over the supervised fine-tuned baseline (MSFT).",
       "evidence": "Table 1 shows MRL at 27.9% executability rate vs MSFT at 23.4% (Section 4).",
       "supported": "moderate"
     },
     {
       "claim": "A 780M parameter model (MRL) trained with RLAIF surpasses a 7B parameter fine-tuned model (MGorilla) by 1.0% in code executability rate.",
       "evidence": "Table 1 shows MRL at 27.9% vs MGorilla at 26.9% (Section 4). However, this 1.0% difference has no statistical test and is within likely noise range for three inference runs without reported variance.",
       "supported": "weak"
     },
     {
       "claim": "MRL outperforms MGorilla across ROUGE, CodeBLEU, AST, and executability rate metrics.",
       "evidence": "Table 1 shows MRL ahead on all four metrics: ROUGE 47.5 vs 41.2, CodeBLEU 42.2 vs 36.8, AST 73.62% vs 71.68%, Executability 27.9% vs 26.9%.",
       "supported": "moderate"
     },
     {
       "claim": "RLAIF using binary questions to GPT-3.5 is an effective alternative to human feedback or code execution-based feedback for API-usage code generation.",
       "evidence": "The framework is described in Section 3 and results in Table 1 show improvement, but no comparison with RLHF or execution-based feedback is provided. The claim rests on a single dataset comparison without ablations.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes an RLAIF framework that uses GPT-3.5 to provide binary feedback on 8 quality dimensions of generated code, which is then used to train a reward model for PPO fine-tuning of GPT-2-large (780M parameters). On the HuggingFace subset of the Gorilla dataset, the RLAIF-trained model improves executability rate by 4.5% over the SFT baseline and marginally outperforms a fine-tuned LLaMA-7B model (27.9% vs 26.9% executability). The approach demonstrates that AI feedback can substitute for expensive code execution feedback in settings where running generated code is prohibitively costly.",
   "red_flags": [
     {
       "flag": "No statistical tests on small differences",
       "detail": "The key claim that the 780M model surpasses the 7B model rests on a 1.0% executability rate difference (27.9% vs 26.9%) with no significance test and no reported variance across the three inference runs. This difference could easily be noise."
     },
     {
       "flag": "No ablation study",
       "detail": "The framework has multiple components (8 evaluation prompts, reward model architecture, PPO training) but no ablation isolates which components contribute to the gains. It is unclear whether all 8 prompts are necessary or whether simpler reward signals would suffice."
     },
     {
       "flag": "Missing variance despite multiple runs",
       "detail": "Results are reported as the mean of three inference runs (Appendix B.2) but no standard deviation or spread measure is provided, making it impossible to assess result stability."
     },
     {
       "flag": "Potential industry conflict of interest",
       "detail": "Three of four authors are Apple employees. Apple has commercial interest in demonstrating that small (<1B) models can perform well (on-device deployment). No conflict of interest statement is provided."
     },
     {
       "flag": "Very low absolute executability rates",
       "detail": "The best model achieves only 27.9% executability rate, meaning over 72% of generated code does not execute. The paper does not discuss whether these low absolute numbers undermine the practical value of the approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "authors": ["Shishir G. Patil", "Tianjun Zhang", "Xin Wang", "Joseph E. Gonzalez"],
       "year": 2023,
       "arxiv_id": "2305.15334",
       "relevance": "Introduced the Gorilla dataset and approach for teaching LLMs to generate API calls, which is the direct baseline and evaluation setting for this paper."
     },
     {
       "title": "RLAIF: Scaling Reinforcement Learning from Human Feedback with AI Feedback",
       "authors": ["Harrison Lee", "Samrat Phatale", "Hassan Mansoor"],
       "year": 2023,
       "arxiv_id": "2309.00267",
       "relevance": "Demonstrated that RLAIF can achieve human-level performance in summarization, foundational work for applying AI feedback to replace human feedback."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Introduced the RLAIF concept combining AI and human preferences for alignment, foundational for AI feedback approaches."
     },
     {
       "title": "CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning",
       "authors": ["Hung Le", "Yue Wang", "Akhilesh Deepak Gotmare"],
       "year": 2022,
       "relevance": "Applied actor-critic RL for code generation using execution feedback, a key prior approach that this paper argues is inapplicable for API-heavy code."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper (InstructGPT) whose pipeline this work adapts by replacing human feedback with AI feedback."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2024,
       "relevance": "Teaches LLMs to use external tools, relevant to the broader context of LLM tool/API usage research."
     },
     {
       "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-World APIs",
       "authors": ["Yujia Qin", "Shihao Liang", "Yining Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Builds on LLMs mastering real-world APIs at scale, directly related to code generation with API usage."
     },
     {
       "title": "Proximal Policy Optimization Algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "PPO algorithm used as the RL training method in this paper's RLAIF framework."
     },
     {
       "title": "WizardMath: Empowering Mathematical Reasoning for Large Language Models via Reinforced Evol-Instruct",
       "authors": ["Haipeng Luo", "Qingfeng Sun", "Can Xu"],
       "year": 2023,
       "arxiv_id": "2308.09583",
       "relevance": "Applied AI feedback to enhance mathematical reasoning in LLMs, a parallel application of RLAIF in a different domain."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced the MBPP benchmark for code generation, relevant to evaluation of LLM code generation capabilities."
     }
   ]
 }