ai-research-survey

scan.json (24670B)

---

 {
   "paper": {
     "title": "APRIL: API Synthesis with Automatic Prompt Optimization and Reinforcement Learning",
     "authors": [
       "Hua Zhong",
       "Shan Jiang",
       "Sarfraz Khurshid"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2509.25196"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. There is no mention of code availability."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 81 tasks from NumPy, scikit-learn, and SciPy, but no dataset download link or benchmark artifact is provided. The specific tasks are not enumerated or released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions using Gemini 2.0 and Python libraries but does not specify library versions or dependency details sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The paper describes the methodology at a high level but does not provide actionable steps to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1-3 report only point estimates (e.g., 93.8% pass rate). No confidence intervals, error bars, or uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims APRIL outperforms the baseline (93.8% vs. 77.8%) but provides no statistical significance test (e.g., McNemar's test, Fisher's exact test) to support the comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: '16.6%, 14.9%, and 16.7% over the baseline on the three benchmarks' (Section 4.2), and both absolute rates are given (e.g., 77.8% baseline vs. 93.8% APRIL), providing enough context to judge magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 81 tasks (36 NumPy, 33 scikit-learn, 12 SciPy) with no justification for why these numbers were chosen or whether they are sufficient for the claims made. The SciPy subset is particularly small (12 tasks)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. It is unclear whether experiments were run multiple times or only once. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares APRIL against a baseline using the initial manually engineered prompt on the same Gemini 2.0 model (Table 2, Section 4.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baseline is the same Gemini 2.0 model with a manually engineered prompt. No comparison is made against other code generation tools, other LLMs (e.g., GPT-4, Claude, open-source models), or prior program synthesis systems like SyPet, FrAngel, or EdSynth that are discussed in related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states: 'Due to time and space constraints, we do not conduct an ablation study isolating the individual contributions of APO and RLVR' (Section 4.2, after RQ2). This is a significant omission for a system with two main components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both executability rate (100%) and test pass rate (93.8%) as separate metrics in Table 1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the synthesized APIs is included. Evaluation is entirely automated via test suites. For API synthesis, human evaluation of code quality, readability, or maintainability would be relevant given the paper's claims about producing 'maintainable, idiomatic' implementations."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 40 of the 81 tasks for APO/RLVR training but does not clearly separate which tasks are training vs. evaluation. It is unclear whether the reported 93.8% pass rate is on held-out tasks only or includes training tasks. The paper does not explicitly state that the evaluation set is disjoint from the training set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per library (NumPy, scikit-learn, SciPy) in Tables 1-3, showing per-benchmark performance rather than just aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports that 5 APIs failed validation tests (76 out of 81 passed) but does not analyze why these failed, what patterns characterize failures, or show examples of incorrect synthesized code."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every configuration shown improves over the baseline. The paper mentions Figure 4 showing 'potential iterative improvement from APO on RLVR fine-tuned Model' but does not report any failed approaches or configurations that did not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'APRIL achieves substantial improvements' and 'a success rate exceeding 93.8%', both of which are supported by Tables 1 and 2. The abstract also claims this is 'a robust, scalable path' which is somewhat strong given the limited evaluation, but the core empirical claims are supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims that 'integrating APO and RLVR provides a robust, scalable path for component-based API synthesis' but without an ablation study, it cannot attribute improvements to either component. The causal mechanism behind the improvement (APO vs. RLVR vs. their combination) is not isolated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on 81 APIs from three specific Python scientific libraries but makes broad claims about 'component-based API synthesis in large libraries' and 'complex API synthesis' generally. The title and abstract do not bound the claims to scientific Python libraries or to Gemini 2.0 specifically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the improvement could be due to overfitting the prompt or RLVR to the specific test suite format, or the baseline prompt could simply be suboptimal. None of these possibilities are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'a customized instance of Gemini 2.0' and 'Gemini 2.5 Pro' but provides no specific model version identifiers, snapshot dates, or API version strings. 'Gemini 2.0' is a marketing name without a precise version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The manually engineered initial prompt is provided in full in Figure 2. It includes the actual prompt text with role conditioning, output format instructions, and template structure with placeholders whose fill values (method signature, module, library, test cases) are described."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: 'sampling temperature is set to 0.7', 'input context window of 32,000 tokens', and 'output context window (maximum generation length) of 8,000 tokens'. RLVR uses 'stochastic decoding (temperature and top-p)' though specific RLVR hyperparameters (learning rate, epochs, K samples) are not fully specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The APO pipeline is described in detail in Section 3.3 (discriminator score, text gradient, beam search), the RLVR pipeline in Section 3.4 (GRPO, binary rewards, clipped surrogate with KL penalty), and the test generation loop in Algorithm 1. The workflow is also shown in Figure 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 81 tasks from three libraries with 40 used for training, but does not document how these specific APIs were selected, what criteria were used for inclusion/exclusion, or how the training/evaluation split was determined."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion (Section 6) does not discuss limitations either."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no discussion of potential issues with the evaluation methodology, benchmark selection, or generalizability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clearly state what the results do NOT show or what settings are excluded. The only scoping statement is that the ablation study is not conducted 'due to time and space constraints.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (synthesized code outputs, test suite contents, per-task results) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 81 tasks from NumPy (36), scikit-learn (33), and SciPy (12) but does not describe how these specific APIs were selected from these large libraries, what criteria guided the selection, or why these particular counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved; the study uses benchmark tasks from public Python libraries."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from API selection to test generation to synthesis evaluation is described at a high level, but specifics are missing: how the 40-task training set was selected from the 81 tasks, whether any tasks were excluded during development, and what the complete evaluation pipeline looked like end-to-end."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: all three authors are from The University of Texas at Austin. The paper evaluates Google's Gemini models, and the authors are not affiliated with Google based on the stated affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper heavily features Google's Gemini products (Gemini 2.0, Gemini 2.5 Pro, Gemini CLI) without disclosing whether Google provided any support (API credits, model access, etc.)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Gemini 2.0 and Gemini 2.5 Pro to generate code for NumPy, SciPy, and scikit-learn APIs but does not state the training data cutoff dates for these models. The benchmark APIs exist in public libraries that are likely in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the target APIs' implementations might already exist in the Gemini models' training data. Since the benchmark tasks are real existing APIs from well-known libraries, the reference implementations are publicly available and likely in the training corpus."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses real APIs from NumPy, scikit-learn, and SciPy — all widely-used open-source libraries whose code is publicly available and almost certainly in Gemini's training data. This is a critical contamination concern that is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, tokens consumed, or wall-clock time is reported. The approach involves multiple LLM calls (APO iterations, RLVR training, test generation) but no cost information is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. RLVR fine-tuning and APO both require significant compute, but no GPU hours, API spend, or training time is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "APRIL achieves a 93.8% success rate in synthesizing correct, test-passing APIs across 81 tasks.",
       "evidence": "Table 1 shows 76 out of 81 tasks pass validation tests: 35/36 NumPy, 30/33 scikit-learn, 11/12 SciPy (Section 4.2, RQ1).",
       "supported": "moderate"
     },
     {
       "claim": "APRIL substantially outperforms baseline LLMs using manually engineered prompts, with improvements of 16.6%, 14.9%, and 16.7% across three benchmarks.",
       "evidence": "Table 2 compares APRIL (93.8% overall) against baseline Gemini 2.0 with manual prompt (77.8% overall), showing improvements per benchmark (Section 4.2, RQ2).",
       "supported": "moderate"
     },
     {
       "claim": "All LLM-generated APIs built and executed successfully (100% executability).",
       "evidence": "Table 1 shows 81/81 executability across all benchmarks (Section 4.2, RQ1).",
       "supported": "moderate"
     },
     {
       "claim": "Gemini-cli generates on average 8.1 tests per API and requires 2.2 iterations to converge to a comprehensive test suite.",
       "evidence": "Table 3 reports test counts and iteration counts per benchmark (Section 4.2, RQ3).",
       "supported": "moderate"
     },
     {
       "claim": "Integrating APO and RLVR provides a robust, scalable path for component-based API synthesis.",
       "evidence": "Only the combined system is evaluated; no ablation separates APO from RLVR contributions. The 81-task benchmark is relatively small for 'scalable' claims. No ablation study is provided (Section 4.2).",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "APRIL combines Automatic Prompt Optimization and Reinforcement Learning from Verifiable Rewards to synthesize APIs from scientific Python libraries (NumPy, scikit-learn, SciPy). On 81 benchmark tasks, APRIL achieves a 93.8% test pass rate compared to 77.8% for a baseline using the same Gemini 2.0 model with a manually engineered prompt. The paper also demonstrates that Gemini CLI can generate comprehensive test suites in an average of 2.2 iterations per API. However, no ablation study separates the contributions of APO and RLVR, and significant contamination concerns exist since the benchmark APIs are from public libraries likely in the model's training data.",
   "red_flags": [
     {
       "flag": "Severe benchmark contamination risk",
       "detail": "The benchmark tasks are real APIs from NumPy, scikit-learn, and SciPy — all widely-used open-source libraries whose complete source code is publicly available and almost certainly in Gemini's training data. The model may be recalling implementations rather than synthesizing them. This is never acknowledged or addressed."
     },
     {
       "flag": "No ablation study",
       "detail": "The paper explicitly acknowledges it does not isolate the contributions of its two main components (APO and RLVR). Without this, the claimed synergy cannot be verified — all gains could come from one component alone."
     },
     {
       "flag": "Single baseline comparison",
       "detail": "The only baseline is the same Gemini 2.0 model with a manually engineered prompt. No comparison against other LLMs, other code generation tools, or prior program synthesis systems (SyPet, FrAngel, EdSynth) discussed in related work."
     },
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "Results are single-run point estimates with no confidence intervals, significance tests, or variance across runs. On a small benchmark (81 tasks, 12 for SciPy), sampling variation could be substantial."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations section, no threats to validity, and no scope boundaries. This is a significant methodological omission."
     },
     {
       "flag": "Unclear train/test separation",
       "detail": "40 of 81 tasks are used for APO/RLVR training, but it is not clear whether the reported 93.8% pass rate is on held-out tasks only or includes training tasks. If training tasks are included, the reported performance is inflated."
     },
     {
       "flag": "Undisclosed relationship with Google products",
       "detail": "The paper heavily features Google's Gemini ecosystem (Gemini 2.0, Gemini 2.5 Pro, Gemini CLI) without disclosing whether Google provided any support, API credits, or early access. No funding or competing interests statement is present."
     }
   ],
   "cited_papers": [
     {
       "title": "Automatic prompt optimization with 'gradient descent' and beam search",
       "authors": ["R. Pryzant", "D. Iter", "J. Li", "Y. Lee", "C. Zhu", "M. Zeng"],
       "year": 2023,
       "relevance": "Foundational APO method used in APRIL; relevant to understanding LLM prompt optimization techniques."
     },
     {
       "title": "Tulu 3: Pushing frontiers in open language model post-training",
       "authors": ["N. Lambert", "J. Morrison", "V. Pyatkin"],
       "year": 2025,
       "relevance": "Describes RLVR methodology adopted in APRIL for fine-tuning LLMs with verifiable rewards."
     },
     {
       "title": "Jigsaw: Large language models meet program synthesis",
       "authors": ["N. Jain", "S. Vaidyanath", "A. Iyer", "N. Natarajan", "S. Parthasarathy", "S. Rajamani", "R. Sharma"],
       "year": 2022,
       "relevance": "Prior work on using LLMs for program synthesis; directly related to LLM-based code generation evaluation."
     },
     {
       "title": "Quality and trust in LLM-generated code",
       "authors": ["C. Spiess", "D. Gros", "K. S. Pai", "M. Pradel"],
       "year": 2024,
       "arxiv_id": "2402.02047",
       "relevance": "Studies quality and trust issues in LLM-generated code, directly relevant to evaluating code generation reliability."
     },
     {
       "title": "Are human rules necessary? Generating reusable APIs with CoT reasoning and in-context learning",
       "authors": ["Y. Mai", "Z. Gao", "X. Hu", "L. Bao", "Y. Liu", "J. Sun"],
       "year": 2024,
       "relevance": "Prior work on using LLMs to generate reusable APIs, directly comparable to APRIL's approach."
     },
     {
       "title": "An approach for API synthesis using large language models",
       "authors": ["H. Zhong", "S. Jiang", "S. Khurshid"],
       "year": 2025,
       "arxiv_id": "2502.15246",
       "relevance": "Predecessor work by the same authors on LLM-based API synthesis."
     },
     {
       "title": "Can large language models transform natural language intent into formal method postconditions?",
       "authors": ["M. Endres", "S. Fakhoury", "S. Chakraborty", "S. K. Lahiri"],
       "year": 2024,
       "relevance": "Evaluates LLM capabilities for formal specification generation, relevant to code generation evaluation methodology."
     },
     {
       "title": "Fuzz4all: Universal fuzzing with large language models",
       "authors": ["C. S. Xia", "M. Paltenghi", "J. Le Tian", "M. Pradel", "L. Zhang"],
       "year": 2024,
       "relevance": "Uses LLMs for test generation (fuzzing), relevant to LLM-based testing and code quality evaluation."
     },
     {
       "title": "On the effectiveness of large language models in writing Alloy formulas",
       "authors": ["Y. Hong", "S. Jiang", "Y. Fu", "S. Khurshid"],
       "year": 2025,
       "arxiv_id": "2502.15441",
       "relevance": "Evaluates LLM effectiveness for formal language generation, relevant to LLM code generation capability assessment."
     },
     {
       "title": "CASCADE: LLM-powered JavaScript deobfuscator at Google",
       "authors": ["S. Jiang", "P. Kovuri", "D. Tao", "Z. Tan"],
       "year": 2025,
       "arxiv_id": "2507.17691",
       "relevance": "Industrial application of LLMs for code transformation at Google, relevant to practical LLM-based SE tools."
     },
     {
       "title": "Generating executable oracles to check conformance of client code to requirements of JDK Javadocs using LLMs",
       "authors": ["S. Jiang", "C. Zhu", "S. Khurshid"],
       "year": 2024,
       "arxiv_id": "2411.01789",
       "relevance": "Uses LLMs for test oracle generation from documentation, directly related to the test generation component of APRIL."
     }
   ]
 }