ai-research-survey

scan.json (27632B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "Evaluating and Mitigating Errors in LLM-Generated Web API Integrations",
     "authors": ["Daniel Maninger", "Leon Chemnitz", "Amir Molzam Sharifloo", "Tushar Lamba", "Jannis Brugger", "Mira Mezini"],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2509.20172"
   },
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Open-source LLMs struggle significantly with generating correct web API invocation code, with the best open-source model (Code Llama 70B) achieving only 30% correctness on full completion tasks. Constrained decoding, which automatically translates OpenAPI specifications to regex-based constraints, eliminates all illegal URLs, methods, and arguments, improving overall correctness by 90-135% on average. The technique is particularly effective for argument completion (in-IDE) scenarios, making mid-size open-source models competitive with larger commercial models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/stg-tud/WAPIIBench (Section 1, footnote 3). Zenodo artifact also provided: https://doi.org/10.5281/zenodo.13758414 (footnote 4)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The WAPIIBench dataset is available on GitHub and all model-generated codes are provided in the Zenodo artifact (Section 1, Appendix A)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D lists technologies used (Hugging Face Transformers, Axios, etc.) and Appendix E lists hyperparameters (fp16, 1 beam, temperature 0), but no requirements.txt, Dockerfile, or detailed environment setup with library versions is mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The GitHub repo is referenced but no README or reproduction guide is mentioned."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '30% correctness', '0.75 precision') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims constrained decoding 'significantly improves' correctness but provides no statistical significance tests. Comparisons between models and between constrained/unconstrained are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative correctness gains are reported with baseline context: e.g., '+90% average gain' for full completion, '+135%' for argument completion, with specific per-model breakdowns from baseline to constrained performance (Figure 5, Tables 18a-d)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset has 395 samples (one per endpoint across 4 APIs). No justification is given for why these 4 APIs or this sample size is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding (temperature=0) is used, producing deterministic single-run results. No variance across runs is reported because there is only one run per model. This means results could be sensitive to prompt wording or other factors without any measure of stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Unconstrained LLM generation serves as the baseline for evaluating constrained decoding. Multiple models across families are compared (Section 3, Tables 3-4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include recent families: DeepSeek-Coder-V2, Qwen2.5-Coder, Llama 3.1, GPT-4o, and GPT-4o mini, all contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of the constraint components. The paper does not test, e.g., constraining only URLs vs. only arguments vs. all, to measure individual contribution of each constraint type."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple fine-grained metrics are used: correct implementations, correct URLs, illegal URLs, correct methods, argument precision, argument recall, argument value conditional accuracy, and more (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via the mock execution pipeline. No human evaluation of generated code quality beyond the initial dataset curation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The entire dataset of 395 samples is used for evaluation. There is no train/dev/test split since the models are not fine-tuned, but the same dataset is used for both analysis and reporting, with no held-out portion."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by API (Tables 12-17), by model family, and by completion setup (full vs. argument). Per-model-family analysis is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 discusses specific failure patterns: Qwen2.5-Coder refusing to continue starter code, Llama 3.1 skipping method parts, hallucinated endpoints and arguments. Error taxonomy is described in RQ1 answer."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that larger models are not always better within model families (Section 3.2), that constrained decoding slightly reduces executability rates (Section 5.2, Limitation 6), and that some models performed worse than expected."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'none of the evaluated open-source models was able to solve more than 40% of the tasks' is supported by Tables 3-4. Claims of '+90% and +135%' gains are supported by Figure 5 and Tables 18a-d."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that constrained decoding improves correctness is justified by controlled comparison: same models, same dataset, same prompts, with only the decoding strategy changed. This is a valid single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly states scope: JavaScript, Axios, OpenAPI-compliant APIs. Section 6 (Limitations) discusses that results may not generalize to other APIs given the limited number of APIs, and that prompt engineering could improve individual model results."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses multiple alternative explanations: API specification quality issues, prompt sensitivity, dataset quality, variable value limitations of constraints. Section 3.2 discusses training data prevalence as explanation for per-API performance differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures request configuration correctness (URL, method, arguments matching ground truth) and clearly frames this as correctness of API invocation code, not broader code quality or developer productivity. The measurements match the claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides exact HuggingFace model identifiers for all open-source models (e.g., 'bigcode/starcoder2-15b', 'deepseek-ai/deepseek-coder-6.7b-base'). GPT-4o and GPT-4o mini are identified by marketing names with footnote links but no snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided: dataset generation prompt in Listing 4 (Appendix F) and code generation prompt in Listing 5 (Appendix F)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E states: fp16, 1 beam, temperature 0.0. Section 3 states greedy decoding for all experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct code generation with optional constrained decoding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the full dataset creation pipeline: Gemini 1.5 Pro generation, automated consistency checks (9 samples failed), manual review of all 395 samples (58 samples had issues and were fixed), with specific criteria for what was checked."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Threats to Validity' contains 6 numbered, substantive limitation discussions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific: dataset may contain faulty samples, Gemini-generated tasks use optional parameters sparingly, constraints act only locally and cannot handle variable values, executability rate slightly lower with constrained models due to token misalignment (Section 6, items 1-6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly states scope: OpenAPI-compliant web APIs, JavaScript/Axios, code completion tasks. Section 6 item 5 states the work is 'not intended for highly exploratory development scenarios.' Limitation 3 notes results 'may not generalize to other APIs.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All model-generated codes are provided in the Zenodo artifact (Appendix A), and the dataset is on GitHub. This enables independent verification of the results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 details dataset creation: 4 APIs selected (Asana, Google Calendar, Google Sheets, Slack), one task per endpoint (395 total), generated with Gemini 1.5 Pro with full specification and detailed prompt."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is synthetic dataset generation from API specifications."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents: Gemini generation → automated consistency checks (9 failures) → manual inspection of all 395 samples → 58 samples fixed. Section 2.3 describes the execution pipeline including mock environment and code truncation heuristics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists funding from Hessian Ministry of Higher Education (3AI cluster), National Research Center ATHENE, and LOEWE initiative."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: TU Darmstadt, hessian.AI, Pariton AI, and ATHENE. Leon Chemnitz's affiliation with Pariton AI (a company) is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from public research institutions (Hessian Ministry, LOEWE, ATHENE) with no apparent financial interest in any specific LLM's performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Leon Chemnitz is affiliated with Pariton AI but no financial interests declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates models' memorized knowledge of API specifications but does not state training cutoff dates for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly discusses that models rely on 'memorized knowledge about the APIs' from training data. Section 3.2 notes models 'have had training exposure to API specifications or examples from which they memorized usage patterns.' The synthetic nature of the dataset mitigates direct overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the tasks are synthetically generated (reducing contamination risk), the paper does not discuss whether the underlying API specifications or similar API usage examples existed in training data. The APIs used (Slack, Google Calendar, etc.) are extremely well-documented online."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed is reported for any of the 21 models evaluated, despite constrained decoding having significant computational overhead."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or hardware specifications are stated. Running 21 models on 395 tasks in two setups (constrained and unconstrained) represents significant compute that is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding (temperature=0) produces deterministic results, so seed sensitivity is not applicable in the traditional sense. However, the paper does not test sensitivity to prompt variations or other sources of variance."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Implicitly stated: greedy decoding with temperature=0 means each experiment is a single deterministic run. This is clear from Appendix E hyperparameters."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search was conducted. The paper uses a single prompt and greedy decoding for all models. While this is a deliberate design choice for fairness, the paper acknowledges in Limitation 3 and Section 3.2 that prompt engineering could change results."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses identical configuration for all models (same prompt, greedy decoding) to ensure fair comparison. No configuration selection was performed."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction does not arise. However, the paper makes many implicit comparisons across 21 models without any statistical framework."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose constrained decoding and evaluate it themselves. No independent evaluation or acknowledgment of author-evaluation bias is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Constrained decoding adds computational overhead (token-by-token constraint checking, timeouts noted in Tables 10-11) but performance is not reported as a function of compute. No cost comparison between constrained and unconstrained generation."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 discusses what WAPIIBench measures (functional correctness of API invocations via mock execution) vs. limitations (Section 6 items 1-3: simplified program context, no response handling, synthetic tasks). The paper is honest about what the benchmark does and doesn't capture."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models generate code directly via completion."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The APIs used (Slack, Google Calendar, etc.) have extensive documentation online predating all models' training. The paper does not discuss whether models saw API documentation or usage examples during training, despite this being central to the evaluation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation explicitly does NOT provide API specifications to models under test (Section 2.2): models must rely on memorized knowledge. The paper clearly separates what information models receive (task description only) from what they do not (specifications)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Tasks within the same API share structural patterns (same base URL, similar argument types). No discussion of whether per-API correlation affects aggregate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection method is applied. The paper does not check whether the specific task formulations or API usage patterns appeared in model training data."
       }
     }
   },
   "claims": [
     {
       "claim": "None of the evaluated open-source models was able to solve more than 40% of the web API invocation tasks.",
       "evidence": "Tables 3-4 show Code Llama 70B achieving 30% (full completion) and 40% (argument completion) as the best open-source model.",
       "supported": "strong"
     },
     {
       "claim": "Constrained decoding increases overall correctness by 90% (full completion) and 135% (argument completion) on average.",
       "evidence": "Figure 5 and Tables 18a-d show per-model relative gains, with averages calculated across all models with non-zero baselines.",
       "supported": "strong"
     },
     {
       "claim": "Constrained decoding eliminates all illegal URLs, HTTP methods, and arguments.",
       "evidence": "Tables 5-6 and 10-11 show 0% illegal URLs, methods, and arguments across all models with constrained decoding.",
       "supported": "strong"
     },
     {
       "claim": "Code Llama 70B with constrained decoding matches the performance of GPT-4o mini.",
       "evidence": "Table 4 shows GPT-4o mini at 63% argument completion correctness; Table 6 shows Code Llama 70B constrained at 63%. Full completion: GPT-4o mini at 39%, Code Llama 70B constrained at 46% (Table 5).",
       "supported": "strong"
     },
     {
       "claim": "LLMs frequently hallucinate endpoint URLs and parameter names in web API invocations.",
       "evidence": "Tables 3-4 show 15-39% illegal URLs and 6-31% illegal arguments across models. Section 3.2 discusses hallucination patterns.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests despite comparative claims",
       "detail": "The paper claims constrained decoding 'significantly improves' correctness but provides no statistical significance tests. All comparisons are based on point estimates from single deterministic runs."
     },
     {
       "flag": "Synthetic benchmark may not reflect real-world API usage",
       "detail": "Section 6 acknowledges that Gemini-generated tasks 'use optional parameters rather sparingly and often use placeholder values instead of realistic example values, limiting the transferability of findings.' Tasks have limited program context and evaluate only outgoing requests."
     },
     {
       "flag": "No cost reporting for constrained decoding",
       "detail": "Constrained decoding adds significant computational overhead (Tables 10-11 show timeouts and unsatisfiable constraints), but no latency or cost comparison with unconstrained generation is provided. This is important for practical applicability."
     },
     {
       "flag": "Only 4 APIs tested",
       "detail": "Results are based on only 4 APIs (Asana, Google Calendar, Google Sheets, Slack). The paper acknowledges this in Limitation 3 but the small number of APIs limits generalizability claims about web API invocations broadly."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and functional correctness evaluation for code generation — foundational to the evaluation methodology used in this paper."
     },
     {
       "title": "What's Wrong with Your Code Generated by Large Language Models? An Extensive Study",
       "authors": ["Shihan Dou"],
       "year": 2024,
       "arxiv_id": "2407.06153",
       "relevance": "Studies errors in LLM-generated code including hallucinated function names and arguments, directly related to API hallucination findings."
     },
     {
       "title": "Monitor-Guided Decoding of Code LMs with Static Analysis of Repository Context",
       "authors": ["Lakshya A. Agrawal"],
       "year": 2023,
       "relevance": "Uses constrained decoding with static analysis to prevent method hallucination in Java — directly comparable approach for local function APIs."
     },
     {
       "title": "Guiding LLMs The Right Way: Fast, Non-Invasive Constrained Generation",
       "authors": ["Luca Beurer-Kellner"],
       "year": 2024,
       "relevance": "Proposes minimally invasive constrained decoding framework that could address executability issues noted in this paper's limitations."
     },
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "authors": ["Shishir G. Patil"],
       "year": 2024,
       "relevance": "Studies LLM API hallucination and uses fine-tuning and RAG to reduce it — complementary approach to the constrained decoding proposed here."
     },
     {
       "title": "Bugs in large language models generated code: an empirical study",
       "authors": ["Florian Tambon"],
       "year": 2025,
       "doi": "10.1007/S10664-025-10614-4",
       "relevance": "Empirical study of bugs in LLM-generated code showing syntax errors are a small part of errors — motivates need for semantic constraints."
     },
     {
       "title": "Copiloting the Copilots: Fusing Large Language Models with Completion Engines for Automated Program Repair",
       "authors": ["Yuxiang Wei"],
       "year": 2023,
       "relevance": "Uses constrained decoding with completion engines (Repilot) to prevent hallucinations in automated program repair."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce"],
       "year": 2022,
       "doi": "10.1109/SP46214.2022.9833571",
       "relevance": "Evaluates security vulnerabilities in AI-generated code, relevant to concerns about correctness in LLM code generation."
     },
     {
       "title": "The effects of generative AI on high-skilled work: Evidence from three field experiments with software developers",
       "authors": ["Zheyuan Kevin Cui"],
       "year": 2025,
       "relevance": "Field experiments measuring LLM impact on developer productivity — provides context for the practical importance of correct API code generation."
     },
     {
       "title": "On Mitigating Code LLM Hallucinations with API Documentation",
       "authors": ["Nihal Jain"],
       "year": 2025,
       "relevance": "Studies SDK-wrapped web API hallucination mitigation via RAG with CloudAPIBench — directly comparable approach for web API invocations."
     },
     {
       "title": "Towards Mitigating API Hallucination in Code Generated by LLMs with Hierarchical Dependency Aware",
       "authors": ["Yujia Chen"],
       "year": 2025,
       "relevance": "MARIN system uses constrained decoding and RAG for Java API hallucination prevention — related constrained decoding approach for local APIs."
     },
     {
       "title": "Applying RLAIF for Code Generation with API-usage in Lightweight LLMs",
       "authors": ["Sujan Dutta"],
       "year": 2024,
       "arxiv_id": "2406.20060",
       "relevance": "Uses RLAIF fine-tuning to improve API usage in code generation — complementary approach to constrained decoding."
     }
   ]
 }