ai-research-survey

scan-v5.json (27760B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Evaluating and Mitigating Errors in LLM-Generated Web API Integrations",
     "authors": [
       "Daniel Maninger",
       "Leon Chemnitz",
       "Amir Molzam Sharifloo",
       "Tushar Lamba",
       "Jannis Brugger",
       "Mira Mezini"
     ],
     "year": 2025,
     "venue": "arXiv / ACM Trans. Softw. Eng. Methodol.",
     "arxiv_id": "2509.20172",
     "doi": "XXXXXXX.XXXXXXX"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are backed by results: <40% for open-source models (Table 3), hallucination patterns quantified (15-39% illegal URLs), and the 90%/135% improvement figures come directly from Table 18a/c averages.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that constrained decoding improves correctness is supported by controlled comparisons on the same models and benchmark (constrained vs. unconstrained), which is adequate for this type of intervention study.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Scope is explicitly restricted throughout: JavaScript/Axios, OpenAPI specifications, 4 specific APIs, base (non-instruction-tuned) models, zero-shot prompting. Section 6 and the conclusion explicitly list what does not transfer.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes 'it remains to be investigated why the quantitative benefit of constrained decoding is so model-dependent' but does not systematically discuss alternative explanations for the main findings (e.g., whether prompt engineering could close the gap, or whether API prevalence in training data fully explains performance differences).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes what is measured (functional request configuration match) from what is claimed, explicitly separating executable vs. total metrics and noting that syntactic similarity is insufficient (Section 2.3).",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Threats to Validity' lists 6 numbered, substantive limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific: synthetic tasks use sparse optional parameters, constraints only act locally (variables used as parameter values are uncontrolled), constraints miss free-text description constraints, 4-API coverage may not generalize, lower executability under constrained decoding due to token/constraint misalignment.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope: JavaScript + Axios only, OpenAPI standard, 4 real-world APIs, base models only (instruction-tuned excluded), zero-shot prompting; conclusion explicitly states evaluation pipeline is specialized and cannot directly transfer.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments name three funders: Hessian Ministry (3AI cluster), ATHENE (Foundational Models for Secure Software Development), and LOEWE initiative with grant number.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed on the first page, including Leon Chemnitz's commercial affiliation with Pariton AI.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "All funders are government or academic bodies (Hessian Ministry, LOEWE, ATHENE); none have a stake in the benchmarked models or the constrained decoding approach evaluated.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests declaration. One author is at a commercial AI company (Pariton AI) but this is not addressed beyond the affiliation listing.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'Endpoint' is explicitly defined as 'unique combination of URL and HTTP method'; 'full completion' and 'argument completion' setups are defined; 'constrained decoding' is introduced with a formal description; request configuration components are specified.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four contributions are explicitly bulleted in the introduction: WAPIIBench dataset, open-source evaluation pipeline, OpenAPI-to-regex constraint generator, and novel empirical insights on correctness with and without constrained decoding.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 systematically distinguishes the work from five categories of related API work (general/domain-specific/SDK/local/tool APIs), contrasts with existing constrained decoding approaches (MGD, PICARD, Synchromesh, ToolDec), and explains why prior methods are unsuitable for this setting.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "WAPIIBench is publicly available on GitHub (github.com/stg-tud/WAPIIBench) and all model-generated codes are at Zenodo (doi:10.5281/zenodo.13758414).",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The 395-sample dataset and all model-generated outputs are available via GitHub and Zenodo respectively.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix D lists technology names (Hugging Face Transformers, Axios, axios-mock-adapter, regex) and hyperparameters (16-bit precision, temperature=0.0), but no requirements.txt, Dockerfile, or versioned dependency specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes the pipeline conceptually in detail but contains no step-by-step instructions for rerunning the evaluation; these would presumably be in the GitHub repo README.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are reported as point estimates. Greedy decoding (temperature=0.0) is used, so no CIs are reported; there is no uncertainty quantification over the 395-sample test set.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes comparative claims (e.g., 'constrained decoding significantly improves') without any statistical significance tests despite comparing across up to 21 models.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Relative percentage gains are reported throughout (e.g., '+90% average full completion', '+135% average argument completion') with absolute baseline values in Table 18.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 395 samples (one per API endpoint across 4 APIs) is justified pragmatically by coverage but no power analysis or statistical justification for this sample size is provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Greedy decoding eliminates run-to-run variance, but no variance across APIs, seeds, or other sources is reported; per-API breakdowns are only shown for two models.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Unconstrained models serve as baselines for constrained decoding comparisons; GPT-4o is included as an upper bound; models are compared against each other across two evaluation setups.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include GPT-4o, Qwen2.5-Coder, DeepSeek-Coder-V2, and Code Llama — all current models at time of writing, selected from coding leaderboards.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "The paper compares constrained vs. unconstrained and full vs. argument completion, but no ablation of constraint components (e.g., URL constraints only, argument constraints only) is performed.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 and Table 7 define 19+ metrics: correct/illegal implementations, correct/illegal URLs, correct/illegal methods, argument precision, recall, Jaccard index, value conditional accuracy, executability.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Human review was used for dataset construction (all 395 samples), not for evaluating LLM output quality. Functional correctness via automated execution is the evaluation method.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "All 395 samples are used as the full evaluation set with no held-out portion; the dataset was constructed by the authors, creating a risk that the constrained decoding approach could be inadvertently tuned to it.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results broken down by API (Asana, Google Calendar, Google Sheets, Slack) in Appendix Tables 12-17 for StarCoder2 and GPT-4o, and by metric type throughout.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 discusses specific failure modes: Qwen2.5-Coder refused to continue starter code, Llama 3.1 skipped the method part, URL hallucinations are quantified at 15-39%, and authorization argument inflation is discussed.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Multiple models (6 Qwen2.5-Coder variants, 2 Llama 3.1 variants) achieved 0% in full completion. Constrained decoding produces slightly lower executability rates. The model-size-performance relationship is non-monotonic.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Appendix E provides exact Hugging Face model IDs for all 24 evaluated models (e.g., 'bigcode/starcoder2-15b', 'deepseek-ai/deepseek-coder-6.7b-base', 'openai/gpt-4o').",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix F provides full prompts for both dataset generation (Listing 4, Gemini 1.5 Pro) and model evaluation (Listing 5, all evaluated models), with all fill-value placeholders described.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix E specifies: 16-bit floating point precision, 1 beam (greedy decoding), temperature=0.0.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "There is no agentic scaffolding; models generate single completions. The constrained decoding framework is described in detail, but this is the intervention, not scaffolding.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Dataset creation is thoroughly documented: Gemini 1.5 Pro generation, automated consistency checks (9 failures manually corrected), manual review of all 395 samples with 58 corrections, and specific criteria for task validity.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All model-generated codes and evaluation results are available via Zenodo (doi:10.5281/zenodo.13758414).",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 thoroughly describes dataset creation: API selection rationale, use of Gemini 1.5 Pro with full specifications, automated checks, and 3-criterion manual validation process.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants were recruited; the dataset was synthetically generated and curated by the authors.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Figure 1 shows the complete 4-stage pipeline (dataset creation → code generation → code execution → correctness analysis), and each stage is described in Sections 2.1-2.4 with the mock execution environment explained.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Model training cutoffs are not stated for any of the evaluated models, despite the evaluation relying on memorized API knowledge where cutoff is directly relevant.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper acknowledges that models 'rely solely on memorized knowledge' and that performance varies by API prevalence in training data, but does not formally discuss train/test overlap as a contamination concern.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "WAPIIBench is new, but the underlying API specifications are public and likely in training data; the paper does not address whether synthetic tasks based on publicly available specs could be partially memorized.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference latency or cost overhead for constrained decoding vs. unconstrained is reported; the paper notes performance optimizations are 'out of scope' but gives no runtime measurements.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget (GPU hours, cost) is stated for running 21+ models across 395 samples in 4 configurations.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "No evaluated open-source model solved more than 40% of full completion tasks; best is Code Llama (70B) at 30%.",
       "evidence": "Table 3 shows Code Llama (70B) at 0.30 correct implementations (t) for full completion; 6 model variants achieve 0%.",
       "supported": "strong"
     },
     {
       "claim": "Constrained decoding increases average correctness by ~90% (full completion) and ~135% (argument completion) relative to unconstrained baselines.",
       "evidence": "Table 18a shows average gain of +90% for full completion (t); Table 18c shows +135% for argument completion (t), excluding zero-baseline models.",
       "supported": "strong"
     },
     {
       "claim": "Constrained decoding eliminates all illegal URLs, HTTP methods, and arguments.",
       "evidence": "Tables 5, 6, 10, 11 all show 0.00 for illegal URLs, illegal methods, and illegal arguments under constrained decoding for all models.",
       "supported": "strong"
     },
     {
       "claim": "LLMs hallucinate endpoints and arguments: 15-39% illegal URLs and 6-31% illegal arguments in unconstrained evaluation.",
       "evidence": "Tables 3 and 4 show illegal URL rates (e) of 0.15-0.39 and illegal argument rates (e) of 0.06-0.25 across models.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o substantially outperforms open-source models (60% vs. 30% full completion correctness).",
       "evidence": "Table 3: GPT-4o achieves 0.60 correct implementations (t) vs. best open-source Code Llama (70B) at 0.30.",
       "supported": "strong"
     },
     {
       "claim": "Larger models within a family are not consistently better; medium-sized variants sometimes underperform both smaller and larger variants.",
       "evidence": "Section 3.2 notes this in DeepSeek-Coder, Qwen2.5-Coder, and Code Llama families, visible in Tables 8-9.",
       "supported": "moderate"
     },
     {
       "claim": "Constrained decoding makes open-source Code Llama (70B) competitive with GPT-4o mini (46% vs. 39% full completion).",
       "evidence": "Table 5: Code Llama (70B) constrained = 0.46; Table 3: GPT-4o mini = 0.39 unconstrained. Direct comparison is valid since GPT-4o mini cannot be constrained.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "LLMs struggle significantly with web API invocation code generation: the best open-source model (Code Llama 70B) achieves only 30% correctness on full completion tasks, while GPT-4o reaches 60%. Primary failure modes are endpoint URL hallucination (15-39% illegal URLs) and incorrect argument usage (6-31% illegal arguments). Constrained decoding derived automatically from OpenAPI specifications eliminates all illegal outputs and yields average correctness gains of ~90% (full completion) and ~135% (argument completion) across 21 open-source models, enabling mid-size models to approach commercial model performance without retraining or RAG.",
   "red_flags": [
     {
       "flag": "Fully synthetic benchmark",
       "detail": "All 395 tasks were generated by Gemini 1.5 Pro from API specifications; the paper acknowledges that synthetic tasks use sparse optional parameters and placeholder values, limiting real-world transferability."
     },
     {
       "flag": "Only 4 APIs",
       "detail": "Coverage limited to Asana, Google Calendar, Google Sheets, and Slack. Per-API breakdowns (Tables 12-17) reveal high variance in model performance (e.g., Google Calendar consistently best), but generalization to other APIs is unvalidated."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims ('significantly improves') are made without hypothesis tests. With 395 samples and small absolute differences between some models, some comparisons may not be statistically distinguishable."
     },
     {
       "flag": "Average gain excludes zero-baseline models",
       "detail": "The '90%' average gain explicitly excludes models that achieved 0% unconstrained (Qwen2.5-Coder, Llama 3.1 variants shown as '+inf%'). Including these would inflate the reported average; the selection criterion biases the headline number."
     },
     {
       "flag": "Base models only",
       "detail": "Instruction-tuned models (the typical practitioner choice) are excluded due to parsing difficulties with their outputs. Results may substantially underestimate capability of deployed models."
     },
     {
       "flag": "No latency or cost reporting",
       "detail": "Constrained decoding has real overhead (timeout errors appear in Tables 10-11), but no inference time or cost comparison is provided, which is critical for practical adoption assessment."
     },
     {
       "flag": "Training cutoffs not stated",
       "detail": "API specifications are public and likely in training data; without stating training cutoffs, the degree of memorization vs. generalization cannot be assessed."
     }
   ],
   "cited_papers": [
     {
       "title": "Monitor-Guided Decoding of Code LMs with Static Analysis of Repository Context",
       "relevance": "Key related constrained decoding approach for local method APIs; paper explicitly contrasts its web API approach against MGD's scope and capabilities."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (Codex/HumanEval)",
       "relevance": "Foundational benchmark paper for code LLM evaluation; cited for functional testing evaluation methodology."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "relevance": "One of the primary open-source code models evaluated in WAPIIBench."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "relevance": "Key open-source code model family evaluated; best open-source full-completion performance comes from Code Llama, but DeepSeek is extensively compared."
     },
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "relevance": "Closely related work on LLMs using APIs with RAG; the paper positions constrained decoding as providing correctness guarantees that RAG cannot."
     },
     {
       "title": "API-Bank: A Comprehensive Benchmark for Tool-Augmented LLMs",
       "relevance": "Related tool-use benchmark; contrasted with WAPIIBench's focus on REST API invocation code rather than tool-calling interfaces."
     },
     {
       "title": "Berkeley Function Calling Leaderboard",
       "relevance": "Contemporary benchmark for API/function calling; cited as related evaluation infrastructure with different scope (tool APIs vs. REST integration code)."
     },
     {
       "title": "Efficient Guided Generation for Large Language Models",
       "relevance": "Key constrained decoding framework; cited as production-quality alternative to authors' custom implementation."
     },
     {
       "title": "Bugs in large language models generated code: an empirical study",
       "relevance": "Empirical evidence for hallucination in LLM code generation; corroborates WAPIIBench findings about function/argument hallucination."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "relevance": "Open-source code model family evaluated; showed unexpected 0% performance in full completion despite strong leaderboard results."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "API integration is ubiquitous in software development; the benchmark (GitHub) and constraint generator (OpenAPI → regex) are directly usable by practitioners building coding assistants."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Multiple results are surprising: some Qwen/Llama models achieve 0% despite leaderboard rankings; larger models aren't consistently better; constrained decoding yields 90-135% gains without any model modification."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Security of LLM-generated code is briefly mentioned in the introduction, but the paper doesn't focus on security risks of hallucinated API calls."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The finding that popular models (Qwen2.5-Coder, Llama 3.1) fail completely on full completion despite strong benchmarks is mildly provocative but not framed confrontationally."
     },
     "demo_ability": {
       "score": 3,
       "justification": "WAPIIBench is available on GitHub and can be run against any Hugging Face model; practitioners can immediately test their models against the benchmark."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "TU Darmstadt and hessian.AI are not globally famous labs; GPT-4o is a famous product being evaluated but not created here."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45381333",
         "title": "Federation of Agents: Semantics-Aware, Large-Scale Communication Fabric",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45381333",
         "created_at": "2025-09-26T01:02:53Z"
       }
     ],
     "top_points": 3,
     "total_points": 3,
     "total_comments": 0
   }
 }