ai-research-survey

scan-v4.json (32806B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Evaluating and Mitigating Errors in LLM-Generated Web API Integrations",
     "authors": [
       "Daniel Maninger",
       "Leon Chemnitz",
       "Amir Molzam Sharifloo",
       "Tushar Lamba",
       "Jannis Brugger",
       "Mira Mezini"
     ],
     "year": 2025,
     "venue": "arXiv (accepted to ACM TOSEM)",
     "arxiv_id": "2509.20172",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'none of the evaluated open-source models was able to solve more than 40% of the tasks' is supported by Tables 3-4. Claims of '+90% and +135%' gains are supported by Figure 5 and Tables 18a-d.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that constrained decoding improves correctness is justified by controlled comparison: same models, same dataset, same prompts, with only the decoding strategy changed. This is a valid single-variable manipulation.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly states scope: JavaScript, Axios, OpenAPI-compliant APIs. Section 6 (Limitations) discusses that results may not generalize to other APIs given the limited number of APIs, and that prompt engineering could improve individual model results.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses multiple alternative explanations: API specification quality issues, prompt sensitivity, dataset quality, variable value limitations of constraints. Section 3.2 discusses training data prevalence as explanation for per-API performance differences.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures request configuration correctness (URL, method, arguments matching ground truth) and clearly frames this as correctness of API invocation code, not broader code quality or developer productivity. The measurements match the claims.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Threats to Validity' contains 6 numbered, substantive limitation discussions.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific: dataset may contain faulty samples, Gemini-generated tasks use optional parameters sparingly, constraints act only locally and cannot handle variable values, executability rate slightly lower with constrained models due to token misalignment (Section 6, items 1-6).",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly states scope: OpenAPI-compliant web APIs, JavaScript/Axios, code completion tasks. Section 6 item 5 states the work is 'not intended for highly exploratory development scenarios.' Limitation 3 notes results 'may not generalize to other APIs.'",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists funding from Hessian Ministry of Higher Education (3AI cluster), National Research Center ATHENE, and LOEWE initiative.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: TU Darmstadt, hessian.AI, Pariton AI, and ATHENE. Leon Chemnitz's affiliation with Pariton AI (a company) is disclosed.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from public research institutions (Hessian Ministry, LOEWE, ATHENE) with no apparent financial interest in any specific LLM's performance.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Leon Chemnitz is affiliated with Pariton AI but no financial interests declaration is made.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines 'endpoint' (URL + HTTP method combination), 'constrained decoding,' and the two evaluation setups (full vs. argument completion); it also provides a taxonomy distinguishing five API invocation categories in Section 7.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four explicit contributions are listed: WAPIIBench dataset, evaluation pipeline, constraint generator translating OpenAPI specs to regex constraints, and empirical insights on LLM performance with and without constrained decoding.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 systematically distinguishes this work from five categories of related work (domain-specific APIs, SDK-wrapped APIs, local function APIs, tool APIs, constrained decoding for code), explaining specific differences rather than just listing citations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "GitHub repository provided: https://github.com/stg-tud/WAPIIBench (Section 1, footnote 3). Zenodo artifact also provided: https://doi.org/10.5281/zenodo.13758414 (footnote 4).",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The WAPIIBench dataset is available on GitHub and all model-generated codes are provided in the Zenodo artifact (Section 1, Appendix A).",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix D lists technologies used (Hugging Face Transformers, Axios, etc.) and Appendix E lists hyperparameters (fp16, 1 beam, temperature 0), but no requirements.txt, Dockerfile, or detailed environment setup with library versions is mentioned.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are described in the paper. The GitHub repo is referenced but no README or reproduction guide is mentioned.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are reported as point estimates (e.g., '30% correctness', '0.75 precision') with no confidence intervals or error bars.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims constrained decoding 'significantly improves' correctness but provides no statistical significance tests. Comparisons between models and between constrained/unconstrained are based solely on comparing numbers.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Relative correctness gains are reported with baseline context: e.g., '+90% average gain' for full completion, '+135%' for argument completion, with specific per-model breakdowns from baseline to constrained performance (Figure 5, Tables 18a-d).",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The dataset has 395 samples (one per endpoint across 4 APIs). No justification is given for why these 4 APIs or this sample size is sufficient.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Greedy decoding (temperature=0) is used, producing deterministic single-run results. No variance across runs is reported because there is only one run per model. This means results could be sensitive to prompt wording or other factors without any measure of stability.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Unconstrained LLM generation serves as the baseline for evaluating constrained decoding. Multiple models across families are compared (Section 3, Tables 3-4).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Models include recent families: DeepSeek-Coder-V2, Qwen2.5-Coder, Llama 3.1, GPT-4o, and GPT-4o mini, all contemporary at time of writing.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "No ablation study of the constraint components. The paper does not test, e.g., constraining only URLs vs. only arguments vs. all, to measure individual contribution of each constraint type.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple fine-grained metrics are used: correct implementations, correct URLs, illegal URLs, correct methods, argument precision, argument recall, argument value conditional accuracy, and more (Table 2).",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "Evaluation is entirely automated via the mock execution pipeline. No human evaluation of generated code quality beyond the initial dataset curation.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "The entire dataset of 395 samples is used for evaluation. There is no train/dev/test split since the models are not fine-tuned, but the same dataset is used for both analysis and reporting, with no held-out portion.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by API (Tables 12-17), by model family, and by completion setup (full vs. argument). Per-model-family analysis is provided.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 discusses specific failure patterns: Qwen2.5-Coder refusing to continue starter code, Llama 3.1 skipping method parts, hallucinated endpoints and arguments. Error taxonomy is described in RQ1 answer.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that larger models are not always better within model families (Section 3.2), that constrained decoding slightly reduces executability rates (Section 5.2, Limitation 6), and that some models performed worse than expected.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Appendix E provides exact HuggingFace model identifiers for all open-source models (e.g., 'bigcode/starcoder2-15b', 'deepseek-ai/deepseek-coder-6.7b-base'). GPT-4o and GPT-4o mini are identified by marketing names with footnote links but no snapshot dates.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full prompts are provided: dataset generation prompt in Listing 4 (Appendix F) and code generation prompt in Listing 5 (Appendix F).",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix E states: fp16, 1 beam, temperature 0.0. Section 3 states greedy decoding for all experiments.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The approach is direct code generation with optional constrained decoding.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 describes the full dataset creation pipeline: Gemini 1.5 Pro generation, automated consistency checks (9 samples failed), manual review of all 395 samples (58 samples had issues and were fixed), with specific criteria for what was checked.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All model-generated codes are provided in the Zenodo artifact (Appendix A), and the dataset is on GitHub. This enables independent verification of the results.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 details dataset creation: 4 APIs selected (Asana, Google Calendar, Google Sheets, Slack), one task per endpoint (395 total), generated with Gemini 1.5 Pro with full specification and detailed prompt.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data source is synthetic dataset generation from API specifications.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 documents: Gemini generation → automated consistency checks (9 failures) → manual inspection of all 395 samples → 58 samples fixed. Section 2.3 describes the execution pipeline including mock environment and code truncation heuristics.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper evaluates models' memorized knowledge of API specifications but does not state training cutoff dates for any of the evaluated models.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.2 explicitly discusses that models rely on 'memorized knowledge about the APIs' from training data. Section 3.2 notes models 'have had training exposure to API specifications or examples from which they memorized usage patterns.' The synthetic nature of the dataset mitigates direct overlap.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "While the tasks are synthetically generated (reducing contamination risk), the paper does not discuss whether the underlying API specifications or similar API usage examples existed in training data. The APIs used (Slack, Google Calendar, etc.) are extremely well-documented online.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, latency, or tokens consumed is reported for any of the 21 models evaluated, despite constrained decoding having significant computational overhead.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total compute budget, GPU hours, or hardware specifications are stated. Running 21 models on 395 tasks in two setups (constrained and unconstrained) represents significant compute that is not quantified.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "Greedy decoding (temperature=0) produces deterministic results, so seed sensitivity is not applicable in the traditional sense. However, the paper does not test sensitivity to prompt variations or other sources of variance.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": true,
           "justification": "Implicitly stated: greedy decoding with temperature=0 means each experiment is a single deterministic run. This is clear from Appendix E hyperparameters.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameter search was conducted. The paper uses a single prompt and greedy decoding for all models. While this is a deliberate design choice for fairness, the paper acknowledges in Limitation 3 and Section 3.2 that prompt engineering could change results.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses identical configuration for all models (same prompt, greedy decoding) to ensure fair comparison. No configuration selection was performed.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "No statistical tests are performed at all, so multiple comparison correction does not arise. However, the paper makes many implicit comparisons across 21 models without any statistical framework.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors propose constrained decoding and evaluate it themselves. No independent evaluation or acknowledgment of author-evaluation bias is provided.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "Constrained decoding adds computational overhead (token-by-token constraint checking, timeouts noted in Tables 10-11) but performance is not reported as a function of compute. No cost comparison between constrained and unconstrained generation.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": true,
           "justification": "Section 2 discusses what WAPIIBench measures (functional correctness of API invocations via mock execution) vs. limitations (Section 6 items 1-3: simplified program context, no response handling, synthetic tasks). The paper is honest about what the benchmark does and doesn't capture.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is involved. Models generate code directly via completion.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The APIs used (Slack, Google Calendar, etc.) have extensive documentation online predating all models' training. The paper does not discuss whether models saw API documentation or usage examples during training, despite this being central to the evaluation.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": true,
           "justification": "The evaluation explicitly does NOT provide API specifications to models under test (Section 2.2): models must rely on memorized knowledge. The paper clearly separates what information models receive (task description only) from what they do not (specifications).",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Tasks within the same API share structural patterns (same base URL, similar argument types). No discussion of whether per-API correlation affects aggregate results.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No leakage detection method is applied. The paper does not check whether the specific task formulations or API usage patterns appeared in model training data.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "No evaluated open-source LLM solves more than 40% of web API invocation tasks (full completion setup)",
       "evidence": "Table 3: best open-source model Code Llama 70B achieves 30% correct implementations (t); GPT-4o achieves 60%",
       "supported": "strong"
     },
     {
       "claim": "Constrained decoding improves overall correctness by ~90% (full completion) and ~135% (argument completion) on average",
       "evidence": "Table 18a shows average unconstrained 9% → constrained 22% (+90%); Table 18c shows 22% → 41% (+135%), excluding models with 0% baseline",
       "supported": "strong"
     },
     {
       "claim": "Constrained decoding eliminates all illegal URLs, HTTP methods, and arguments (reduces to 0%)",
       "evidence": "Tables 5 and 6 show illegal URLs (e) = 0.00 and illegal methods (e) = 0.00 for all constrained models; Table 11 shows illegal arguments (e) = 0.00",
       "supported": "strong"
     },
     {
       "claim": "LLMs frequently hallucinate endpoint URLs and parameter names rather than failing to understand task structure",
       "evidence": "Section 3.2: 14-39% of URLs are illegal (e) in full completion; 6-31% of arguments are illegal (e) in argument completion; models show higher method accuracy (84-91%) than URL accuracy (32-72%)",
       "supported": "strong"
     },
     {
       "claim": "Larger models are not always better for web API code generation",
       "evidence": "Section 3.2 observes that medium-sized variants in DeepSeek-Coder, Qwen2.5-Coder, and Code Llama families perform worse than both smaller and larger variants",
       "supported": "moderate"
     },
     {
       "claim": "Constrained decoding makes mid-size open-source models competitive with larger or commercial models",
       "evidence": "Section 5.2: Code Llama 70B with constrained decoding matches GPT-4o mini performance",
       "supported": "moderate"
     },
     {
       "claim": "Constrained decoding slightly reduces executability rates compared to unconstrained generation",
       "evidence": "Comparing Tables 8 vs 10 and 9 vs 11: unconstrained executability typically 94-99%, constrained 80-94%, due to token-constraint misalignment",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "WAPIIBench reveals that state-of-the-art open-source LLMs struggle severely with web API invocation code generation, with the best open-source model achieving only 30% correct implementations in the full completion setting, primarily due to hallucinated endpoint URLs and illegal argument names. Constrained decoding using regex-based constraints automatically derived from OpenAPI specifications eliminates all hallucinated (illegal) endpoints and arguments by construction, yielding average relative correctness gains of 90% (full completion) and 135% (argument completion) across models. The technique is especially effective for IDE-style code completion where partial code context is available, and makes mid-size open-source models competitive with GPT-4o mini. The approach provides hard guarantees on specification compliance rather than stochastic improvements from RAG or fine-tuning, at the cost of a small reduction in code executability.",
   "red_flags": [
     {
       "flag": "Training data contamination unaddressed",
       "detail": "The paper evaluates models on 4 well-known, publicly documented APIs (Asana, Google Calendar, Slack, Google Sheets) whose OpenAPI specifications are likely present in training data. The paper itself notes models appear to have memorized API specs, yet never discusses training cutoffs or the possibility that results reflect recall rather than generalization."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All performance comparisons between models and conditions are presented as point estimates without confidence intervals or significance tests, making it impossible to determine whether observed differences are meaningful or due to chance."
     },
     {
       "flag": "Benchmark generalizability limited",
       "detail": "Only 4 APIs are included, and the limitations section notes results may not generalize to other APIs; per-API breakdowns (Tables 12-17) show large within-model variance (e.g., StarCoder2: 46% on Google Calendar vs 13% on Slack for full completion)."
     },
     {
       "flag": "Synthetic benchmark circularity risk",
       "detail": "The dataset was generated by Gemini 1.5 Pro and then manually reviewed; Gemini is excluded from evaluation 'as it generated the dataset.' This is handled correctly but the synthetic nature limits ecological validity — tasks use placeholder values and sparse optional parameters."
     },
     {
       "flag": "+inf% gain inflation",
       "detail": "Several models with 0% unconstrained baseline show '+inf%' gains under constrained decoding; these are excluded from the average but represent a substantial fraction (8/21 models for full completion), which may inflate the practical significance of the approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Foundational benchmark for LLM code generation; establishes the execution-based evaluation paradigm this work extends to web API invocations"
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "relevance": "One of the primary evaluated open-source code models; baseline for comparison"
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "relevance": "Key evaluated model family; represents code-specialized LLMs"
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "relevance": "Key evaluated model family with unexpectedly poor full-completion performance"
     },
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "relevance": "Related work on LLM API usage combining RAG and fine-tuning; direct point of comparison for the alternative approaches discussed"
     },
     {
       "title": "Monitor-Guided Decoding of Code LMs with Static Analysis of Repository Context",
       "relevance": "Most closely related constrained decoding work for code generation; differs by targeting local method calls rather than web API invocations"
     },
     {
       "title": "Guiding LLMs The Right Way: Fast, Non-Invasive Constrained Generation",
       "relevance": "Core constrained decoding framework referenced as compatible alternative engine for the constraint implementation"
     },
     {
       "title": "PICARD: Parsing Incrementally for Constrained Auto-Regressive Decoding from Language Models",
       "relevance": "Directly analogous constrained decoding approach applied to SQL generation; establishes precedent for the approach"
     },
     {
       "title": "What's Wrong with Your Code Generated by Large Language Models? An Extensive Study",
       "relevance": "Prior characterization of LLM code errors including hallucinations; confirms findings extend to web API domain"
     },
     {
       "title": "Out of the BLEU: How should we assess quality of the Code Generation models?",
       "relevance": "Motivation for functional evaluation over syntactic similarity metrics used in this benchmark"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses a common developer pain point (LLMs generating wrong API calls) with a deployable, implementation-agnostic solution that works in IDE code completion workflows."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that LLMs struggle with API code is expected; the magnitude of constrained decoding improvement (+90-135%) is notable but not dramatically counterintuitive."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Paper briefly notes that directly executing LLM-generated API code is unsafe as it could send arbitrary requests to external servers, motivating the sandboxed evaluation environment."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial findings or conflict with established results; straightforward benchmark paper with constructive improvement approach."
     },
     "demo_ability": {
       "score": 2,
       "justification": "WAPIIBench and constraint generator are publicly available on GitHub; practitioners can apply constrained decoding to their own OpenAPI specs using the released code."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "TU Darmstadt and Hessian Center for AI are recognized academic institutions but not top-tier industry labs; no famous product affiliation."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45381333",
         "title": "Federation of Agents: Semantics-Aware, Large-Scale Communication Fabric",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45381333",
         "created_at": "2025-09-26T01:02:53Z"
       }
     ],
     "top_points": 3,
     "total_points": 3,
     "total_comments": 0
   }
 }