ai-research-survey

scan.json (20438B)

---

 {
   "paper": {
     "title": "Automated Program Repair Based on REST API Specifications Using Large Language Models",
     "authors": ["Katsuki Yamagishi", "Norihiro Yoshida", "Erina Makihara", "Katsuro Inoue"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.25148",
     "doi": "10.48550/arXiv.2510.25148"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A Zenodo archive is provided (https://zenodo.org/records/16556822), referenced in the introduction as a footnote containing the dcFix implementation and misuse examples."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the misuse examples are 'published them on the website' alongside the dcFix implementation at the Zenodo archive link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or library versions are mentioned anywhere in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the experiments are provided. The paper describes the method conceptually but gives no commands or procedure to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw counts (e.g., 13/19, 6/8) with no confidence intervals or error bars despite the small sample sizes."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims dcFix achieves higher fix rates than the baseline but provides no statistical significance tests. Comparisons are based solely on raw counts."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw fix counts are reported (e.g., 13/19 vs 1/19). No effect sizes, odds ratios, or percentage improvements with baseline context are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The total dataset is 31 misuse cases (21 for Philips Hue, 10 for SwitchBot). No justification for this sample size is given, and no discussion of whether it is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The LLM is run 5 times per prompt and a case is considered successful 'if at least one attempt produced a correct result.' No variance across the 5 runs is reported — only the binary success/fail outcome."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A baseline approach is included: prompts to the LLM that omit unsatisfied specifications or deviation points. Results are compared in Table III."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baseline is 'prompts without specification information.' No comparison against any existing automated repair tool, LLM-based or otherwise, is included. The related work discusses tools like MUDetect and InferFix but none are compared against."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "dcFix has two key components: (1) deviation point detection and (2) prompt augmentation with unsatisfied specifications. The baseline omits both, but there is no ablation testing each component individually (e.g., including only deviation points without unsatisfied specifications, or vice versa)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only one metric is reported: fix rate (number of successfully repaired cases). No other metrics such as code quality, time to fix, or partial correctness are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated fixes is described. Correctness appears to be determined by the authors but no formal evaluation protocol is documented."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no separation of development and test data. The entire dataset of 31 cases is used for evaluation, and presumably the prompts were developed using the same or similar examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables II and III provide per-category breakdowns by API (Philips Hue vs SwitchBot) and by misuse type (Endpoint, Request Headers, Request Body)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Summary section mentions that 'there were cases where the LLM failed to generate correct fixes because it had not been trained on the latest REST API specifications.' However, this is a brief mention rather than detailed error analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that dcFix failed to detect some deviations (2/21 for Philips Hue, 2/10 for SwitchBot) and failed to fix some detected deviations (6/19 for Philips Hue, 2/8 for SwitchBot). These are honestly reported in Tables II and III."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims dcFix 'accurately detects misuse and outperforms the baseline approach.' Tables II and III support these claims with 27/31 detection rate and higher fix rates than baseline (19/27 vs 2/27)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implicitly claims that including deviation points and unsatisfied specifications in prompts causes better repair outcomes. However, the comparison is only between full prompt vs no-specification prompt, with no control for confounds (e.g., prompt length, additional context in general)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Automated Program Repair Based on REST API Specifications' broadly, but evaluation is limited to only two APIs (SwitchBot and Philips Hue) with 31 total cases. No discussion of whether results generalize to other REST APIs or programming languages beyond Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The improvement could be partly due to the additional context in the prompt rather than the specific deviation point / specification information. This is not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper never specifies which LLM was used. It refers only to 'an LLM' or 'the LLM' throughout without naming the model, version, or provider."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Listing 2 shows the prompt template used for repair instructions. While it contains a placeholder '${target program}', the actual template text (the system instruction portion) is provided in full."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, etc.) are reported for the LLM. The paper states the LLM was run 5 times per prompt but provides no sampling configuration."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-step pipeline (S1: deviation point detection via static analysis, S2: LLM-based repair) is described in Section III with Figure 1 providing an overview. The static analysis steps are detailed in Section III-A."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes how misuse cases were collected: SwitchBot cases from closed issues on GitHub, Philips Hue cases from commits involving the /clip/v2 endpoint. Selection criteria are stated, though briefly."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The only limitation mentioned is a brief remark in the Summary about LLM not being trained on latest specifications."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The brief mention of LLM knowledge staleness is a single sentence in the summary, not a substantive limitations discussion."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what settings are excluded, or what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo archive (referenced in footnote 1) is stated to contain the misuse examples alongside the implementation, allowing independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes data collection: SwitchBot cases from closed GitHub issues where API specifications were modified in commits; Philips Hue cases from repositories with /clip/v2 endpoint changes."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The dataset consists of code examples from public GitHub repositories, which is a standard data source not involving recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the sources but not the full pipeline. It states 'We manually extracted programs containing code fragments that were inconsistent with the specifications' but does not document how many candidates were considered, how many were filtered, or the specific criteria for inclusion/exclusion beyond 'violated API specifications.'"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed with their affiliation at Ritsumeikan University with email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funding cannot be assessed. The absence of disclosure does not establish that the work is unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses an LLM to generate fixes but never states which LLM is used, let alone its training data cutoff. The Summary section acknowledges 'the LLM failed to generate correct fixes because it had not been trained on the latest REST API specifications,' implying awareness of the issue but providing no concrete cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLM may have seen the test code or API specifications during training. The misuse cases come from public GitHub repositories which are likely in LLM training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The misuse examples come from public GitHub issues and commits which are very likely in LLM training corpora. This contamination risk is not discussed at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or latency is reported despite the method calling an LLM 5 times per misuse case."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, hardware specifications, or total API spend is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "dcFix accurately detects deviation points from REST API specifications, identifying 19/21 for Philips Hue and 8/10 for SwitchBot.",
       "evidence": "Table II shows detection results: 12/14 endpoints, 7/7 request bodies for Philips Hue; 5/5 endpoints, 2/4 request headers, 1/1 request body for SwitchBot (Section IV-B).",
       "supported": "moderate"
     },
     {
       "claim": "dcFix achieves higher fix rates compared to the baseline approach for both APIs.",
       "evidence": "Table III shows dcFix fixed 13/19 vs baseline 1/19 for Philips Hue, and 6/8 vs 1/8 for SwitchBot. However, no statistical tests were applied and the sample sizes are very small.",
       "supported": "moderate"
     },
     {
       "claim": "Including deviation point and unsatisfied specification information in LLM prompts leads to better repair outcomes than omitting them.",
       "evidence": "Table III comparison between dcFix and baseline. The difference is substantial in raw numbers (19/27 vs 2/27 total) but lacks statistical testing, and no confounds are controlled.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "dcFix is a method for detecting and automatically repairing REST API misuses in client programs by combining static analysis of deviation points with LLM-based repair. Evaluated on 31 misuse cases from SwitchBot and Philips Hue APIs, dcFix detected 27 out of 31 deviations and repaired 19 of the 27 detected cases, compared to only 2/27 for the baseline approach that omits specification information from prompts. The study demonstrates that enriching LLM prompts with specific violation context improves repair effectiveness, though the evaluation is limited to two APIs and a small dataset.",
   "red_flags": [
     {
       "flag": "Unidentified LLM",
       "detail": "The paper never names which LLM was used. This is a critical omission — different models have vastly different capabilities, and the results cannot be replicated or interpreted without knowing the model."
     },
     {
       "flag": "Very small sample size",
       "detail": "Only 31 misuse cases total across two APIs. Some subcategories have as few as 1 case (SwitchBot Request Body). The results are too sparse to draw general conclusions."
     },
     {
       "flag": "No statistical analysis",
       "detail": "Despite claiming dcFix 'outperforms' the baseline, no statistical tests are applied. With such small samples, the observed differences could be due to chance."
     },
     {
       "flag": "Weak baseline",
       "detail": "The only baseline is 'no specification information in the prompt.' No comparison against existing APR tools mentioned in the related work (MUDetect, InferFix, etc.)."
     },
     {
       "flag": "Pass@5 metric without reporting per-run results",
       "detail": "The LLM is run 5 times and success is counted if at least one run succeeds. This inflates apparent success rates. Per-run success rates and variance are not reported."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section despite significant methodological weaknesses (small sample, two APIs only, unknown LLM, no statistical tests)."
     },
     {
       "flag": "Data contamination risk",
       "detail": "Misuse cases are from public GitHub repositories likely present in LLM training data. The LLM may already know the correct code, making the baseline comparison unreliable."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating automatic program repair capabilities to repair API misuses",
       "authors": ["M. Kechagia", "S. Mechtaev", "F. Sarro", "M. Harman"],
       "year": 2022,
       "doi": "10.1109/TSE.2021.3095706",
       "relevance": "Evaluates APR tools on API misuse repair, directly relevant to LLM-based program repair quality assessment."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Demonstrates LLM outperformance over traditional APR tools, foundational work for LLM-based repair evaluations."
     },
     {
       "title": "Inferfix: End-to-end program repair with LLMs",
       "authors": ["M. Jin", "S. Shahriar", "M. Tufano", "X. Shi", "S. Lu", "N. Sundaresan", "A. Svyatkovskiy"],
       "year": 2023,
       "relevance": "Combines static analysis with LLMs for program repair, directly comparable methodology to dcFix."
     },
     {
       "title": "RESTler: Stateful REST API fuzzing",
       "authors": ["V. Atlidakis", "P. Godefroid", "M. Polishchuk"],
       "year": 2019,
       "relevance": "REST API testing tool generating API call sequences from specifications, relevant to API quality and testing research."
     },
     {
       "title": "Boosting API misuse detection via integrating API constraints from multiple sources",
       "authors": ["C. Li", "J. Zhang", "Y. Tang", "Z. Li", "T. Sun"],
       "year": 2024,
       "relevance": "API misuse detection using multi-source constraints, relevant to automated code quality assessment."
     },
     {
       "title": "Generating REST API specifications through static analysis",
       "authors": ["R. Huang", "M. Motwani", "I. Martinez", "A. Orso"],
       "year": 2024,
       "relevance": "Static analysis for REST API specification generation, relevant to API conformance checking research."
     }
   ]
 }