ai-research-survey

scan.json (25220B)

---

 {
   "paper": {
     "title": "LIDL: LLM Integration Defect Localization via Knowledge Graph-Enhanced Multi-Agent Analysis",
     "authors": ["Gou Tan", "Zilong He", "Min Li", "Pengfei Chen", "Jieke Shi", "Zhensu Sun", "Ting Zhang", "Danwen Chen", "Lwin Khin Shar", "Chuanfu Zhang", "David Lo"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.05539",
     "doi": "10.48550/arXiv.2601.05539"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LIDL achieves 0.64 Top-3 accuracy and 0.48 MAP on 146 real-world LLM integration defects, outperforming the best baseline (AutoCodeRover) by 64.1%. It reduces cost by 92.5% compared to AutoCodeRover ($0.008 vs $0.106 per instance). Ablation shows annotation-based retrieval contributes most (-17.2% Top-3 when removed). The largest gains are in LLM System Management defects (+170.8% over best baseline).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All benchmark data and our implementation are publicly available at: https://github.com/IntelligentDDS/LIDL' (Section I)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset of 146 defect instances is stated as publicly available at the same GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A states 'Ubuntu 24.04, 64-core Intel Xeon Gold 6326 CPU, 128GB RAM, and 6 NVIDIA A40 48GB GPUs' and 'Python 3.10.16 with popular libraries.' Hardware is detailed but specific library versions are not enumerated."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The GitHub link is provided but no README content or reproduction scripts are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., Top-3: 0.64, MAP: 0.48). No confidence intervals, error bars, or uncertainty quantification is provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LIDL 'significantly outperforms' baselines but no statistical significance tests (p-values, t-tests, etc.) are reported. Comparisons are based solely on point metric differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., '64.1% improvement over AutoCodeRover (0.39)' and '120.7% over SWE-agent (0.29)' (Section IV-B)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 146 instances after filtering from two sources, but no justification is given for why this sample size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread across runs is reported. Results appear to be from single runs (temperature=0.0 is stated but no mention of multiple runs)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baselines are compared: SWE-agent, Agentless, AutoCodeRover, SWE-agent*, and Agentless* (Section IV)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent state-of-the-art methods: SWE-agent (NeurIPS 2024), Agentless (2024), AutoCodeRover (ISSTA 2024), RepoGraph (ICLR 2025). All are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-D (RQ3) presents a systematic ablation removing each of the four components individually (direct extraction, symptom inference, annotation retrieval, validator) with results in Table VI."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Top-1, Top-3, MAP, and MRR. Additionally, cost and token consumption are reported (Section IV-B, IV-C)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of LIDL's outputs is conducted. Evaluation is entirely automated against ground-truth patches. Human evaluation could assess whether the localization reports are useful to developers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Parameter settings were determined through a 'pilot study on 15 defects' (Section IV-A) but there is no explicit statement that these 15 were excluded from the final evaluation, or that a separate held-out test set was used."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 7 provides per-category breakdowns across all four defect categories (Prompt/Context, LLM Interface, Tool Integration, LLM System) for all methods."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The case study in Section IV-E only shows a success case. No failure analysis is provided — the 36% of instances where LIDL fails to locate the defect in Top-3 are not analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that adding RepoGraph to baselines actually hurts performance: 'SWE-agent* drops from 0.29 to 0.21 Top-3 (-27.6%)' (Section IV-B). This is a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 0.64 Top-3 accuracy, 0.48 MAP, 64.1% improvement, and 92.5% cost reduction are all supported by Tables V and the results in Section IV."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('each core component contributes meaningfully') are supported by the ablation study in Table VI, which systematically removes components. This is adequate for single-variable manipulation claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and abstract frame this as a general solution for 'LLM integration defects' but the evaluation is limited to Python applications from GitHub. The limitations section acknowledges Python-only support but the title/abstract do not bound this scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed improvements are discussed. For example, LIDL's advantage could partly stem from the specific composition of the benchmark rather than generalizable superiority."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures file-level localization accuracy (Top-k, MAP, MRR) and frames claims at that same granularity. It does not overreach to claim broader capabilities like 'defect repair' from localization metrics."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are listed: 'Llama3.3-70B-Instruct', 'Qwen2.5-72B-Instruct', 'DeepSeek-V3.2', 'Kimi-K2', 'GPT-5.1', 'Claude-Sonnet-4.5', and 'BGE-M3' (Section IV-A)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described in natural language (e.g., 'The LLM is prompted to perform three reasoning steps') but the actual prompt text used is not provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A reports temperature=0.0, and all key parameters: ks=10, kh=1, ke=5, ki=kr=5, wc=0.7, wd=0.3. Pilot study ranges are also given."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent architecture is described in detail in Section III with workflow diagrams (Fig. 3, Fig. 4), showing the three agents, their coordination, and the knowledge graph construction pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes the data cleaning process: removing instances with missing repos, incomplete information, uncertain categories. Two annotators independently labeled defects with Cohen's kappa=0.9351."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V includes 'Threats to Validity' (V-A) and 'Limitations and Future Work' (V-B), providing substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: dataset reduction may introduce selection bias (mitigated by two independent annotators), Python-only support, pattern library limited to popular frameworks, and model pricing variability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B explicitly states three limitations: cost varies by model, pattern library covers only popular frameworks, and LIDL currently supports Python only. Section V-A notes the dataset 'may not represent industrial codebases.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is stated as publicly available at the GitHub repository. The defects are sourced from public GitHub issues with direct links provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes two data sources: Hydrangea (888 original defects from 105 GitHub applications) and AgentIssue-Bench (50 original defects from 16 agent systems), with cleaning criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are existing public datasets (Hydrangea and AgentIssue-Bench), which are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from 888+50 original defects to 146 final instances is described with explicit filtering criteria: missing repos, incomplete info, uncertain categories. Annotator agreement (Cohen's kappa 0.9351) is reported."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. No grants or sponsors are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sun Yat-sen University, Singapore Management University, and Monash University. No conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure does not establish unfunded status."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs on a benchmark but does not state training cutoff dates for any of the six models used. The benchmark defects are from public GitHub issues that could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the GitHub issues in the benchmark appeared in the training data of the evaluated LLMs. The Hydrangea and AgentIssue-Bench datasets are public and could be in training sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from public GitHub issues. The LLMs may have seen these issues and their solutions during training, which could inflate performance. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C reports average cost per instance for all methods across all models. LIDL costs $0.008 per instance on kimi-k2, with detailed token consumption (19.5k input, 0.3k output)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified (6 NVIDIA A40 48GB GPUs, 64-core CPU, 128GB RAM). Per-instance costs and token budgets are reported for all methods in Table V."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0.0 for reproducibility but no seed sensitivity analysis is reported. Even at temperature 0, LLM outputs can vary across API calls."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs produced the reported results. Single-run results are implied but not explicitly confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A states parameters were determined through a 'pilot study on 15 defects' with specific ranges tested: ks∈{5,10,15}, k∈{1,2}, ke,ki,kr∈{3,5,8}. Performance stability across ranges is noted."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The pilot study approach and the specific ranges tested are described. The paper notes 'Performance remained stable across these ranges,' justifying the selected configuration."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares LIDL against 5 baselines across 6 models and 4 metrics with no correction for multiple comparisons. No statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement LIDL and compare against their own implementations/configurations of baselines. No discussion of author-evaluation bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C explicitly compares cost vs accuracy, showing LIDL achieves the best accuracy-cost trade-off: '60% more than Agentless ($0.008 vs. $0.005) but improves Top-3 accuracy by 68.4%.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 146-defect benchmark adequately represents the space of LLM integration defects. The benchmark is constructed from two existing datasets without analysis of its construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LIDL uses a fundamentally different architecture (knowledge graph + multi-agent) than baselines. Performance differences could be attributed to the scaffold/architecture rather than the specific techniques. This confound is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark defects are from public GitHub issues. No discussion of whether these issues and their solutions existed before the LLMs' training cutoff dates."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The defect descriptions include error traces and file paths from real GitHub issues. No discussion of whether the evaluation setup provides information that would not be available in practice."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple defects may come from the same repositories (105 repos for 146 defects). No discussion of non-independence between instances from the same codebase."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No analysis of whether models have seen the benchmark issues during training."
       }
     }
   },
   "claims": [
     {
       "claim": "LIDL achieves 0.64 Top-3 accuracy and 0.48 MAP, representing a 64.1% improvement over the best-performing baseline (AutoCodeRover).",
       "evidence": "Table V shows kimi-k2 results: LIDL Top-3=0.64, MAP=0.48 vs AutoCodeRover Top-3=0.39, MAP=0.28. Section IV-B.",
       "supported": "moderate"
     },
     {
       "claim": "LIDL reduces cost by 92.5% compared to AutoCodeRover while maintaining superior accuracy.",
       "evidence": "Table V: LIDL $0.008 vs AutoCodeRover $0.106 per instance on kimi-k2. Section IV-C.",
       "supported": "strong"
     },
     {
       "claim": "LIDL uniquely localizes 18 defects (12.3%) that all baselines miss.",
       "evidence": "Overlap analysis in Fig. 8 and Section IV-B shows 18 unique defects found only by LIDL.",
       "supported": "moderate"
     },
     {
       "claim": "Annotation-based retrieval is the most critical component, with removal causing a 17.2% drop in Top-3.",
       "evidence": "Ablation study in Table VI: LIDL w/o R shows Top-3=0.53 vs full LIDL Top-3=0.64. Section IV-D.",
       "supported": "moderate"
     },
     {
       "claim": "Adding RepoGraph to baselines shows no improvement and can hurt performance.",
       "evidence": "Table V and Section IV-B: SWE-agent* drops from 0.29 to 0.21 Top-3 (-27.6%), Agentless* drops from 0.38 to 0.36 (-5.3%).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "The paper repeatedly claims LIDL 'significantly outperforms' baselines but provides no statistical tests. All comparisons are based on point estimates from what appear to be single runs. With 146 instances, bootstrap or permutation tests would be appropriate."
     },
     {
       "flag": "No variance or multiple runs reported",
       "detail": "Even with temperature=0.0, LLM API calls can produce different outputs. No repeated runs are reported, making it impossible to assess result stability."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "The benchmark consists of public GitHub issues with their solutions. Models like GPT-5.1 and Claude-Sonnet-4.5 likely trained on GitHub data including these issues. Performance differences between models could partly reflect training data overlap rather than method quality."
     },
     {
       "flag": "No failure case analysis",
       "detail": "LIDL fails to locate defects in Top-3 for 36% of instances. No analysis of these failures is provided — only a success case study is shown."
     },
     {
       "flag": "Pilot study not separated from evaluation",
       "detail": "Parameters were tuned on 15 defects from the same pool but there is no statement that these were excluded from the 146-instance evaluation."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez"],
       "year": 2024,
       "relevance": "Major baseline for repository-level defect localization using LLM agents."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Lightweight hierarchical approach to LLM-based defect localization, key baseline."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan"],
       "year": 2024,
       "relevance": "Best-performing baseline using code search APIs for defect localization."
     },
     {
       "title": "RepoGraph: Enhancing AI Software Engineering with Repository-Level Code Graph",
       "authors": ["Songlin Ouyang", "Wentao Yu"],
       "year": 2025,
       "relevance": "Repository graph approach for augmenting LLM-based SE tools."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li"],
       "year": 2025,
       "relevance": "Open platform for LLM-based software development agents."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["Qingyun Wu", "Gagan Bansal"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent framework frequently used as subject system in LLM integration defect studies."
     },
     {
       "title": "Are LLMs Correctly Integrated into Software Systems?",
       "authors": ["Yongqiang Shao", "Yuntong Huang"],
       "year": 2025,
       "relevance": "Prior study identifying and categorizing LLM integration defects (Hydrangea dataset source)."
     },
     {
       "title": "Can Agents Fix Agent Issues?",
       "authors": ["Alexander W. Rahardja", "Jiaxin Liu"],
       "year": 2025,
       "arxiv_id": "2505.20749",
       "relevance": "AgentIssue-Bench dataset source for evaluating agent defect fixing capabilities."
     },
     {
       "title": "CodexGraph: Bridging Large Language Models and Code Repositories via Code Graph Databases",
       "authors": ["Xiangyan Liu", "Bo Lan"],
       "year": 2025,
       "relevance": "Repository-structured approach using code graph databases for LLM reasoning about code."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mehmet Cemri", "Michael Z. Pan"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analysis of failure modes in multi-agent LLM systems, directly relevant to LLM integration defects."
     },
     {
       "title": "Defining and Detecting the Defects of the Large Language Model-based Autonomous Agents",
       "authors": ["Kaiwen Ning", "Jiacheng Chen"],
       "year": 2024,
       "arxiv_id": "2412.18371",
       "relevance": "Taxonomy of defects in LLM-based autonomous agents."
     }
   ]
 }