ai-research-survey

scan.json (26122B)

---

 {
   "paper": {
     "title": "Assessing the Impact of Code Changes on the Fault Localizability of Large Language Models",
     "authors": [
       "Sabaat Haroon",
       "Ahmad Faraz Khan",
       "Ahmad Humayun",
       "Waris Gill",
       "Abdul Haddi Amjad",
       "Ali R. Butt",
       "Mohammad Taha Khan",
       "Muhammad Ali Gulzar"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.04372",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The evaluation framework, datasets, and code generated and analyzed during the study are publicly available on Zenodo: https://zenodo.org/records/15550782' in the Data Availability section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo archive includes datasets used in the study, as stated in the Data Availability section."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA L40S GPU' and '48 GB RAM, 48 cores' (Table IV) for open-source models and that proprietary models used their respective APIs, but no requirements.txt, Dockerfile, or detailed dependency/version listing is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The Zenodo link is given but the paper does not describe how to run the framework, replicate experiments, or which commands to execute."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., accuracy percentages in Tables II, Figures 6-11). No confidence intervals, error bars, or uncertainty measures are provided for any of the reported metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Claude family achieves the best overall performance', 'Misleading Comments result in lower average accuracy compared to Misleading Variable Names') but no statistical significance tests are used to support any of these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout, e.g., 'SPMs cause an LLM to fail to localize the same fault it correctly localized earlier in 78% of cases', 'dead code alone reducing average accuracy to 20.38%', 'accuracy drops by 1.04% per mutation step in Java and 1.93% per step in Python'. These provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 10 LLMs were chosen, why 1,307 seed programs are sufficient, or any power analysis. The scale is large (750K tasks) but the choice of parameters is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across runs. It is unclear whether experiments were run multiple times. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper establishes baselines by measuring fault localization accuracy before applying SPMs (Table II), then compares post-SPM accuracy against this baseline. It also compares against prior benchmarks in Table V (HumanEval+, DebugBench, LiveCodeBench, SOAPFL, FlexFL)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared benchmarks include recent work: FlexFL (2025), SOAPFL (2025), LiveCodeBench (2025), and the LLMs evaluated are contemporary (Claude 4.5 Sonnet, Gemini 2.5 Flash, etc.)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies individual components: SPM types (dead code, misleading comments, misleading variable names, function reordering), SPM strengths (1-8), fault locations (quartiles), and SPM combinations (Mc, Mv, Md, Mc(Mv), Mc(Mv(Md))), effectively serving as an ablation study of which mutation factors affect performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only a single metric: fault localization accuracy (whether the LLM identifies the correct faulty line number). No additional metrics such as rank of the fault in the model's output, distance from the correct line, or partial credit measures are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the LLMs' fault localization outputs is performed. All evaluation is automated (comparing predicted line number against injected fault line number). Given the paper's claims about LLM code reasoning quality, human inspection of reasoning traces would be relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning training study. The evaluation framework dynamically generates tasks; there is no train/dev/test split to hold out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: by fault type (Table II), by language (Figure 7), by SPM type (Figure 6), by fault location quartile (Figure 9), by LLM category (Figure 10), by SPM strength (Figure 8), and by model family version (Figure 11)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific failure cases, e.g., the N-Queen motivating example (Section II/Figures 1-3), the autonomous car dead code example (Section V-B), and the Blender API make_tile example (Section V-B)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The primary finding is itself a negative result: LLMs fail to localize faults in 78% of cases after SPMs. The paper also reports that newer model versions show only marginal 1-2% improvements (RQ5), which is an honest negative finding about model evolution."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'SPMs cause an LLM to fail to localize the same fault it correctly localized earlier in 78% of cases' and '56% of correctly localized faults appear within the first 25% of program lines, compared to only 6% in the final 25%'. These are directly supported by the results in Sections V-A and V-C."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (e.g., SPMs 'cause' LLMs to fail). The experimental design supports this: the same LLM is tested on the same program before and after SPMs, with the SPM being the only controlled variable. This is a valid single-variable manipulation design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Large Language Models' generically but the study tests only 10 specific LLMs on Python and Java single-file programs. The abstract and conclusion make broad claims about 'LLMs' code-reasoning' without bounding to the tested setting. The threats to validity section mentions language and model limitations but the framing throughout is general."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses alternative explanations: whether prompt engineering could improve results, whether single-file programs are too simple (and argues this is a conservative lower bound), and whether code representation changes (CFGs, CPGs) could address the limitations. The threats section also discusses alternative explanations for fault injection coverage."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table III lists models as 'GPT-4o', 'Claude 3.7 Sonnet', 'Claude 4.5 Sonnet', 'Gemini 2.0-Flash', 'Gemini 1.5-Pro', 'Gemini 2.5-Flash', 'Qwen2.5-coder', 'Llama3.1', 'Phi4', 'Qwen-QWQ'. These are marketing names without API versions or snapshot dates (e.g., no 'gpt-4o-2024-05-13' or similar)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt used for fault localization is provided in Section II: 'This code is designed to solve the N-Queen problem. Given an input N, it should return all valid arrangements of N queens on an N x N board such that no two queens attack each other. However, the code produces incorrect output. Can you identify the specific line of code responsible for the error? The program is attached below. <CODE>'. This is the actual uniform baseline prompt used across experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling hyperparameters are reported for any of the 10 LLMs. These significantly affect LLM output and are not mentioned."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLMs are given a single prompt and return a single response. There is no multi-turn interaction, tool use, or feedback loop."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes data selection criteria (>50 LOC, specifications required, avoiding contaminated benchmarks, context window limits), with specific counts: 18,612 Python programs and 812 Java programs filtered to 637 Python and 670 Java. Section IV-B describes the fault injection and underspecification filtering pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threats to Validity' provides a dedicated discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: injected faults may not cover all execution paths, the selection of 10 specific LLMs may not generalize to all models, results limited to Python and Java, four fault types may not capture real bug diversity (e.g., multi-location bugs), and SPMs do not cover all semantic-preserving transformations (e.g., refactorings, API-equivalent rewrites)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The threats section explicitly states what is not tested: other programming languages, multi-location bugs, more complex semantic-preserving transformations, and models beyond the 10 tested. Section VI also states they 'intentionally avoid extensive prompt engineering' and test single-file programs only."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo archive (https://zenodo.org/records/15550782) is stated to contain the evaluation framework, datasets, and code, making raw data available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes data collection in detail: Python programs from iamtarun/python_code_instructions_18k_alpaca on HuggingFace, Java programs from CodeSearchNet. Selection criteria (LOC >= 50, spec availability, context limits) and filtering steps are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section IV and Figure 4: seed program procurement (18,612 + 812 programs) -> size/context filtering (637 + 670) -> fault injection (4 types x 4 locations) -> underspecification filtering -> SPM application (6 variants per program) -> 750,013 total tasks. Each transformation step is explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper. There is no mention of grants, sponsorships, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: seven authors from Virginia Tech and one from Carnegie Mellon University. Neither institution is directly affiliated with any of the evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses contamination as a key motivation (Defects4J and BugsInPy are in training data) and designs the framework to avoid it, but does not state the training data cutoff dates for any of the 10 evaluated LLMs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses contamination: Section I identifies that 'datasets have become part of LLM training data, leading to biased results', and the entire framework design addresses this by dynamically injecting unseen faults. Section IV-A explicitly avoids Defects4J and BugsInPy for contamination reasons. However, the seed programs themselves (from HuggingFace) could be in training data, which is partially addressed by the mutation approach."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a central concern of the paper. The framework generates novel fault localization tasks by injecting unseen faults and SPMs into programs, specifically designed so that 'LLMs undergo continuous training and are eventually exposed to public datasets' (Section I). The dynamic generation approach is the paper's key contribution for addressing contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports total tokens analyzed (3.8 billion) and total prompts (1,163,686) but does not report API costs, cost per task, wall-clock time, or total expenditure for running 750K+ tasks across 10 LLMs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table IV mentions '48 GB RAM, 48 cores with NVIDIA L40S GPU' as hardware but does not state total GPU hours, total API spend, or total computation time for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "SPMs cause LLMs to fail to localize the same fault they correctly localized earlier in 78% of cases.",
       "evidence": "Section V-A reports this aggregate figure across all 10 LLMs, fault types, and SPM types. Figure 7 shows the per-model breakdown.",
       "supported": "strong"
     },
     {
       "claim": "Dead code injection reduces LLM fault localization accuracy to 20.38% on average.",
       "evidence": "Section V-B and Figure 6 show dead code has the most substantial impact among SPM types, with 20.38% average accuracy.",
       "supported": "strong"
     },
     {
       "claim": "56% of correctly localized faults appear within the first 25% of program lines, compared to only 6% in the final 25%.",
       "evidence": "Section V-C and Figure 9 present a heatmap showing fault localization accuracy by fault location quartile across all fault types.",
       "supported": "strong"
     },
     {
       "claim": "Fault localization accuracy drops by 1.04% per mutation strength step in Java and 1.93% per step in Python.",
       "evidence": "Section V-B and Figure 8 present mutation strength analysis from strength 1 to 8, showing near-linear degradation.",
       "supported": "strong"
     },
     {
       "claim": "Newer Claude and Gemini variants show only very modest fault localizability gains (1-2%).",
       "evidence": "Section V-E and Figure 11 show Gemini 2.0 Flash to 2.5 Flash improves 1.8%, Claude 3.7 Sonnet to 4.5 Sonnet improves 1.0%.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning and multimodal models achieve the highest fault localization accuracy, while coding-specialized models yield the lowest performance.",
       "evidence": "Section V-D and Figure 10 categorize LLMs and show reasoning/multimodal models outperform generic and coder models.",
       "supported": "moderate"
     },
     {
       "claim": "Function reordering causes an 83% accuracy drop in Java programs.",
       "evidence": "Section V-B states 'We observe a significant accuracy drop of 83%' for function reordering mutations on Java.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "LLMs' fault localization reasoning is highly fragile to semantic-preserving code changes: across 750K tasks on 10 LLMs, applying SPMs causes failure in 78% of cases where the model initially succeeded. Dead code insertion and misleading comments are the most disruptive mutation types. LLMs exhibit strong positional bias, localizing faults in the first 25% of code 9x more often than in the final 25%. Newer model versions show only marginal (1-2%) improvements in fault localization robustness, suggesting model scaling alone does not address fundamental limitations in code reasoning.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "The paper makes numerous comparative claims (model A outperforms model B, mutation type X causes more degradation than Y) but provides no statistical tests. With 750K tasks, even trivially small differences would be statistically significant, making the absence notable."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "No confidence intervals, error bars, or variance measures are reported for any results. It is unclear whether experiments involve any randomness (e.g., API temperature settings) or were run multiple times."
     },
     {
       "flag": "Missing hyperparameter reporting",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the 10 LLMs. These settings can significantly affect output quality and reproducibility, especially for deterministic fault localization."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only exact line-number match accuracy is used. This binary metric does not capture near-misses (identifying the right function but wrong line) or cases where the model identifies the fault correctly but reports a slightly different line number. This could inflate the apparent fragility."
     },
     {
       "flag": "Seed program contamination not fully addressed",
       "detail": "While the paper's key contribution is addressing contamination through dynamic fault injection, the seed programs themselves come from public HuggingFace datasets that are likely in LLM training data. LLMs may have memorized the correct versions, which could confound the analysis of semantic-preserving mutations."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational benchmark (HumanEval) for LLM code generation evaluation, relevant to survey scope on LLM programming capabilities."
     },
     {
       "title": "Can large language models reason about code?",
       "authors": ["T. Li", "Z. Wei", "M. Allamanis"],
       "year": 2024,
       "relevance": "Directly evaluates LLM code reasoning capabilities, central to survey scope on AI programming quality."
     },
     {
       "title": "Investigating data contamination in modern benchmarks for large language models",
       "authors": ["C. Deng", "Y. Zhao", "X. Tang", "M. Gerstein", "A. Cohan"],
       "year": 2024,
       "relevance": "Addresses benchmark contamination in LLM evaluation, a key methodological concern for the survey."
     },
     {
       "title": "Are large language models memorizing bug benchmarks?",
       "authors": ["D. Ramos", "C. Mamede", "K. Jain", "P. Canelas", "C. Gamboa", "C. Le Goues"],
       "year": 2025,
       "relevance": "Directly relevant to benchmark contamination in LLM code evaluation, a key methodological issue."
     },
     {
       "title": "Large language models for test-free fault localization",
       "authors": ["A. Z. H. Yang", "C. Le Goues", "R. Martins", "V. Hellendoorn"],
       "year": 2024,
       "relevance": "Evaluates LLM-based fault localization capabilities, directly relevant to AI programming quality assessment."
     },
     {
       "title": "SOAPFL: A standard operating procedure for LLM-based method-level fault localization",
       "authors": ["Y. Qin", "S. Wang", "Y. Lou", "J. Dong", "K. Wang", "X. Li", "X. Mao"],
       "year": 2025,
       "doi": "10.1109/TSE.2025",
       "relevance": "LLM-based fault localization methodology, relevant to survey scope on AI-assisted software engineering."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "relevance": "HumanEval+ rigorous evaluation of LLM code generation, relevant to methodology quality assessment."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2025,
       "relevance": "Contamination-free LLM code benchmark, relevant to evaluation methodology in the survey."
     },
     {
       "title": "DebugBench: Evaluating debugging capability of large language models",
       "authors": ["R. Tian", "Y. Ye", "Y. Qin"],
       "year": 2024,
       "relevance": "Benchmark for LLM debugging capabilities, directly relevant to AI programming evaluation."
     },
     {
       "title": "FlexFL: Flexible and effective fault localization with open-source large language models",
       "authors": ["C. Xu", "Z. Liu", "X. Ren", "D. Lo"],
       "year": 2025,
       "relevance": "Open-source LLM fault localization, relevant to survey scope on AI-assisted debugging."
     },
     {
       "title": "Top score on the wrong exam: On benchmarking in machine learning for vulnerability detection",
       "authors": ["N. Risse", "J. Liu", "M. Böhme"],
       "year": 2024,
       "arxiv_id": "2408.12986",
       "relevance": "Critiques benchmarking methodology in ML for vulnerability detection, relevant to evaluation methodology quality."
     },
     {
       "title": "DIP: Dead code insertion based black-box attack for programming language model",
       "authors": ["C. Na", "Y. Choi", "J.-H. Lee"],
       "year": 2023,
       "relevance": "Dead code insertion attacks on code models, relevant to robustness evaluation of AI programming tools."
     }
   ]
 }