ai-research-survey

scan.json (27729B)

---

 {
   "paper": {
     "title": "Fixing Rust Compilation Errors using LLMs",
     "authors": ["Pantazis Deligiannis", "Akash Lal", "Nikita Mehrotra", "Aseem Rastogi"],
     "year": 2023,
     "venue": "ICSE 2025",
     "arxiv_id": "2308.05177",
     "doi": "10.1109/ICSE55347.2025.00022"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'We plan to open-source both our dataset as well as the implementation of RustAssistant' (Section 1) and 'We plan to release our dataset to enable further research' (Section 8). These are promises of future release, not actual releases."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as above — 'We plan to release our dataset' is a promise, not a release. No download link or repository URL is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency versions are listed. The Rust compiler version (1.67.1) is mentioned for benchmark creation but not as a reproducibility specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The algorithm is described at a high level but no commands, scripts, or README instructions are included."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '92.59%', '73.63%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 'performs better than GPT-3.5' and compares prompt variants, but no statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context — e.g., GPT-3.5 at 52.96% vs GPT-4 at 92.22% on micro-benchmarks (Table 1), and absolute numbers (143/270 vs 249/270). The Clippy comparison shows '2.4x more errors' fixed. Sufficient context to judge magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes (270 micro-benchmarks, 50 SO questions, 182 commits). The 50 SO questions are described as 'the first 50 most relevant questions' with no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be single-run numbers (temperature 0.2, top_p=1 for deterministic generation, but N=5 completions involves randomness)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5 serves as a baseline against GPT-4. For Clippy errors, Clippy's own auto-fix is used as a baseline (Table 6). Prompt ablation variants (P0-P4) provide internal baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5 and GPT-4 were state-of-the-art LLMs at the time of writing (August 2023). Clippy auto-fix is the standard tool baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 RQ2 presents ablation studies: five prompt variants (P0-P4, Table 5), varying number of completions (N=1 vs N=5), and toggling error grouping on/off (Table 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Fix% (programs fixed), plus breakdown into Format errors, Build errors, and Test failures (Tables 1-2). For top-100 crates: both Commits% and Errors% (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 5) involves qualitative manual examination of all 134 fixed commits from the top-100 crates, categorized into 4 semantic correctness categories (Table 7). The paper states 'a structured consensus-based manual evaluation involving multiple evaluators' (Section 6)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The three datasets (micro-benchmarks, SO, top-100 crates) are separate evaluation sets. Micro-benchmarks and SO have independent unit tests. No tuning was done on the test data — the prompt was fixed across evaluations."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows fix rate broken down by error category (Syntax, Type, Generics, Traits, Ownership, Lifetime). Table 6 breaks Clippy results by category (Complexity, Pedantic, Style). Table 7 categorizes fix quality."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific failure modes: format errors, build errors, test failures. Concrete examples are given (e.g., the f64 bitwise operator example where GPT-4 uses wrong constant). Loop/stuck behavior and .toml editing limitations are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "P0 (basic prompt) achieves only ~10% accuracy (Table 4). Turning off error grouping significantly drops performance. GPT-4 N=5 on SO (72%) is slightly worse than N=1 (74%), an unexpected negative result. The paper reports cases where LLM gets stuck in fix/undo loops."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'peak accuracy of roughly 74% on real-world compilation errors' which matches Table 3 (73.63% commits fixed). The claim of 'high accuracy' is supported by the micro-benchmark result of 92.59%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'the changelog format improves accuracy' are supported by controlled ablation studies (P0 vs P4 in Tables 4-5) with single-variable manipulation. The claim that 'error grouping helps' is supported by toggling it on/off."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its scope: 'We focus on fixing code-related issues' (Section 2.1), explicitly excludes .toml configuration changes and unsafe code. Results are presented separately for each dataset. The title says 'Rust Compilation Errors' not 'all software bugs'."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Threats to Validity) discusses data contamination, the subjectivity of semantic correctness evaluation, and construct validity of handwritten test cases. The paper considers whether GPT-4's performance could be due to memorization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes what it measures: compilation success (objective) vs semantic correctness (subjective). Section 2.1 explicitly states 'The former is an objective criterion while the latter is subjective.' RQ4 specifically addresses the gap between passing the compiler and producing correct fixes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-3.5-turbo (300B parameters)' and 'GPT-4' without specific version snapshots (e.g., gpt-4-0613). No API version dates are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides the full RustAssistant prompt template with all sections: preamble, error context, instructions, and detailed changelog format instructions with examples. The actual template text is reproduced verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 reports: frequency_penalty=0, presence_penalty=0, top_p=1, temperature=0.2, maximum output length=800 tokens."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 provides the full RustAssistant pipeline: compilation → error extraction → prompt construction → LLM invocation → changelog parsing → patch application → recompilation loop. Error grouping, termination heuristics, and best-completion selection are all described in detail (Section 3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents dataset construction: micro-benchmarks cover 270/506 error codes with explicit exclusion criteria, SO questions filtered for relevance with stated criteria, top-100 crates commits identified by cloning and building locally with out-of-scope error filtering."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated, substantive discussion of internal and external threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: data contamination for the specific repositories used, the subjective nature of semantic correctness evaluation mitigated by consensus-based multi-evaluator review, construct validity of handwritten tests, and the limitation that Clippy evaluation only checks clippy passing (not test suite)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly states scope boundaries: only .rs files (not .toml), no unsafe code, no foreign function interop errors. Section 6 acknowledges generalizability may vary across different datasets."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The dataset is not released. Only aggregate results are shown. The promise to release is not fulfilled in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes collection in detail: micro-benchmarks written per error code referencing the Rust catalog, SO questions manually scraped with filtering criteria, top-100 crates from crates.io with commit history examination and local compilation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are public repositories, Stack Overflow, and compiler error codes."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw data to analysis is documented: for SO, filtering criteria are stated; for top-100 crates, commits are cloned, built, filtered for in-scope errors (182 found); for micro-benchmarks, 270/506 error codes selected with exclusion criteria stated. Numbers at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as Microsoft Research employees with their locations (Redmond and Bengaluru)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Microsoft Research employees. Microsoft has commercial interest in AI-powered developer tools (GitHub Copilot). The paper evaluates OpenAI models (GPT-3.5, GPT-4), and Microsoft is a major investor in OpenAI. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Microsoft's investment in OpenAI and its commercial developer tool products (GitHub Copilot, VS Code) create potential financial interests that are not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-3.5 or GPT-4. The paper acknowledges contamination risk in Section 6 but does not state when training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly discusses this: 'it might be possible that fixes to the compilation issues that we mined from open source might have already been included in the training data.' They argue the fixes 'were never presented online in the form of a fix or alongside the corresponding compiler error' though the fixed code may appear in later repository versions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses contamination conceptually (Section 6), it does not use any concrete detection method. The SO questions and top-100 crate commits were publicly available before GPT-4's training. The paper acknowledges 'There is no ideal way to completely remove contamination' but does not apply temporal splits or detection methods."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM performance on code benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-example costs are reported despite the tool making multiple LLM calls per error (up to 15 iterations shown in Figures 6-7) with N=5 completions each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, token consumption, or wall-clock time for the evaluation is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0.2 (not 0) and top_p=1, so outputs are not fully deterministic. No results across multiple seeds or runs are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "It is not stated how many times the full evaluation was run. Results appear to be from a single run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The hyperparameters (temperature=0.2, max_tokens=800, window=±50 lines) appear chosen but no search budget or justification for these values is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The prompt ablation study (P0-P4 in Table 5) systematically varies features and reports all configurations. The N=1 vs N=5 variation is also reported for all datasets. Selection of best prompt is transparent."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons across the many comparisons made (2 models × 2 N values × 3 datasets × 5 prompt variants)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (RustAssistant) and do not acknowledge potential bias in their implementation of the pipeline or their manual evaluation of fix quality."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 with N=5 uses roughly 5x the API calls of N=1, but no cost-performance tradeoff analysis is presented. The iterations (Figures 6-7) show compute scaling but not cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses what its benchmarks test: micro-benchmarks test isolated error fixing, SO tests harder real-world patterns, and top-100 crates test multi-error real-world scenarios. Section 2.1 distinguishes compilation correctness (objective) from semantic correctness (subjective)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The scaffold (RustAssistant pipeline) is fixed across model comparisons (GPT-3.5 vs GPT-4). The ablation study explicitly varies scaffold components (error grouping, prompt format) as independent variables."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The SO questions and open-source crate commits predate GPT-4's training. The paper acknowledges contamination risk but does not analyze temporal overlap between benchmark creation dates and model training periods."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The tool provides compiler error messages (including suggestions like 'help: consider annotating with...') directly to the LLM. This is by design, but the paper does not discuss whether this constitutes feature leakage relative to a real developer scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the micro-benchmarks or SO examples overlap with or are similar to code in the LLMs' training data beyond the general contamination acknowledgment."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is used. The paper only discusses contamination conceptually and notes 'There is no ideal way to completely remove contamination.'"
       }
     }
   },
   "claims": [
     {
       "claim": "RustAssistant achieves 92.59% fix rate on micro-benchmarks with GPT-4 (N=5)",
       "evidence": "Table 1: 250/270 micro-benchmarks fixed. Unit tests verify semantic correctness.",
       "supported": "strong"
     },
     {
       "claim": "RustAssistant achieves 73.63% commit fix rate on top-100 Rust crates with GPT-4 (N=5)",
       "evidence": "Table 3: 134/182 commits fixed, 846/925 individual errors fixed (91.46%).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 significantly outperforms GPT-3.5 for fixing Rust compilation errors",
       "evidence": "Tables 1-3 consistently show GPT-4 outperforming GPT-3.5 across all datasets (e.g., 92.59% vs 73.70% on micro-benchmarks, 73.63% vs 35.71% on top-100 crates).",
       "supported": "moderate"
     },
     {
       "claim": "The changelog prompt format is critical, raising accuracy from ~10% to ~74%",
       "evidence": "Table 4 shows P0 (basic) at 9.89% vs P4 (full changelog) at comparable result. Table 5 shows P1 at 10.74% vs P4 at 73.70% on micro-benchmarks with GPT-3.5.",
       "supported": "strong"
     },
     {
       "claim": "RustAssistant fixes 2.4x more Clippy errors than Clippy's own auto-fix",
       "evidence": "Table 6: 259/346 (74.86%) vs 109/346 (31.50%).",
       "supported": "moderate"
     },
     {
       "claim": "Most GPT-4 fixes on real-world code are semantically correct",
       "evidence": "Table 7: Of 134 fixed commits, only 9 have different runtime behavior. 55 are unambiguous, 41 match developer fix, 29 are non-matching but same behavior.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RustAssistant, an LLM-based tool for fixing Rust compilation errors, achieves 92.59% on micro-benchmarks and 73.63% on real-world commits from top-100 Rust crates using GPT-4. The changelog prompt format is critical — accuracy drops to ~10% without it. Ablation studies show each prompt component (line prefixes, localization, fix description) contributes incrementally. Manual inspection of 134 fixed commits found only 9 with potentially altered runtime behavior.",
   "red_flags": [
     {
       "flag": "Company evaluating commercially-aligned technology",
       "detail": "All authors are Microsoft Research employees. Microsoft is a major investor in OpenAI (whose GPT-3.5/GPT-4 are evaluated) and sells AI-powered developer tools (GitHub Copilot). This conflict is not acknowledged."
     },
     {
       "flag": "No statistical tests on any comparison",
       "detail": "Claims of superiority (GPT-4 vs GPT-3.5, prompt variants) are based purely on comparing point estimates with no significance tests, confidence intervals, or variance reporting."
     },
     {
       "flag": "Single-run results with stochastic components",
       "detail": "Temperature=0.2 and N=5 completions introduce randomness, but results appear to be from a single run with no reproducibility analysis."
     },
     {
       "flag": "Unreleased artifacts despite promises",
       "detail": "The paper promises to release both the dataset and tool implementation, but as of the paper text, neither is released. No repository URL is provided."
     },
     {
       "flag": "No cost reporting despite expensive pipeline",
       "detail": "The tool may call GPT-4 dozens of times per error (up to 15 iterations × 5 completions = 75 API calls per error) but no cost analysis is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated Repair of Programs from Large Language Models",
       "authors": ["Zhiyu Fan", "Xiang Gao", "Martin Mirchev", "Abhik Roychoudhury", "Shin Hwei Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00128",
       "relevance": "Evaluates LLMs (Codex) for automated program repair on Java defects, directly comparable approach."
     },
     {
       "title": "Practical Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2210.14179",
       "relevance": "Extensive study of 9 pre-trained language models for APR across three languages, establishes that larger models perform better."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00129",
       "relevance": "Experiments with GPT-3 series, CodeT5, InCoder on Defects4J and other benchmarks for LLM-based APR."
     },
     {
       "title": "InferFix: End-to-End Program Repair with LLMs",
       "authors": ["Matthew Jin", "Syed Shahriar", "Michele Tufano", "Xin Shi", "Shuai Lu", "Neel Sundaresan", "Alexey Svyatkovskiy"],
       "year": 2023,
       "arxiv_id": "2303.07263",
       "relevance": "Uses fine-tuned LLM to fix static analysis errors (CodeQL), related approach of using formal tool feedback with LLMs."
     },
     {
       "title": "Examining Zero-Shot Vulnerability Repair with Large Language Models",
       "authors": ["Hammond Pearce", "Benjamin Tan", "Baleegh Ahmad", "Ramesh Karri", "Brendan Dolan-Gavitt"],
       "year": 2023,
       "doi": "10.1109/SP46215.2023.10179420",
       "relevance": "Large-scale study of LLMs for zero-shot cybersecurity bug repair, evaluates multiple models."
     },
     {
       "title": "Can OpenAI's Codex Fix Bugs?: An evaluation on QuixBugs",
       "authors": ["Julian Aron Prenner", "Hlib Babii", "Romain Robbes"],
       "year": 2022,
       "doi": "10.1145/3524459.3527351",
       "relevance": "Early evaluation of Codex for bug fixing on Java and Python, establishes LLM capability for APR."
     },
     {
       "title": "Repair Is Nearly Generation: Multilingual Program Repair with LLMs",
       "authors": ["Harshit Joshi", "José Pablo Cambronero Sánchez", "Sumit Gulwani", "Vu Le", "Ivan Radicek", "Gust Verbruggen"],
       "year": 2022,
       "arxiv_id": "2208.11640",
       "relevance": "RING system uses retrieval-augmented few-shot prompting to fix syntactic errors in multiple languages."
     },
     {
       "title": "Revisiting the Plastic Surgery Hypothesis via Large Language Models",
       "authors": ["Chunqiu Steven Xia", "Yifeng Ding", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2303.10494",
       "relevance": "FitRepair combines LLMs with plastic surgery hypothesis and fine-tuning strategies for APR."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the two LLMs evaluated in this study."
     },
     {
       "title": "Retrieval-Based Prompt Selection for Code-Related Few-Shot Learning",
       "authors": ["Noor Nashid", "Mifta Sintaha", "Ali Mesbah"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00205",
       "relevance": "Retrieval-based prompt engineering for code tasks, relevant to the prompting techniques used in RustAssistant."
     },
     {
       "title": "DeepFix: Fixing Common C Language Errors by Deep Learning",
       "authors": ["Rahul Gupta", "Soham Pal", "Aditya Kanade", "Shirish K. Shevade"],
       "year": 2017,
       "relevance": "Early deep learning approach to fixing compilation errors, predecessor to LLM-based APR."
     }
   ]
 }