ai-research-survey

scan.json (24210B)

---

 {
   "paper": {
     "title": "Practical Program Repair in the Era of Large Pre-trained Language Models",
     "authors": [
       "Chunqiu Steven Xia",
       "Yuxiang Wei",
       "Lingming Zhang"
     ],
     "year": 2022,
     "venue": "arXiv",
     "arxiv_id": "2210.14179"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in Section VI that they 'released the correct patches and code used to perform the experiments for public evaluation [74]' with a Figshare link provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (Defects4J 1.2, Defects4J 2.0, QuixBugs-Java, QuixBugs-Python, ManyBugs) and released their experimental data via Figshare [74]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-B mentions 'PyTorch versions of each PLM' and the hardware used ('32-Core workstation with Ryzen Threadripper PRO 3975WX CPU, 256 GB RAM and NVIDIA RTX A6000 GPU, running Ubuntu 20.04.4 LTS') but does not provide a requirements.txt, Dockerfile, or detailed dependency list with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the approach and mentions releasing code via Figshare, it does not provide step-by-step reproduction instructions in the paper itself. The Figshare link is a placeholder URL (https://figshare.com/s/temp)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only point estimates (number of correct/plausible patches) across all tables (Tables III-X). No confidence intervals or error bars are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparative claims ('substantially outperform all existing APR techniques') based on comparing raw counts of bugs fixed without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides baseline counts alongside its own results (e.g., Codex fixes 99 vs AlphaRepair 67 on Defects4J 1.2, Tables VII-VIII), and reports percentage improvements like '40% (62/154)' of bugs fixed, providing sufficient context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 200 samples per bug was chosen as the default, nor why 2000 was selected for the improvement experiments. The choice of 200 is attributed to 'default setting used in previous work [26], [30]' without independent justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports single-run results. No standard deviation, variance, or spread measures across multiple runs are reported. With nucleus sampling (temperature=0.8), different runs could yield different results, but this is not investigated."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against 20 different APR tools including 8 learning-based tools (AlphaRepair, RewardRepair, Recoder, DeepDebug, CURE, CoCoNuT, DLFix, SequenceR) and 12 traditional tools (TBar, PraPR, etc.), as detailed in Section IV-D and Tables VII-VIII."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include recent state-of-the-art tools from 2021-2022 such as AlphaRepair (2022), RewardRepair (2022), and Recoder (2021). These represent the current best at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation-like analysis: comparing generative vs infilling models (Tables III-V), comparing prefix-only vs prefix+suffix (Table V, Codex single-line vs Codex suffix), and the effect of increasing sample size and adding templates (RQ4, Table X)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics: number of correct patches, number of plausible patches, patch generation speed (patches/min in Table VI), syntactic/semantic error rates (Figure 4), and mean/sum entropy for ranking (Table IX, Figure 7)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-E states they 'manually inspect each plausible patch for semantic equivalency' to determine correct patches, which is standard practice in APR research."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The models are applied in a zero-shot setting (no finetuning on APR data), and the evaluation uses standard benchmarks (Defects4J, QuixBugs, ManyBugs) with developer-written test suites for validation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (Defects4J 1.2, 2.0, QuixBugs-Java, QuixBugs-Python, ManyBugs), by model, by repair setting (complete function, code infilling, single line), and by model size in Tables III-VIII."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses unique successful fixes (Figure 6) but does not analyze failure cases or discuss specific examples where PLMs fail to generate correct patches."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that generative models produce high syntactic error rates for single line generation (Figure 4, Section V-A3), and that smaller models like GPT-Neo 125M often fix very few bugs. The speed-performance tradeoff is also honestly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that PLMs 'substantially outperform all existing APR techniques on all our datasets' are supported by Tables VII-VIII showing Codex fixing 99 bugs on Defects4J 1.2 vs the best baseline's 67. All other abstract claims (scaling effect, suffix importance, patch ranking, improvements via templates) are supported in Sections V-A through V-D."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are present (e.g., 'suffix code is important in generating more fixes', 'increasing sample size boosts performance'). These are supported by controlled comparisons: same model with/without suffix (Codex single-line vs Codex suffix in Table V), and same model with 200 vs 2000 samples (Table X). The ablation design is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VI External validity explicitly states: 'However, our findings may still not generalize to other datasets or languages.' The study covers 3 languages and 5 datasets but acknowledges limitations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses data leakage as an alternative explanation for the strong results, analyzing what percentage of correct patches match developer patches and whether those appeared in training data. They found 66% of fixes differ from developer patches, and only 15% of exact-match fixes were in training data."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table I specifies exact model names and parameter counts (GPT-Neo 125M/1.3B/2.7B, GPT-J 6.7B, GPT-NeoX 20B, Codex 12B, CodeT5 220M, INCODER 1.3B/6.7B). Section IV-B specifies 'code-davinci-002 engine' for Codex. Parameter sizes serve as version identifiers for the open-source models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1-3 provide the actual prompt templates used for all three repair settings (complete function generation, correct code infilling, single line generation). The Fibonacci example prompt is shown in full in Figure 1, and infilling token placement is shown in Figures 2-3."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B states: 'Our default setting for generation uses nucleus sampling with top p = 0.95, temperature = 0.8 and 200 samples per bug.' These are the key generation hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The PLMs are applied directly with single-pass generation — no retry logic, tool use, feedback loops, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes filtering benchmarks to find bugs fitting the repair settings, with Table II providing exact counts at each filter level (#Bugs → #SF → #SH → #SL). Section IV-C3 also notes that only 91 of 185 ManyBugs bugs were reproducible and used."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides a dedicated section discussing both internal and external threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses specific threats: (1) manual validation of patches may have errors, addressed by releasing patches; (2) data leakage with specific statistics (66% of fixes differ from developer patches, only 15% found in training data); (3) evaluation limited to 5 datasets and 3 languages."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states that 'our findings may still not generalize to other datasets or languages' and notes the focus is on 'bugs where the fix is within a single function' (Section IV-C), bounding the scope of claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The Figshare link [74] (https://figshare.com/s/temp) appears to be a placeholder URL. While they claim to release 'correct patches and code,' the actual availability cannot be verified from the paper. The underlying benchmarks (Defects4J, QuixBugs, ManyBugs) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes each dataset in detail, including where bugs are collected from, how many bugs exist, and the filtering criteria applied. The model selection process is described in Section III-A."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses existing code benchmarks and pre-trained models."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from bug selection (Table II with filtering counts) through patch generation (Section III-B with 3 settings) to validation (Section III-C with entropy ranking and test suite validation) is documented. Section IV-C3 explains filtering ManyBugs from 185 to 91 reproducible bugs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with the University of Illinois at Urbana-Champaign, disclosed at the top of the paper. No authors work at companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. The paper evaluates Codex (OpenAI) among other models but the authors are university researchers, not OpenAI employees."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper identifies training datasets for some models (The Pile for GPT-Neo/J/NeoX, CodeSearchNet/BigQuery for CodeT5), no training data cutoff dates are stated. For Codex and INCODER, training data is listed as 'N.R.' (not released)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI provides substantial analysis of train/test overlap: 'only 15% (20/128) of those patches are also found in the original training data.' They checked whether fixed functions appeared in training datasets of models where training data was accessible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI extensively discusses data leakage. They analyze whether developer patches appear in training data, report that 66% of correct fixes differ from developer patches, and note QuixBugs 'is not part of the training data as it has low number of stars on GitHub and contains synthetic bugs.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study. It is a benchmark evaluation of pre-trained language models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table VI reports patch generation speed (patches/min) for each model across repair settings. Section V-B1 notes 'generating 200 patches for each of the 3 settings (i.e., at most 600 patches in total) costs no more than 2.5 hours for each model.' Figure 4 reports compilation rates."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B specifies the hardware: '32-Core workstation with Ryzen Threadripper PRO 3975WX CPU, 256 GB RAM and NVIDIA RTX A6000 GPU, running Ubuntu 20.04.4 LTS.' Time per bug is bounded at 2.5 hours and per-model speed is reported in Table VI."
       }
     }
   },
   "claims": [
     {
       "claim": "Directly applying state-of-the-art PLMs can substantially outperform all existing APR techniques on all studied datasets.",
       "evidence": "Table VII shows Codex fixes 99 bugs on Defects4J 1.2 vs AlphaRepair's 67 (best baseline). Combined PLMs fix 109 bugs. Table VIII shows similar outperformance on Defects4J 2.0 (52 vs 35), QuixBugs-Java (38 vs 28), and QuixBugs-Python (40 vs 27).",
       "supported": "strong"
     },
     {
       "claim": "The scaling effect exists for APR where larger models tend to achieve better performance.",
       "evidence": "Tables III-V consistently show increasing correct patches as model size increases within the GPT family (125M → 1.3B → 2.7B → 6.7B → 20B) and INCODER family (1.3B → 6.7B) across all datasets.",
       "supported": "strong"
     },
     {
       "claim": "Suffix code after the buggy line is important in generating more fixes and patches with higher compilation rate.",
       "evidence": "Table V shows Codex suffix outperforms Codex single-line across all datasets (e.g., 39 vs 32 correct on Defects4J 1.2). Figure 4 shows infilling models produce lower syntactic/semantic error rates.",
       "supported": "strong"
     },
     {
       "claim": "PLMs consider correct patches to be more natural than incorrect patches based on entropy.",
       "evidence": "Table IX shows mean entropy values for correct patches are consistently lower than non-plausible patches across all models and settings (e.g., Codex function gen on Defects4J 1.2: 0.04 correct vs 0.08 non-plausible).",
       "supported": "strong"
     },
     {
       "claim": "PLM-based APR can be further substantially boosted by increasing sample size and incorporating fix templates.",
       "evidence": "Table X shows INCODER 6.7B improves from 37 to 64 correct fixes on Defects4J 1.2 when going from 200 to 2000 samples, and further to 78 with templates, surpassing all baselines.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Directly applying 9 pre-trained language models (ranging from 125M to 20B parameters) for automated program repair substantially outperforms 20 existing APR tools across 5 benchmarks in 3 languages. The scaling effect holds for APR, with larger models consistently fixing more bugs. Infilling models that use both prefix and suffix context outperform generative models that use prefix only, and entropy-based patch ranking enables faster identification of correct patches. Combining PLMs with simple repair templates and increased sampling further boosts performance, with INCODER 6.7B achieving 78 correct fixes on Defects4J 1.2 (vs 74 for the best baseline).",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are based on raw counts of bugs fixed without any statistical tests. Given the stochastic nature of nucleus sampling, the results could vary across runs, but no variance or significance testing is reported."
     },
     {
       "flag": "Placeholder data release URL",
       "detail": "The Figshare link [74] points to 'https://figshare.com/s/temp' which appears to be a placeholder, undermining the claim of public artifact release."
     },
     {
       "flag": "Potential data leakage not fully resolved",
       "detail": "While the paper acknowledges and partially investigates data leakage (Section VI), training data for Codex and INCODER is listed as 'N.R.' (not released), so overlap cannot be checked for the best-performing models. The 15% confirmed overlap rate among checkable models is non-trivial."
     },
     {
       "flag": "Missing funding disclosure",
       "detail": "No funding source or competing interests statement is provided, which is unusual for a paper from a major university research group."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex, a key model evaluated for code generation and program repair capabilities."
     },
     {
       "title": "Less training, more repairing please: Revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2207.08281",
       "relevance": "Introduces AlphaRepair, the zero-shot PLM-based APR baseline using CodeBERT."
     },
     {
       "title": "Incoder: A generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "Presents INCODER, a code infilling model evaluated for program repair tasks."
     },
     {
       "title": "Patch generation with language models: Feasibility and scaling behavior",
       "authors": ["Samuel D. Kolak", "Ruben Martins", "Claire Le Goues", "Vincent J. Hellendoorn"],
       "year": 2022,
       "relevance": "Prior work on PLMs for APR showing scaling behavior, evaluated on a smaller scale."
     },
     {
       "title": "Can OpenAI's Codex fix bugs?: An evaluation on QuixBugs",
       "authors": ["Julian A. Prenner", "Hlib Babii", "Romain Robbes"],
       "year": 2022,
       "relevance": "Prior small-scale evaluation of Codex for bug fixing on the QuixBugs benchmark."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Demonstrates LLM capability in generating competition-level code, relevant to code generation evaluation."
     },
     {
       "title": "Cure: Code-aware neural machine translation for automatic program repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "relevance": "State-of-the-art learning-based APR baseline using code-aware NMT techniques."
     },
     {
       "title": "A syntax-guided edit decoder for neural program repair",
       "authors": ["Qihao Zhu", "Zeyu Sun"],
       "year": 2021,
       "relevance": "Introduces Recoder, a learning-based APR tool used as a key baseline comparison."
     },
     {
       "title": "Neural program repair with execution-based backpropagation",
       "authors": ["He Ye", "Matias Martinez", "Martin Monperrus"],
       "year": 2022,
       "relevance": "Introduces RewardRepair, an execution-guided APR technique used as baseline."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Yue Wang", "Weishi Wang"],
       "year": 2021,
       "relevance": "Presents CodeT5, an encoder-decoder model for code tasks evaluated for program repair."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational work on scaling laws that motivates the study of model size effects on APR."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown", "Benjamin Mann"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "Introduces GPT-3 and few-shot learning paradigm used in the study's approach."
     }
   ]
 }