ai-research-survey

scan-v5.json (25834B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Impact of Code Language Models on Automated Program Repair",
     "authors": [
       "Nan Jiang",
       "Kevin Liu",
       "Thibaud Lutellier",
       "Lin Tan"
     ],
     "year": 2023,
     "venue": "International Conference on Software Engineering",
     "arxiv_id": "2302.05020",
     "doi": "10.1109/ICSE48619.2023.00125"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are traceable to tables: 72% improvement (Table II, InCoder-6B 105 vs KNOD 61), 31%-1267% fine-tuning gains (Table IV), and buggy-line degradation (Table III).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (fine-tuning improves fixing; buggy lines hurt pre-trained CLMs) are tested via controlled ablations: same models evaluated with/without fine-tuning and with/without buggy lines provided.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Paper explicitly states focus on Java single-hunk bugs and acknowledges HumanEval-Java contains 'mostly small programs'; implications section calls for larger benchmarks, bounding the scope.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes CodeT5's poor pre-tuning performance to its pre-training tasks and CLM confusion about buggy lines to lack of pre-training signal, but does not systematically consider alternative explanations (e.g., architecture differences, prompt format biases).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly argues against BLEU scores as proxies and validates patches via developer-written test cases followed by manual semantic equivalence checking, directly measuring bug-fixing success.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII 'Threats to Validity and Limitations' is a dedicated section covering data leakage, Codex exclusion, and patch correctness evaluation concerns.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific: identifies that four Defects4J repositories appear in CodeSearchNet and the whole Defects4J repository is in BigQuery; discusses Codex's continuous model updates as a specific evaluation barrier; notes BLEU vs test-case divergence with concrete numbers (RewardRepair CodeBLUE 36.76 vs CodeT5 33.47 despite 19 fewer correct fixes).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Paper states it focuses on Java single-hunk bugs, excludes encoder-only models (CodeBERT, GraphCodeBERT) with justification, excludes Codex because it cannot be fine-tuned, and notes HumanEval-Java contains mostly small programs as a scope limitation.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section explicitly states: 'This work is partially supported by a J.P. Morgan AI Faculty Research Award.'",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed on the title page: Purdue University, Lynbrook High School, and University of Alberta.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "J.P. Morgan is a financial institution with no direct stake in CLM vs. DL-based APR tool performance; the funder does not develop any of the evaluated models.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, no declaration of patents, equity, or consulting relationships; only funding source is mentioned.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "CLM is defined and categorized by architecture (encoder-only, decoder-only, encoder-decoder) in Section II; APR, fine-tuning, single-hunk bugs, plausible vs. correct patches are all explicitly defined.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section I.D enumerates five explicit contributions including the new HumanEval-Java benchmark, the CLM evaluation study, fine-tuning experiments, buggy-line analysis, and efficiency analysis.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section IX explains how this work differs from prior CLM code refinement studies (no BLEU-only metrics, full project context, fine-tuning) and from prior APR techniques; differences are substantive rather than just listing references.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Replication package at https://github.com/lin-tan/clm explicitly includes source code for reproduction, fine-tuned CLM models, HumanEval-Java benchmark, and generated patches.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "HumanEval-Java (new benchmark) is released in the replication package; existing benchmarks (Defects4J, QuixBugs) are publicly available; fine-tuning data comes from prior work [10] which is publicly available.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions GPU memory requirements and batch-size-of-1 hardware constraints but provides no requirements.txt, Dockerfile, or specific CUDA/library version specifications.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The replication package is referenced but no step-by-step reproduction instructions appear in the paper; a reader could not follow the paper alone to reproduce experiments.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are reported as raw counts of correct fixes with no confidence intervals or error bars across any benchmark or model comparison.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are used for any comparative claim; '72% more bugs' and '164% more bugs' are stated as plain comparisons without p-values or effect size tests.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Percentage improvements are reported throughout (31%-1,267% for fine-tuning, 72% for best CLM vs. best APR tool), providing practical effect size information with baseline context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No power analysis or justification is given for the benchmark sizes (130, 108, 40, 164 bugs); existing benchmarks are used as-is without discussion of statistical adequacy.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "A single fixed random seed is used for fine-tuning runs; no variance across multiple runs or seeds is reported for any result.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Four state-of-the-art DL-based APR techniques (CURE, RewardRepair, Recoder, KNOD) are used as baselines throughout Tables II and IV.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include KNOD (ICSE 2023, same venue), RewardRepair (ICSE 2022), and Recoder (FSE 2021); all are top-performing published systems at time of submission.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Multiple ablations are conducted: CLMs with vs. without buggy lines (Table III), CLMs with vs. without fine-tuning (Tables II/IV), and fine-tuning with varying data sizes (Figure 9).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation uses number of correct fixes, compilation rate (Figure 5), and size/time/memory efficiency (Figure 10), providing multiple complementary metrics.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Plausible patches are manually checked for semantic equivalence to developer patches, going beyond automated test-case passing to verify correctness.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Benchmarks are test sets entirely separate from fine-tuning data; fine-tuning uses GitHub commits, and test benchmarks (Defects4J, QuixBugs, HumanEval-Java) are held out.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by all four benchmarks (Defects4J v1.2, v2.0, QuixBugs, HumanEval-Java) and by each individual CLM in Tables II, III, and IV.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Figures 6, 7, and 8 provide concrete code-level examples of failure cases with analysis of why CLMs fail (e.g., insufficient context for Math-75, confusion from buggy lines for FLIP_CASE).",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Multiple negative results are explicitly reported: CodeT5 performs poorly without fine-tuning, all CLMs fix fewer bugs when buggy lines are provided, and too much fine-tuning data degrades performance for CodeT5 and CodeGen.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model names with parameter counts are specified for all 10 CLMs (e.g., PLBART-base 140M, CodeT5-large 770M, CodeGen-6B, InCoder-6B) with data sources documented in Table I.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Figure 3 shows exact prompt templates for all four CLM architectures (PLBART, CodeT5, CodeGen, InCoder) both with and without buggy lines, using a real example from Defects4J.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Fine-tuning hyperparameters are fully reported: batch size 1, Adam optimizer, learning rate 1e-5, one training epoch, fixed random seed, and 10 candidate patches per bug.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; CLMs are applied directly via prompts without tool use or multi-step orchestration.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Fine-tuning data preprocessing is documented: 143,666 total instances randomly split 90/10 into training (129,300) and validation (14,366); HumanEval-Java creation process (manual Python-to-Java conversion, bug injection) is described in detail.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Replication package explicitly includes 'the generated patches for all four benchmarks by all CLMs,' making raw model outputs available for independent verification.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "HumanEval-Java creation is described in detail (Section III-B); existing benchmark sources are cited; fine-tuning data source (GitHub commits from prior work [10]) is specified with provenance.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants were recruited; the study uses existing code benchmarks and developer-written test cases.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Full pipeline is documented: prompt construction → CLM generation of 10 candidate patches → test case execution for plausibility → manual semantic verification for correctness.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": true,
           "justification": "Training data sources and composition are documented for each CLM in Table I and text (e.g., BigQuery, CodeSearchNet, GitHub repositories); the contamination threat is treated as a primary concern with data source documentation.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Extensively discussed: 'four repositories used by the Defects4J benchmark are also in CodeSearchNet, and the whole Defects4J repository is included by BigQuery'; this motivates the creation of HumanEval-Java.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "A new benchmark (HumanEval-Java) is explicitly created to avoid contamination, and Codex is flagged as 'particularly susceptible to the data-leaking threat' due to continuous model updates.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Time efficiency (GPU seconds per correct fix) and memory efficiency (GPU GB required) are reported for all 10 CLMs in Figure 10(b) and 10(c), enabling practical cost comparison.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware constraints are mentioned (batch size 1) and per-model memory is shown, but total GPU-hours for fine-tuning all 10 models or running all experiments is not reported.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "InCoder-6B without fine-tuning fixes 72% more bugs than the best DL-based APR technique (KNOD) across four benchmarks.",
       "evidence": "Table II: InCoder-6B fixes 105 total bugs vs. KNOD's 61 across all four benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "Providing buggy lines to CLMs without fine-tuning consistently reduces their bug-fixing performance by 6%-78%.",
       "evidence": "Table III shows all 10 CLMs fix fewer bugs when buggy lines are provided (reductions range from -6 for InCoder-6B to -36 for PLBART-base).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning with APR data improves CLMs' fixing capabilities by 31%-1,267%, with fine-tuned CLMs outperforming the best DL-based APR technique by 46%-164%.",
       "evidence": "Table IV: InCoder-6B fine-tuned fixes 161 bugs vs. KNOD's 61 (164% more); PLBART-base gains 31% improvement.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuned CLMs over-rely on buggy lines, missing bugs that pre-trained CLMs can fix without fine-tuning.",
       "evidence": "Figure 8(b) shows SORT_NUMBERS fixed by pre-trained CLMs but all fine-tuned variants fail; Section V-B discusses the mechanism.",
       "supported": "moderate"
     },
     {
       "claim": "CodeT5 and InCoder models have the best size efficiency; PLBART, CodeT5, and InCoder have better time and memory efficiency than CodeGen.",
       "evidence": "Figure 10 shows CodeT5 and InCoder fix most bugs per parameter count; CodeGen requires 3.64-13.88s/fix vs. PLBART's 0.70-0.89s/fix.",
       "supported": "strong"
     },
     {
       "claim": "Too much fine-tuning data reduces performance: CodeT5 and CodeGen peak at 10,000 instances; more data causes 8%-19% regression.",
       "evidence": "Figure 9 shows peak at 10K for CodeT5-large (59 fixes) and CodeGen-6B, declining with full 129K dataset.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Pre-trained CLMs already outperform specialized DL-based APR techniques, with InCoder-6B fixing 72% more bugs than KNOD without any task-specific training. Counterintuitively, providing buggy lines to pre-trained CLMs reduces their performance, suggesting they are not pre-trained to use error information. Fine-tuning resolves this and yields 31%-1,267% improvement, enabling fine-tuned CLMs to fix 46%-164% more bugs than the best existing APR technique, but introduces over-reliance on buggy lines as a new failure mode. CodeT5 and InCoder architectures offer the best size efficiency, suggesting future APR work should focus on scaling these model families rather than training from scratch.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (72% more bugs, 164% more bugs) are made without p-values, confidence intervals, or any statistical significance tests, making it impossible to assess whether differences could be due to chance."
     },
     {
       "flag": "Single run, no variance reported",
       "detail": "A single fixed random seed is used for all fine-tuning experiments; no variance across seeds or runs is reported, so reliability of the performance numbers is unknown."
     },
     {
       "flag": "Manual patch verification subjectivity",
       "detail": "Patch correctness is determined by manual inspection for semantic equivalence; no inter-rater reliability or second reviewer is mentioned, introducing potential bias from authors evaluating their own approach."
     },
     {
       "flag": "Batch size constrained by hardware",
       "detail": "Fine-tuning uses batch size of 1 'due to hardware constraints,' which is non-standard and may produce suboptimal fine-tuned models, making performance comparisons less reliable."
     },
     {
       "flag": "HumanEval-Java contamination assumption",
       "detail": "The paper assumes HumanEval-Java is unseen by CLMs because it was manually created, but does not verify this empirically (e.g., by checking if HumanEval Python is in training corpora that CLMs might have learned Java analogs from)."
     }
   ],
   "cited_papers": [
     {
       "title": "Defects4J: A Database of Existing Faults to Enable Controlled Testing Studies for Java Programs",
       "relevance": "Primary evaluation benchmark providing Java bugs with developer-written test cases"
     },
     {
       "title": "KNOD: Domain Knowledge Distilled Tree Decoder for Automated Program Repair",
       "relevance": "State-of-the-art DL-based APR baseline and key comparison target"
     },
     {
       "title": "Neural Program Repair with Execution-Based Backpropagation (RewardRepair)",
       "relevance": "Contemporary DL-based APR baseline using execution feedback during training"
     },
     {
       "title": "A Syntax-Guided Edit Decoder for Neural Program Repair (Recoder)",
       "relevance": "DL-based APR baseline with AST-level patch generation"
     },
     {
       "title": "CURE: Code-Aware Neural Machine Translation for Automatic Program Repair",
       "relevance": "DL-based APR baseline and prior work by the same first and last authors"
     },
     {
       "title": "CodeT5: Identifier-Aware Unified Pre-Trained Encoder-Decoder Models for Code Understanding and Generation",
       "relevance": "One of four CLM architectures evaluated, with multi-task pre-training design studied"
     },
     {
       "title": "A Conversational Paradigm for Program Synthesis (CodeGen)",
       "relevance": "Decoder-only CLM architecture evaluated in the APR context"
     },
     {
       "title": "InCoder: A Generative Model for Code Infilling and Synthesis",
       "relevance": "Best-performing CLM in the study, using masked span prediction for code infilling"
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (Codex/HumanEval)",
       "relevance": "Source of HumanEval benchmark used as basis for new HumanEval-Java benchmark; Codex discussed as excluded baseline"
     },
     {
       "title": "Trust Enhancement Issues in Program Repair",
       "relevance": "Survey cited for developer willingness thresholds (93% review ≤10 patches, 63% expect <1 hour response) motivating the evaluation protocol"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable for software engineering practitioners — shows that fine-tuning off-the-shelf CLMs on APR data outperforms purpose-built APR tools, with efficiency analysis enabling informed deployment choices."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Two counterintuitive findings: CLMs without fine-tuning already beat specialized APR tools, and providing more information (buggy lines) makes CLMs perform worse."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns raised; paper focuses on software reliability improvement."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly challenges the direction of specialized DL-based APR research by showing general CLMs dominate, but framed constructively rather than controversially."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Replication package with fine-tuned models and HumanEval-Java benchmark is released, allowing practitioners to test the models directly."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Purdue University and University of Alberta; no famous AI lab or high-profile product association."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }