ai-research-survey

scan.json (27367B)

---

 {
   "paper": {
     "title": "Assessing the Latent Automated Program Repair Capabilities of Large Language Models using Round-Trip Translation",
     "authors": [
       "Fernando Vallecillos Ruiz",
       "Anastasiia Grishina",
       "Max Hort",
       "Leon Moonen"
     ],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology (TOSEM)",
     "arxiv_id": "2401.07994",
     "doi": "10.1145/3771922"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a Zenodo replication package: 'The replication package is available for download from Zenodo: https://doi.org/10.5281/zenodo.10500593' (footnote 1, Section 1). The code for RTT and results are released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All four benchmarks (QuixBugs, HumanEval-Java, Defects4J v1.2, Defects4J v2.0) are publicly available. The paper also releases their manual assessment of over 5,000 patches alongside the replication package (Section 5.2.2)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.9 specifies hardware: '3 NVIDIA V100 GPUs or 2 NVIDIA A100 GPUs' and '32-Core AMD EPYC 7601 CPU with 2TB RAM.' Model versions and hyperparameters are in Tables 1 and 2. Specific model URLs are provided in footnotes 4-10. However, no requirements.txt or Dockerfile is mentioned in the paper itself; the environment details are described in prose and via model links."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a Zenodo replication package (footnote 1) and states 'We release the code for RTT and results obtained to ensure replication and verification of our work.' The pipeline is described in detail in Sections 4.3-4.6 with exact prompts, hyperparameters, and postprocessing steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports standard deviations (e.g., Table 3 and Table 4 with '± STD') but does not report confidence intervals or error bars. Standard deviation is reported as a spread measure but no formal confidence intervals are constructed."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'a strong correlation is observed,' 'larger models consistently provide better results') and reports Pearson's r correlations but does not perform statistical significance tests (no p-values, no hypothesis tests) for any comparisons between models or approaches."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute counts of bugs fixed with baseline context (e.g., 'plausible patches for 100 of 164 bugs with GPT-4,' '97 were found to be correct,' '46 bugs that were missed by LLMs specifically fine-tuned for APR'). Tables 3-7 provide full breakdowns. Pearson's r correlation values are reported (0.60, 0.77, 0.22, 0.15)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 10 runs, the selection of 9 models, or the use of these specific 4 benchmarks beyond stating they are 'diverse' and 'widely used.' No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported across 10 runs for all open-source models in Tables 3, 4, and 7 (e.g., 'SantaCoder: 31.4 ± 2.0'). The paper also reports Any Run and Every Run aggregations to show variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against prior work by Jiang et al. [25], including base models, fine-tuned models, and DL-based APR techniques (CURE, Reward, Recoder, KNOD) in Tables 5 and 6. Figure 4 directly compares GPT-4 RTT against 10 LLMs from prior work."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines from Jiang et al. [25] are from ICSE 2023 and include models like InCoder-6.7B and fine-tuned variants. The paper also discusses related concurrent work using GPT-3.5 with 200 patches [74]. The comparison is against reasonably recent APR methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies multiple components: intermediate representation (PL vs NL, Section 5.1 vs 5.2), temperature settings (Section 5.3, Table 8), model sizes (Tables 3-4), and specific intermediate programming languages (C#, C++, Python). These variations function as ablations of the RTT pipeline."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 lists seven metrics: compilability, plausibility, test pass rate, exact match, BLEU, and CodeBLEU. Results are reported across multiple metrics in Sections 5.1-5.4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.8 describes manual assessment of correctness for HumanEval-Java patches. Table 7 reports manually verified correct fixes. Over 5,000 patches were manually assessed. Section 5.5 provides qualitative analysis of generated patches."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks have fixed test suites. HumanEval-Java was specifically constructed after training of most models to mitigate data leakage (Section 4.2). The paper explicitly discusses this: 'HumanEval-Java was not available during training of most the LLMs used in this study.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (QuixBugs, HumanEval-Java, Defects4J v1.2, v2.0), per model, per intermediate language, and per temperature setting. Tables 3-8 and Figures 3-8 provide detailed breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 provides detailed qualitative analysis of failure cases: misleading method names, common logical problems (e.g., CORRECT_PARENTHESIS generating checks for multiple bracket types), linked list handling failures, repetitive token generation by smaller models. Section 5.4.6 discusses limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that RTT through PL yields very low plausibility rates (Table 3, at most 5 bugs). The paper explicitly states 'RTT does not outperform state-of-the-art NMT and cloze-style fine-tuned models for APR' (Section 8). Section 5.4.6 discusses RTT diluting coding style."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'plausible patches for 100 of 164 bugs with GPT-4 on HumanEval-Java' (Table 4), '97 are found to be correct' (Table 7), and '46 bugs that were missed by LLMs specifically fine-tuned for APR' (Section 5.2.1). The abstract also notes limitations, consistent with Sections 5.4.6 and 8."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim is that RTT can fix bugs via 'regression toward the mean' (Section 3.1). This is framed as a hypothesis and tested empirically. The ablation-style experiments (varying intermediate languages, temperatures, model sizes) provide controlled evidence. The paper is appropriately cautious, stating 'we hypothesize' rather than making strong causal assertions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Threats to Validity) explicitly bounds generalization: 'we focused our evaluation on single-hunk bugs, therefore, effectiveness may not transfer to more complex multi-hunk or multi-file bugs.' The paper notes results are limited to Java and to nine transformer-based models. External validity threats are specifically discussed."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses data leakage as an alternative explanation ('there is a risk that they were used during training'), mitigated by HumanEval-Java and low exact match rates. The paper also discusses that test-suite-based plausibility may overfit to tests (construct validity). Section 5.5 discusses data leakage for specific examples (e.g., is_prime method possibly memorized)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 3 specifies exact API model versions: 'gpt-3.5-turbo-1106, gpt-4-0613, and gpt-4o-mini-2024-07-18.' Open-source models are specified by name and parameter count (e.g., 'PLBART-base (140M)', 'InCoder 1.3B and 6.7B', 'StarCoderBase 15.5B') with links to specific model pages in footnotes 4-10."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 provides exact prompt text for both GPT models (system message and user prompts for forward and backward translation) and open-source models (infilling prompt templates with actual format). The prompts include the actual text sent to models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports all hyperparameters: number of beams, temperature for both legs, and top-p for all nine models. Section 4.4 explains the rationale for each setting. Section 5.3 provides a temperature sweep analysis."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The RTT pipeline is not agentic scaffolding — it is a two-step translation pipeline without feedback loops, memory, or tool use. The pipeline is fully described but does not constitute agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes the four-step pipeline (preprocessing, translation, postprocessing, evaluation). Section 4.6 details postprocessing for method signature overwriting (Figure 2). The preprocessing includes extracting buggy functions, adding prefixes/suffixes/masks, and removing newlines (Step 1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 ('Threats to Validity') provides a dedicated, substantive discussion of four threat categories: internal, construct, external, and conclusion validity, structured following Wohlin et al. [70]. Section 5.4.6 also discusses limitations of RTT for APR."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: data leakage risk for specific benchmarks, plausibility vs. correctness distinction, single-hunk bug limitation, GPT models run only once due to seed unavailability, manual correctness assessment limited to HumanEval-Java due to resource constraints (Section 4.8), and cost constraints for GPT-4 temperature sweep."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'we focused our evaluation on single-hunk bugs, therefore, effectiveness may not transfer to more complex multi-hunk or multi-file bugs' (Section 6). It states RTT was only applied to Java. It notes 'we applied the approach using only nine transformer-based models' and that extending requires models meeting RTT requirements (Section 6)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replication package (https://doi.org/10.5281/zenodo.10500593) contains the code and results. The manual assessment of over 5,000 patches is released (Section 5.2.2). Results were also reported to Weights & Biases (Section 4.7)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks are well-established and described in Section 4.2 (QuixBugs: 40 bugs, HumanEval-Java: 164 bugs, Defects4J v1.2: 130 single-hunk bugs, Defects4J v2.0: 89 single-hunk bugs). The generation procedure (5 forward translations x 5 backward translations = 25 candidate patches per bug, 10 runs with different seeds) is described in Section 4.3."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The manual assessment was done by the paper's authors (reviewers examining patches). Standard benchmarks were used rather than recruited data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 4.3 (four steps: preprocessing, round-trip translation, postprocessing, evaluation). Section 4.6 describes testability ensurance. The pipeline handles edge cases (context window overflow for one bug with StarCoderBase, Section 4.6)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists funding: Research Council of Norway (secureIT project, IKTPLUSS #288787), European Union Horizon Europe Marie Sklodowska-Curie Actions (#101151798), eX3 infrastructure (#270053), and Sigma2/LUMI supercomputer access."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are affiliated with Simula Research Laboratory, Norway. Affiliations are listed on the first page. No author is affiliated with any of the model providers being evaluated (OpenAI, Meta, etc.)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from public research councils (Research Council of Norway, EU Horizon Europe) that have no financial stake in whether RTT-based program repair succeeds or fails."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state explicit training data cutoff dates for the models. It notes that 'HumanEval-Java was not available during training of most the LLMs used in this study' and that 'HumanEval-Java was constructed after the training of any of the models used in this work (except GPT-4o-mini)' (Section 6), but no specific cutoff dates are provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses data leakage: 'there is a risk that they were used during training.' The paper uses HumanEval-Java specifically to mitigate this. Section 5.1 notes only 0.03% exact match rate across generated patches. Section 5.5 discusses specific cases where data leakage may explain model behavior (is_prime method)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this: HumanEval-Java was created after model training. For older benchmarks, the paper notes '0.03% of the generated candidate patches' are exact matches (Section 6). The qualitative analysis in Section 5.5 examines whether models memorized benchmarks, concluding they did not copy-paste answers."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The manual patch assessment was performed by the paper's authors as expert reviewers, not as human subjects research."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on code benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 13 states: 'The approximate cost for the single run using GPT-4 was ~140USD.' Section 5.3 notes GPT-4 was not included in the temperature sweep because 'it is significantly more expensive than any other model.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.9 specifies hardware: '3 NVIDIA V100 GPUs or 2 NVIDIA A100 GPUs' for patch generation and '32-Core AMD EPYC 7601 CPU with 2TB RAM' for test validation. LUMI supercomputer access is acknowledged. However, total GPU hours or total API spend across all experiments are not explicitly stated."
       }
     }
   },
   "claims": [
     {
       "claim": "RTT through English generates plausible patches for 100 of 164 bugs with GPT-4 on HumanEval-Java, and 97 are correct in manual assessment.",
       "evidence": "Table 4 shows GPT-4 achieves 100 plausible patches on HumanEval-Java in a single run. Table 7 shows 97 manually verified correct patches. Section 5.2.2 describes the manual assessment methodology.",
       "supported": "strong"
     },
     {
       "claim": "RTT uniquely generates plausible patches for 46 bugs not fixed by LLMs fine-tuned for APR.",
       "evidence": "Section 5.2.1, Figure 4, and Table 6 show the comparison. The 46 unique bugs are from the union across all RTT models and benchmarks compared to results from Jiang et al. [25].",
       "supported": "strong"
     },
     {
       "claim": "There is a strong correlation between model size and plausibility rate (Pearson's r = 0.77).",
       "evidence": "Section 5.2 reports Pearson's r = 0.77 for model size vs plausibility rate and r = 0.60 for model size vs compilable patches, excluding models with undisclosed sizes (GPT models).",
       "supported": "moderate"
     },
     {
       "claim": "RTT does not outperform state-of-the-art NMT and cloze-style fine-tuned models for APR overall.",
       "evidence": "Table 5 shows fine-tuned InCoder-6.7B achieves 56/38/24/70 plausible patches on the four benchmarks, while RTT with the same model achieves 10/8/14/41 (Table 4, Any Run). Section 8 explicitly states this limitation.",
       "supported": "strong"
     },
     {
       "claim": "Temperature affects RTT repair performance, with different optimal temperatures for different models.",
       "evidence": "Table 8 shows GPT-3.5 peaks at T=1.0 (130 total plausible patches) while GPT-4o-mini peaks at T=0.6 (138 total). Section 5.3 provides detailed analysis.",
       "supported": "strong"
     },
     {
       "claim": "RTT's hypothesis is that it fixes bugs through regression toward the mean, where LLMs replace infrequent buggy patterns with frequent bug-free patterns.",
       "evidence": "Section 3.1 presents the theoretical motivation based on software naturalness (Ray et al. [54]). The empirical results support the plausibility of this hypothesis but do not definitively prove causation. CodeBLEU analysis (Section 5.4.4) shows patches regress toward similar distributions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper presents the first systematic evaluation of round-trip translation (RTT) with LLMs for automated program repair, testing 9 LLMs across 4 Java benchmarks. GPT-4 using natural language as an intermediate representation achieves 100 plausible patches (97 correct) on 164 HumanEval-Java bugs, and RTT uniquely fixes 46 bugs missed by fine-tuned APR models. RTT through programming languages yields very low fix rates, indicating the intermediate representation must differ sufficiently from the source. While RTT does not outperform state-of-the-art fine-tuned approaches overall, it offers complementary bug-fixing capabilities at no fine-tuning cost.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper reports Pearson correlations and compares models based on raw counts but never performs statistical significance tests. Claims like 'a strong correlation is observed' and comparative statements between models lack formal statistical backing."
     },
     {
       "flag": "GPT models run only once",
       "detail": "GPT-3.5, GPT-4, and GPT-4o-mini were each run only once (due to no seed control at the time and cost constraints), while open-source models were run 10 times. This makes the GPT results less reliable as single-run outcomes, yet GPT-4 is the paper's top-performing model."
     },
     {
       "flag": "Manual assessment limited to one benchmark",
       "detail": "Correctness was manually verified only for HumanEval-Java. For QuixBugs and Defects4J, only plausibility (test pass rate) is reported, which the paper acknowledges 'has inherent limitations, such as ignoring potential overfitting to test code' (Section 6)."
     }
   ],
   "cited_papers": [
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "doi": "10.1109/icse48619.2023.00125",
       "relevance": "Primary comparison baseline; evaluated LLMs for APR on the same four benchmarks with NMT and cloze-style approaches."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark (basis for HumanEval-Java used in this study) and Codex for code generation."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-Trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "doi": "10.1109/icse48619.2023.00129",
       "relevance": "Evaluated LLMs for APR showing that generating more patches improves results; relevant comparison for RTT sampling strategy."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "Demonstrated conversational APR with ChatGPT achieving high fix rates; relevant comparison for RTT's cost-effectiveness."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "doi": "10.1145/3540250.3549101",
       "relevance": "Explored zero-shot LLMs for cloze-style APR; related approach that RTT builds upon conceptually."
     },
     {
       "title": "Fully Autonomous Programming Using Iterative Multi-Agent Debugging with Large Language Models",
       "authors": ["Anastasiia Grishina", "Vadim Liventsev", "Aki Härmä", "Leon Moonen"],
       "year": 2025,
       "doi": "10.1145/3719351",
       "relevance": "Iterative multi-agent debugging framework from the same group; represents agentic approach to program repair that RTT could complement."
     },
     {
       "title": "Reflexion: An Autonomous Agent with Dynamic Memory and Self-Reflection",
       "authors": ["Noah Shinn", "Beck Labash", "Ashwin Gopinath"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Self-reflection framework for LLM agents; relevant to discussion of iterative debugging approaches that RTT could integrate with."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "LLM self-debugging approach relevant to understanding RTT's place in the broader landscape of iterative repair methods."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "doi": "10.1109/sp46214.2022.9833571",
       "relevance": "Assessed security vulnerabilities in LLM-generated code; relevant to RTT's limitation of potentially introducing security flaws."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, the best-performing model in the RTT evaluation."
     },
     {
       "title": "StarCoder: May the Source Be with You!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM used in the RTT evaluation; relevant to understanding open LLMs for code tasks."
     },
     {
       "title": "A Survey of Learning-based Automated Program Repair",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yuxiang Ma", "Weisong Sun", "Zhenyu Chen"],
       "year": 2023,
       "arxiv_id": "2301.03270",
       "relevance": "Survey of learning-based APR methods providing context for where RTT fits in the APR landscape."
     }
   ]
 }