ai-research-survey

scan.json (32767B)

---

 {
   "paper": {
     "title": "Lost in Translation: A Study of Bugs Introduced by Large Language Models while Translating Code",
     "authors": [
       "Rangeet Pan",
       "Ali Reza Ibrahimzada",
       "Rahul Krishna",
       "Divya Sankar",
       "Lambert Pouguem Wassi",
       "Michele Merler",
       "Boris Sobolev",
       "Raju Pavuluri",
       "Saurabh Sinha",
       "Reyhaneh Jabbarvand"
     ],
     "year": 2024,
     "venue": "ICSE 2024",
     "arxiv_id": "2308.03109",
     "doi": "10.1145/3597503.3639226"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Artifacts including manual labeling and automation scripts are publicly available at GitHub (reference [7]: https://github.com/Intelligent-CAT-Lab/PLTranslationEmpirical). Explicitly stated: 'our artifacts, including manual labeling and automation scripts for evaluating LLMs, are publicly available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the dataset consists of '1,700 code samples in five PLs with 10K+ tests, 43K+ translated code, 1,748 manually labeled bugs, and 1,365 bug-fix pairs' and is publicly available via the artifact website [7]. Additionally, all source benchmarks (CodeNet, Avatar, EvalPlus) are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 specifies compute resources (16 A100 80GB GPUs) and exact compiler/runtime versions: Python 3.10, g++ 11, GCC Clang 14.0, Java 11, Go 1.20, Rust 1.73, .Net 7.0.14. Model sizes and context windows are in Table 1."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a public artifact website [7] with 'automation scripts for evaluating LLMs' and states all prompts used are in the replication package. Combined with the detailed methodology in §2-§6, this is sufficient for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-5 and Figures 2, 6-7 are reported as point estimates (percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 outperforms other models and that prompt crafting improves results, but no significance tests are applied. The only statistical measure mentioned is a correlation coefficient r (0.64-0.85) in §3.1 without a significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context: e.g., 'correct translations ranging from 2.1% to 47.3%' (Table 2), 'improves the success rate of GPT-4, StarCoder, Codegen, and Llama 2 by 12.33%, 3.55%, 2.65%, 1.97%, respectively' (§6). Absolute percentages with baselines provide effect size context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 200 samples per language in CodeNet, 249/250 in Avatar, or 164 in EvalPlus. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 explicitly acknowledges: 'for each translation task, we performed the translation once.' Single-run results with no variance, standard deviation, or spread reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 compares LLM-based translation against non-LLM-based approaches: CxGo (C→Go), C2Rust (C→Rust), and JavaToCSharp (Java→C#). Table 5 presents the comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The LLMs selected include the then-recent GPT-4 (Mar 2023), StarCoder (May 2023), Llama 2 (Jul 2023). Non-LLM baselines (CxGo, C2Rust, JavaToCSharp) are actively maintained open-source tools. Selection rationale is explained in §2."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The prompt crafting approach (§6) has identifiable components (source code, incorrect translation, error details, expected behavior) but these are not ablated individually. No experiments isolate the contribution of each prompt component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by outcome type: compilation error, runtime error, functional error, non-terminating execution, and successful translation (§3.2, Table 3, Figure 2). Bug categories provide additional evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Eight human labelers manually investigated 1,748 buggy translations over 630 person-hours to create the bug taxonomy (§4.1.1). This is direct human evaluation of the LLM translation outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses established benchmarks (CodeNet, Avatar, EvalPlus) with their provided test suites. No training or tuning is performed — LLMs are evaluated in zero-shot fashion, so the test data is fully independent."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns provided: per source/target language (Table 2, Table 3), per model (Figure 2), per bug category (Table 4), per dataset (Table 2), per iteration of prompt crafting (Figure 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure analysis is the core contribution. Section 4 presents 15 categories of translation bugs with concrete code examples. Section 4.3 discusses specific failure modes in real-world projects."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: most LLMs perform poorly (2.1-5.3% success for 5/7 models), all models fail on real-world projects (0% except GPT-4 at 8.1-13.6%), prompt crafting has diminishing returns (iter2 only +1.7%), and some prompt crafting attempts degrade outcomes (§6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'correct translations ranging from 2.1% to 47.3%' matches Table 2; '15 categories of translation bugs' matches §4; 'prompt-crafting approach... improves performance by 5.5% on average' matches §6; '1,700 code samples from three benchmarks and two real-world projects' matches §2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that prompt crafting with error context improves translation (§6). This is supported by a controlled before/after comparison using the same models and data, with the only variable being the prompt content. The design (vanilla → iter1 → iter2) is adequate for this claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly discusses generalization boundaries: results are tied to five specific PLs, seven specific LLMs, and specific datasets. The paper notes 'our selection is drawn from the ranking at the time of our experimentation' and acknowledges these choices may limit generalizability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses several alternatives: LLM non-determinism could change success rates, sensitivity to prompt templates could affect results (they followed best practices per model), lack of inter-rater reliability metric could affect taxonomy, and weak test suites could overcount successes."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures test-case passing as a proxy for translation correctness and explicitly discusses this limitation in §8 (Construct Validity): 'inadequate or weak test suites can cause buggy translations that pass the test suites to be considered correct.' They note CodeNet has only one test per sample, making it more susceptible."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 lists model names with parameter counts and release dates, but no exact version identifiers. GPT-4 is listed without a version/snapshot (e.g., 'gpt-4-0613'). Open-source models lack commit hashes or version tags. Only generic names like 'Llama 2 13B' are given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the three vanilla prompting templates for different models (CodeGeeX, GPT-4, other models). Figure 5 shows the complete prompt crafting template. The paper also states 'All the prompts that we used are in the replication package [7].' The fill values ($SOURCE_CODE, $SOURCE_LANG, $TARGET_LANG) are mechanical substitutions from the benchmarks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for any of the seven LLMs. Context window sizes are listed in Table 1, but no inference-time hyperparameters are stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct prompting of LLMs with source code, with no tool use, retry logic, or agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes dataset selection criteria (PL popularity, paradigm coverage, dataset quality), how real-world projects were broken into classes/files, and that all comments were removed to fit context windows. The evaluation setup (compilers, test execution) is also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Threats to Validity' is a dedicated section with three subsections: External Validity, Internal Validity, and Construct Validity, providing substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: single translation per task and its impact on success rates (Internal), lack of inter-rater reliability metric for manual labeling (Internal), CodeNet having only one test case per sample (Construct), prompt sensitivity mitigated by following model-specific best practices (Internal), and potential bugs in automation scripts (Internal)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "External validity section explicitly states scope boundaries: limited to five PLs selected by specific criteria, limited LLMs constrained by 20B parameter budget, specific datasets chosen. Internal validity notes single-run limitation. Construct validity notes test suite adequacy concerns."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The artifact [7] includes 1,700 code samples, 10K+ tests, 43K+ translated code, 1,748 manually labeled bugs, and 1,365 bug-fix pairs. All raw translations and labels are publicly available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the dataset collection: three benchmarks (CodeNet, Avatar, EvalPlus) selected by stated criteria, two real-world projects (Commons CLI, Click) selected as well-maintained Java/Python projects providing command-line processing APIs. Table 2 provides detailed statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The eight labelers who built the bug taxonomy are members of the research team, not recruited participants. Data sources are standard public benchmarks and open-source projects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is described: dataset collection and selection (§2) → LLM prompting with specific templates (§3) → execution and outcome classification (§3.2) → manual bug labeling in two phases with 8 labelers (§4.1.1) → prompt crafting iteration (§6). Real-world project preprocessing (breaking into files, removing comments) is also documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work is supported by IBM-Illinois Discovery Accelerator Institute and NSF CCF 22-38045 CAR grants.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are prominently listed: 8 authors from IBM Research Yorktown Heights and 2 from University of Illinois Urbana-Champaign. One author was an IBM Research intern."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "IBM funds the research via the IBM-Illinois Discovery Accelerator Institute, and 8/10 authors are IBM Research employees. IBM has commercial interests in code translation and LLM-based developer tools (e.g., watsonx Code Assistant), making the funder not fully independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper. Given 8/10 authors are IBM employees and IBM has commercial products in the code translation space, this is a notable omission."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the seven LLMs. Table 1 lists release dates but not training data cutoffs. This is critical since CodeNet (2021), Avatar (2021), and EvalPlus (2023) could overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (CodeNet, Avatar, EvalPlus code samples) appeared in any model's training data. Given these are public datasets and the models were released in 2023, overlap is plausible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeNet was published in 2021 and Avatar in 2021, before the training cutoffs of all seven models (released 2023). No discussion of contamination risk despite the high likelihood that these benchmarks are in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The labelers building the bug taxonomy are researchers/co-authors, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on benchmarks and involves researcher-performed manual analysis, not human subjects research."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The eight labelers are described as 'researchers or software engineers in the industry' but they are part of the research team, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, wall-clock time per translation, or cost per example are reported despite performing 43,379 translations including GPT-4 API calls."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (16 A100 80GB GPUs) but total GPU hours, training time, or total API spend are not quantified. The 630 person-hours for manual labeling is stated but computational budget is not."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 acknowledges: 'for each translation task, we performed the translation once. Performing a translation task multiple times may change the success rates in translation as LLMs are inherently non-deterministic.' Single-run results only."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states: 'for each translation task, we performed the translation once.' The number of runs (1) is clearly stated, even though it is a limitation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Temperature, sampling parameters, and alternative prompt templates were not systematically explored. The paper follows 'best practices described in the respective artifacts/papers/reports' but reports no search process."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates are justified by following each model's recommended format: 'we followed the templates similar to those we found in the artifacts, papers, or technical reports associated with each model' (§3.1). No configuration search was performed, making cherry-picking unlikely."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes dozens of comparisons across 7 models, 31 language pairs, and 5 datasets without any statistical testing, let alone multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompt crafting technique (§6) is the authors' own contribution evaluated against vanilla prompting, but no discussion of author-evaluation bias. The authors also re-implement the evaluation for non-LLM baselines without acknowledging this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 13B parameters to GPT-4 (unknown size) with vastly different computational costs. GPT-4 (47.3%) is compared directly to 13B models (2.1-5.3%) without discussing the compute difference. Performance is not reported as a function of compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Construct Validity) discusses whether the evaluation actually measures translation quality: test-based assessment vs. static metrics, risk of weak test suites (especially CodeNet with 1 test per sample), and why they chose execution-based over static metrics. Section 3.1 also explains why they rejected static metrics like CodeBLEU."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models receive direct prompts without agentic workflows, tool use, or scaffolding layers."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. CodeNet (2021), Avatar (2021), and EvalPlus (2023) contain code that could have been in the training data of models released in 2023. No temporal analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation provides source code in one language and expects translation to another — it is not analyzed whether the models might have seen the target code paired with the source code during training."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. CodeNet problems come from online judge systems (AIZU, AtCoder) where solutions in multiple languages exist publicly. Models could have trained on both source and target language versions of the same problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, n-gram overlap analysis, or decontamination is mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs achieve correct translations ranging from 2.1% to 47.3% across studied models",
       "evidence": "Table 2 shows total/average success rates: GPT-4 47.3%, StarCoder 14.5%, CodeGen 8.1%, TB-Airoboros 5.3%, Llama 2 3.5%, CodeGeeX 2.8%, TB-Vicuna 2.1% (§3.1)",
       "supported": "strong"
     },
     {
       "claim": "77.8% of unsuccessful translations result in compilation errors, indicating LLMs struggle with code syntax",
       "evidence": "Table 3 and Figure 2 break down unsuccessful translations: 77.8% compilation errors, 8.4% runtime errors, 13.4% functional errors, 0.4% non-terminating (§3.2)",
       "supported": "strong"
     },
     {
       "claim": "15 categories of translation bugs organized into 5 groups were identified through manual analysis",
       "evidence": "630 person-hours of manual labeling by 8 human labelers examining 1,748 buggy GPT-4 translations across 31 language pairs (§4.1). Taxonomy presented in Figure 3 and Table 4.",
       "supported": "strong"
     },
     {
       "claim": "More than one-third (33.5%) of translation bugs are data-related bugs",
       "evidence": "Table 4 shows data-related bugs (C1+C2+C3) account for 33.5% of all bugs, with incorrect input parsing (18.1%) and incorrect data type (11.5%) being the most common (§4.2)",
       "supported": "strong"
     },
     {
       "claim": "LLMs fail to translate real-world projects effectively",
       "evidence": "Table 2: Commons CLI Java→Python success rate 13.6% for GPT-4, 0% for all others. Click Python→Java: 0% for all models including GPT-4 (§3.1, §4.3)",
       "supported": "strong"
     },
     {
       "claim": "C2Rust achieves 95% success rate on C-to-Rust translation, 34% better than GPT-4",
       "evidence": "Table 5 shows C2Rust 95% vs GPT-4 61% for C→Rust on CodeNet. However, C2Rust generates unsafe, non-idiomatic code (§5, Figure 4)",
       "supported": "strong"
     },
     {
       "claim": "Prompt crafting with error context improves translation success rate by 5.5% on average across LLMs",
       "evidence": "Figure 6 shows iter1 improvements: GPT-4 +12.33%, StarCoder +3.55%, CodeGen +2.65%, Llama 2 +1.97%. GPT-4 iter2 adds +1.7%. Average ~5.5% (§6)",
       "supported": "moderate"
     },
     {
       "claim": "LLM-based and non-LLM-based translation techniques have complementary strengths",
       "evidence": "Table 5 and §5: CxGo achieves 62.3% on C→Go (vs GPT-4 72.5%), C2Rust 95% on C→Rust (vs GPT-4 61%), but JavaToCSharp 0% on Java→C# (vs GPT-4 49-59.2%). Table 6 summarizes: non-LLM better at broader context and determinism; LLMs better at target-language idioms.",
       "supported": "moderate"
     },
     {
       "claim": "Strong correlation exists between test suite rigor and unsuccessful translation detection",
       "evidence": "§3.1 reports correlation coefficients r ranging from 0.64 to 0.85 across all models between average tests per sample and unsuccessful translation rate.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "LLMs are not yet reliable for automated code translation, with success rates ranging from 2.1% (TheBloke-Vicuna) to 47.3% (GPT-4) across five programming languages. Manual analysis of 1,748 buggy GPT-4 translations revealed 15 bug categories in 5 groups, with data-related bugs (33.5%) and syntactic/semantic differences (30.5%) being the most prevalent. Real-world project translation is essentially infeasible for current LLMs. An iterative prompt-crafting technique providing error context improves success by 5.5% on average, with GPT-4 seeing the largest gain (12.3%), but considerable room for improvement remains.",
   "red_flags": [
     {
       "flag": "Single-run evaluation",
       "detail": "Each translation task was performed only once despite LLMs being inherently non-deterministic. The paper acknowledges this in §8 but argues it does not affect bug characterization. However, reported success rates could vary significantly across runs."
     },
     {
       "flag": "No contamination analysis",
       "detail": "CodeNet (2021) and Avatar (2021) are public datasets that likely appeared in the training data of models released in 2023. No contamination analysis is performed, which could inflate success rates (models may have memorized solutions) or distort the bug distribution."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (GPT-4 vs other models, prompt crafting improvements, LLM vs non-LLM) are based on comparing point estimates without statistical testing. With single-run evaluations, the observed differences may not be reliable."
     },
     {
       "flag": "No inter-rater reliability metric",
       "detail": "The bug taxonomy was constructed by 8 labelers over 630 hours, but no Cohen's kappa, Fleiss' kappa, or other inter-rater reliability metric is reported. The paper acknowledges this in §8 and describes a discussion-based resolution process, but quantitative agreement is not measured."
     },
     {
       "flag": "Hyperparameters not reported",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the 7 LLMs. These significantly affect output quality and reproducibility."
     },
     {
       "flag": "Potential funder conflict",
       "detail": "8/10 authors are IBM Research employees and the work is partially funded by IBM. IBM has commercial interests in code translation tools, though the paper's findings (LLMs are unreliable) do not obviously promote IBM products."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark for evaluating LLM code generation capabilities."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "One of the best-performing open-source code LLMs evaluated in the study (14.5% success rate)."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang"],
       "year": 2022,
       "relevance": "Open-source code LLM evaluated in the study for code translation capability."
     },
     {
       "title": "CodeGeeX: A Pre-trained Model for Code Generation with Multilingual Evaluations on HumanEval-X",
       "authors": ["Qinkai Zheng", "Xiao Xia"],
       "year": 2023,
       "arxiv_id": "2303.17568",
       "relevance": "Code LLM specifically trained for multilingual code generation and translation, evaluated in the study."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "EvalPlus benchmark used in this study; addresses rigorous evaluation of LLM-generated code."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["Meta AI"],
       "year": 2023,
       "relevance": "Open-source general LLM evaluated in the study for code translation."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Best-performing LLM in the study (47.3% success rate); the primary model for the bug taxonomy analysis."
     },
     {
       "title": "Avatar: A parallel corpus for Java-Python program translation",
       "authors": ["Wasi Uddin Ahmad", "Md Golam Rahman Tushar"],
       "year": 2021,
       "arxiv_id": "2108.11590",
       "relevance": "Parallel corpus benchmark used in the study for evaluating code translation between Java and Python."
     },
     {
       "title": "CodeNet: A Large-Scale AI for Code Dataset for Learning a Diversity of Coding Tasks",
       "authors": ["Ruchir Puri", "David S Kung"],
       "year": 2021,
       "relevance": "Primary benchmark dataset with 1,000 code samples in 5 PLs used for the translation evaluation."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "LLM-based program repair via zero-shot prompting; inspired the prompt crafting approach for fixing translation bugs."
     },
     {
       "title": "Repair is nearly generation: Multilingual program repair with LLMs",
       "authors": ["Harshit Joshi", "José Cambronero Sanchez"],
       "year": 2023,
       "relevance": "Demonstrates LLMs can repair code bugs, motivating the iterative prompt-crafting approach for translation bugs."
     },
     {
       "title": "Unsupervised translation of programming languages",
       "authors": ["Baptiste Roziere", "Marie-Anne Lachaux"],
       "year": 2020,
       "relevance": "Neural approach to code translation between C++, Java, and Python using unsupervised learning."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["Frank F Xu", "Uri Alon"],
       "year": 2022,
       "relevance": "Systematic evaluation of code LLMs including PolyCoder; relevant to understanding LLM code capabilities."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained model for code understanding; foundational work in code LLMs."
     }
   ]
 }