ai-research-survey

scan.json (21583B)

---

 {
   "paper": {
     "title": "CoTran: An LLM-based Code Translator using Reinforcement Learning with Feedback from Compiler and Symbolic Execution",
     "authors": ["Prithwish Jana", "Piyush Jha", "Haoyang Ju", "Gautham Kishore", "Aryan Mahajan", "Vijay Ganesh"],
     "year": 2023,
     "venue": "ECAI-2024 (27th European Conference on Artificial Intelligence)",
     "arxiv_id": "2306.06755"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/PrithwishJana/CoTran. Appendix A.1 states code and dataset are available there."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The AVATAR-TC dataset is released at the same GitHub repository. The paper states 'The AVATAR-TC dataset and all our code can be accessed at https://github.com/PrithwishJana/CoTran.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 states the repository includes 'library dependencies' and Appendix A.2 specifies hardware (four NVIDIA V100 GPUs, 32GB memory, six CPU cores per GPU) and mentions PyTorch. The README reportedly outlines library dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 states 'We have also made it user-friendly with a README file in the root folder that outlines the folder structure, library dependencies, and instructions for running the code.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 2 and 3 report only point estimates. One instance of mean ± stdev is given for EpC in Finding 5 (e.g., '10.98±9.67'), but no confidence intervals or error bars for the main metrics (FEqAcc, CompAcc)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CoTran outperforms baselines based solely on comparing numbers in Tables 2/3. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are consistently reported, e.g., '+14.89% FEqAcc and +8.14% CompAcc for Python-to-Java' with absolute values in tables showing from/to context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set size of 1,746 programs is stated but not justified. No discussion of whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No standard deviation across seeds or multiple runs reported for the main metrics. The EpC mean±stdev in Finding 5 is for error counts, not across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive comparison against 14 tools including 3 human-written transpilers, 3 unsupervised LLM tools, ChatGPT, and 7 supervised LLM tools (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include PPOCoder (2023), TransCoder-ST (2022), ChatGPT (gpt-3.5-turbo-0301, 2023), CodeT5 (2021), PLBART (2021). These are contemporary and competitive for the task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Thorough ablation: CoTran baseline vs. +CF vs. +CF+SF, RL-only vs. RL+SFT interleaved, non-RL alternatives (CoTran+, CoTran×), and kw-Tok contribution. Tables 2 and 3 and Appendix A.3."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six metrics used: FEqAcc, CompAcc, errPos1st, CodeBLEU, BLEU, EM. Plus proposed EpC and f/c-rate metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of translation quality. All evaluation is automated (compilation checks, test-case-based equivalence, BLEU scores). Human judgment of code quality, readability, or correctness of translations is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Explicit train/validation/test split described in Section 4. 'No problem overlaps across splits' ensuring out-of-distribution testing. Results reported on the 1,746 test programs."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides per-sub-dataset breakdown across 7 coding platforms (Aizu, AtCoder, Codeforces, G-CodeJam, LeetCode, GFG, ProjEuler)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Finding 5 discusses compilation error counts. Finding 7 discusses why P2J is harder. Finding 8 discusses failure of function-level tools on whole-program tasks. Appendix A.4 discusses transpiler failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Finding 1 and Appendix A.3 report that non-RL schemes (CoTran+, CoTran×) were tried and underperformed. ChatGPT outperforms CoTran on J2P FEqAcc, which is acknowledged."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 48.68% FEqAcc and 76.98% CompAcc for P2J, and improvements over PLBART-base and CodeT5, are supported by Tables 2/3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about CF and SF improving performance are supported by controlled ablation studies (single-variable manipulation: baseline → +CF → +CF+SF, RL-only vs. RL+SFT). The ablation design is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Code Translator' generally, but results are only for Java-Python pair. While Section 1 mentions 'It can be effortlessly adapted for other language pairs,' no evidence supports this claim. The paper does not bound its generalization to the tested Java-Python setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, improvements could partly be due to kw-Tok rather than the feedback mechanism, but this interaction is not explored. No threats-to-validity section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "CodeT5-base from HuggingFace is specified. ChatGPT version is 'gpt-3.5-turbo-0301' with knowledge cutoff date. Other baselines reference specific model architectures."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The ChatGPT prompt is provided verbatim: 'Translate [S] to [T]:[scode]\\n Do not return anything other than the translated code.' For CoTran itself, prompting is not applicable as it is fine-tuned end-to-end."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 reports: LoRA rank r=16, scaling factor α=32, Adam lr=10^-4, PPO lr=1.41×10^-5, max sequence length 512, ChatGPT temperature=0, top_p=0."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. CoTran is a fine-tuned model, not an agent with tools or feedback loops at inference time."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes data cleaning: parsing with javalang and tokenize modules, manual correction of minor faults, discarding pairs with major issues, output matching criteria (case insensitivity, whitespace removal, etc.). Statistics in Table 1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section in the paper. The conclusion mentions future work (extending to legacy code) but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to the Java-Python pair or the competitive programming domain of AVATAR-TC."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The AVATAR-TC dataset and all translations generated by tools are available at the GitHub repository, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes web-crawling 7 competitive coding platforms, collecting human-written test cases, and the relationship to the AVATAR dataset."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is from public competitive programming platforms (standard benchmark source)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the pipeline: scraping from 7 platforms, parsing with javalang/tokenize, manual correction, discarding faulty pairs, resulting in 57,368 pairs with train/validation/test splits shown in Table 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Georgia Institute of Technology, University of Toronto, UC San Diego, Columbia University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found. The paper uses Symflower, a commercial tool, but no disclosure about the relationship with Symflower is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "For ChatGPT, the paper states 'gpt-3.5-turbo-0301' with 'knowledge cutoff of March 1, 2023.' For CodeT5-base, it is a pre-trained model fine-tuned on AVATAR-TC with known pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'no problem overlaps across splits.' For ChatGPT, Section 5.1 notes the knowledge cutoff 'predates the public release of AVATAR-TC on GitHub' to minimize contamination risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly addresses this: ChatGPT's knowledge cutoff predates AVATAR-TC's public release, 'minimizing the risk of AVATAR-TC (Test) pairs being included in ChatGPT's training data.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost reported for CoTran or the baselines."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (four V100 GPUs) and Appendix A.2 mentions '~75% reduction in training time' from LoRA, but no total training time, GPU hours, or API spend for ChatGPT experiments are stated."
       }
     }
   },
   "claims": [
     {
       "claim": "CoTran + CF + SF (RL+SFT interleaved) achieves 53.89% FEqAcc for J2P and 48.68% for P2J, outperforming all similar-size baselines.",
       "evidence": "Table 2 shows these numbers. Nearest competitor PPOCoder gets 44.27% J2P FEqAcc; PLBART-base gets 38.26% P2J FEqAcc.",
       "supported": "strong"
     },
     {
       "claim": "Fine-grained compiler feedback is more effective than Boolean compiler feedback for RL-based fine-tuning.",
       "evidence": "Finding 2 and Figure 2 show that Boolean feedback (CompCoder) provides constant reward for many translations, while CoTran's CF varies continuously. CoTran + CF outperforms PPOCoder by +11.57% FEqAcc in P2J (Table 2).",
       "supported": "strong"
     },
     {
       "claim": "Interleaving RL and SFT improves performance compared to RL-only fine-tuning.",
       "evidence": "Finding 3: RL+SFT interleaved gives +12.94% J2P FEqAcc vs. +9.50% for RL-only, relative to CodeT5-base (Table 2).",
       "supported": "strong"
     },
     {
       "claim": "The keyword tokenizer (kw-Tok) improves code translation performance.",
       "evidence": "Finding 4: kw-Tok accounts for +3.57% J2P and +6.62% P2J FEqAcc improvement (CoTran baseline vs. CodeT5-base in Table 2).",
       "supported": "moderate"
     },
     {
       "claim": "CoTran outperforms ChatGPT on all metrics in P2J and all but FEqAcc in J2P.",
       "evidence": "Table 2 confirms this. However, ChatGPT is gpt-3.5-turbo, not GPT-4, so the comparison is with an older model.",
       "supported": "strong"
     },
     {
       "claim": "CoTran can be 'effortlessly adapted' for other language pairs.",
       "evidence": "Section 1 claims this but provides no evidence beyond noting the requirements (dataset, compilers, test generation tool). No experiments on other language pairs.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CoTran fine-tunes CodeT5-base using RL with fine-grained compiler feedback (CF) and symbolic execution feedback (SF), achieving state-of-the-art results on Java-Python code translation. On the AVATAR-TC benchmark of 57,000+ code pairs, CoTran + CF + SF with interleaved RL+SFT training achieves 53.89% functional equivalence accuracy for J2P and 48.68% for P2J, outperforming 14 competing tools including ChatGPT (gpt-3.5-turbo) on P2J. The ablation study demonstrates that each component (kw-Tok, CF, SF, interleaved training) contributes incrementally, and that non-RL approaches to incorporating feedback are ineffective.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats-to-validity, or scope-bounding discussion. For a paper tested on a single language pair from competitive programming, this is a significant omission."
     },
     {
       "flag": "No variance or significance testing",
       "detail": "All results appear to be single-run numbers with no error bars, standard deviations across seeds, or significance tests. Given the small margins between some methods (e.g., CoTran baseline 44.52% vs. PPOCoder 44.27% on J2P FEqAcc), it is impossible to know if differences are meaningful."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper claims the approach can be 'effortlessly adapted for other language pairs' with no supporting evidence. Results are limited to Java-Python competitive programming problems, but the title and framing suggest general code translation capability."
     },
     {
       "flag": "Commercial tool dependency undisclosed",
       "detail": "CoTran relies on Symflower, a commercial symbolic execution engine. No disclosure of any relationship with Symflower is provided, and no competing interests statement exists."
     }
   ],
   "cited_papers": [
     {
       "title": "Unified Pre-training for Program Understanding and Generation",
       "authors": ["W. Ahmad", "S. Chakraborty", "B. Ray", "K.-W. Chang"],
       "year": 2021,
       "relevance": "PLBART model used as a key baseline for LLM-based code translation."
     },
     {
       "title": "AVATAR: A Parallel Corpus for Java-Python Program Translation",
       "authors": ["W. U. Ahmad", "M. G. R. Tushar", "S. Chakraborty", "K.-W. Chang"],
       "year": 2023,
       "relevance": "Source dataset that AVATAR-TC builds upon for code translation benchmarking."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "relevance": "Pre-trained code model used as baseline for code translation evaluation."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Y. Wang", "W. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2021,
       "relevance": "Base model architecture used by CoTran; key baseline for code generation and translation."
     },
     {
       "title": "Execution-based Code Generation using Deep Reinforcement Learning",
       "authors": ["P. Shojaee", "A. Jain", "S. Tipirneni", "C. K. Reddy"],
       "year": 2023,
       "relevance": "PPOCoder: RL-based code translation framework, primary competing method."
     },
     {
       "title": "Unsupervised Translation of Programming Languages",
       "authors": ["B. Roziere", "M.-A. Lachaux", "L. Chanussot", "G. Lample"],
       "year": 2020,
       "relevance": "TransCoder: foundational unsupervised code translation approach used as baseline."
     },
     {
       "title": "TransCoder-ST: Leveraging Automated Unit Tests for Unsupervised Code Translation",
       "authors": ["B. Roziere", "J. Zhang", "F. Charton"],
       "year": 2022,
       "relevance": "Uses unit tests for code translation self-training; key related work on test-guided translation."
     },
     {
       "title": "CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning",
       "authors": ["H. Le", "Y. Wang", "A. D. Gotmare"],
       "year": 2022,
       "relevance": "RL-based code generation method using unit test feedback, directly related to CoTran's approach."
     },
     {
       "title": "RLTF: Reinforcement Learning from Unit Test Feedback",
       "authors": ["J. Liu", "Y. Zhu", "K. Xiao"],
       "year": 2023,
       "relevance": "RL framework using test feedback for code generation, closely related methodology."
     },
     {
       "title": "Compilable Neural Code Generation with Compiler Feedback",
       "authors": ["X. Wang", "Y. Wang", "Y. Wan"],
       "year": 2022,
       "relevance": "CompCoder: uses Boolean compiler feedback for RL-based code generation, direct predecessor to CoTran's fine-grained CF."
     },
     {
       "title": "CodeTransOcean: A Comprehensive Multilingual Benchmark for Code Translation",
       "authors": ["W. Yan", "Y. Tian", "Y. Li"],
       "year": 2023,
       "relevance": "Multilingual code translation benchmark; standardized evaluation protocol used for ChatGPT prompting."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis"],
       "year": 2022,
       "relevance": "Parameter-efficient fine-tuning technique used in CoTran's training pipeline."
     }
   ]
 }