ai-research-survey

scan.json (26125B)

---

 {
   "paper": {
     "title": "Collaboration is all you need: LLM Assisted Safe Code Translation",
     "authors": ["Rabimba Karanjai", "Sam Blackshear", "Lei Xu", "Weidong Shi"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2503.11237",
     "doi": "10.1145/3696630.3728521"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The framework is described architecturally but no implementation is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: CodeNet (Puri et al., 2021), Avatar (Ahmad et al., 2021), and XLCoST (Zhu et al., 2022). These are standard public datasets the authors did not modify. However, the NLI training dataset created from programming language documentation is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper lists model names used in the garden (Section 3) but provides no information about library versions, hardware, or software dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The paper describes the architecture conceptually but does not provide enough implementation detail to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., '61.6%', '79.38'). No confidence intervals, error bars, or uncertainty measures are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims UniTranslator 'rivals and surpasses' GPT-4 and outperforms SolMover based solely on comparing raw numbers. No statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw success rates and CodeBLEU scores are reported. No effect sizes (Cohen's d, relative improvement with context) are provided. The reader must manually compute differences from the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are stated (200 samples per language in Table 1, 734 tasks in Table 2, 249/250 in Avatar) but no justification is provided for why these sizes are sufficient for the claims being made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across any experimental runs. It is unclear whether results are from a single run or averaged across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against CodeGeeX, StarCoder, GPT-4, and Llama 2. Table 2 compares against SolMover. Table 3 compares against SteloCoder and XLCoST. Multiple baselines are included across experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are mixed. GPT-4 is contemporary, but the paper says just 'GPT-4' without specifying which version. CodeGeeX and StarCoder are from 2023. SteloCoder is from 2023. No comparison against more recent code translation systems from 2024-2025. The paper was submitted in March 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "UniTranslator has multiple components (Director LLM, concept agent, NLI checker, compiler feedback, agent quorum) but no ablation study is provided to show which components contribute to performance. The system is evaluated only as a whole."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 use success rate (functional correctness), while Table 3 uses CodeBLEU scores. Two distinct metrics are used across experiments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of translation quality is included. All evaluation is automated (compilation success rate and CodeBLEU). For a code translation system making claims about semantic soundness and idiomatic translations, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test/validation splits. The paper does not clarify whether the benchmarks used for evaluation overlap with any data used to fine-tune the NLI model or concept agent."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 3 provide per-language-pair breakdowns. Table 1 shows results for C++, Go, Java, and Python source languages. Table 3 shows results for five different translation directions to Python."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are shown or analyzed. The paper does not discuss examples where UniTranslator failed, what kinds of code it cannot translate, or qualitative error analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Table 3 shows UniTranslator underperforming SteloCoder on C# to Python, Java to Python, and PHP to Python, but these are not discussed as negative results or analyzed. Every aspect of the system is presented positively."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims UniTranslator 'achieves a level of accuracy and efficiency that rivals larger, monolithic models.' Table 3 shows UniTranslator scoring 54.31 on C# to Python vs SteloCoder's 74.83, and 53.22 on PHP to Python vs SteloCoder's 71.11. These substantial losses are not acknowledged. The abstract also claims the approach 'mitigates common issues such as code artifacts and hallucinations' but no evidence for this is presented."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims: 'the success of our concept agent underscores the transformative potential of incorporating explicit programming knowledge' (Section 4), and claims the feedback mechanism drives improvement. Without an ablation study isolating these components, these causal claims are not justified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Safe Code Translation' broadly but testing is limited to specific benchmarks and language pairs. The abstract claims the approach works for 'complex code translation tasks' generally. The paper acknowledges results are 'preliminary' but the title and abstract make broad claims. The paper also claims to address 'low-resource languages' but only tests Solidity-to-Move as an example."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider whether performance gains might come from specific model capabilities rather than the multi-agent framework, whether the benchmark selection favors the approach, or other confounding factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 lists model names: 'granite-code, codegemma, deepseek-coder-v2, starcoder2, codegeex4, codestral, deepseek-coder, codellama, codeqwen, qwen2.5-coder, gemma2, llama3.2, opencoder, llama3.3.' No version numbers, parameter counts, or snapshot dates are specified. The baselines in Table 1 are listed only as 'CodeGeeX', 'StarCoder', 'GPT-4', 'Llama 2' without versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts are provided. The paper describes concepts like 'few-shot prompting' and 'dynamic prompt generation system' (Section 1) but does not include any actual prompt text used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported. Temperature, top-p, max tokens, number of refinement iterations, convergence criteria, and other settings are not specified anywhere in the paper."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The multi-agent scaffolding is described at a high conceptual level (Section 2: Director LLM, concept agent, NLI checker, compiler feedback loop). However, critical implementation details are missing: how agents communicate, what the convergence criteria are, how many iterations are typical, how the Director selects agents in practice, and the actual workflow logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 mentions 'We sourced code from a diverse set of GitHub projects, prioritizing those with substantial community recognition (measured by repository stars)' but provides no details on selection criteria, filtering steps, or how the final evaluation set was constructed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations' section is present between Section 4 (Discussion) and Section 5 (Conclusion), acknowledging that smaller LLMs may not match larger ones, potential biases and hallucinations, and varying performance across languages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section contains only generic statements: 'performance of smaller LLMs may not always match that of larger models', 'Potential biases and hallucinations in LLMs can lead to errors', 'framework's performance may vary across different programming languages and domains.' None of these are specific to this study's design or results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No specific scope boundaries are stated. The paper does not explicitly say what it did NOT test, what populations/settings are excluded, or what claims it is NOT making. The Limitations section speaks generically about future work needed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual translation outputs, per-example pass/fail results, logs) is available. Only aggregated success rates and CodeBLEU scores in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states code was sourced from GitHub projects with 'substantial community recognition' but does not specify which projects, how many, what selection criteria were applied, or the time period of collection. The evaluation datasets are named but the process of using them is not described in detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper uses standard code translation benchmarks, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw code to evaluation results is not documented. It is unclear how code samples were selected from the benchmarks, how translations were executed (batch vs. sequential), or how success/failure was determined for each sample."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research was supported by the Sui Foundation Academic Grant (SARA)' and 'compute credits through the Google Developer Expert (GDE) and Google Cloud Research Innovator program.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Rabimba Karanjai and Weidong Shi at University of Houston, Sam Blackshear at Mysten Labs, Lei Xu at Kent State University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The research is funded by the Sui Foundation, and co-author Sam Blackshear is from Mysten Labs (the company behind the Sui blockchain and the Move language). The paper evaluates Solidity-to-Move translation (Table 2), which directly relates to the Sui ecosystem. The funder has a clear interest in demonstrating that Move is a viable translation target."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Given that a co-author works at Mysten Labs (the company behind the Sui blockchain) and the paper is funded by the Sui Foundation while evaluating Move language translation, this is a significant omission."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses multiple LLMs (granite-code, codegemma, deepseek-coder-v2, etc.) to perform code translation on public benchmarks (CodeNet, Avatar, XLCoST). No training cutoff dates are stated for any of the models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks used (CodeNet 2021, Avatar 2021, XLCoST 2022) predate many of the models used. No discussion of whether these benchmark solutions appeared in the training data of the LLMs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeNet (2021), Avatar (2021), and XLCoST (2022) were all published before the training cutoffs of the 2024-era models used. The paper does not discuss whether benchmark contamination could inflate results. This is a serious concern given that the models may have memorized benchmark solutions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper's stated motivation includes 'enabling deployment on commonly available hardware' and 'resource-constrained environments.' Despite this, no inference costs, latency, tokens consumed, or wall-clock time are reported for the multi-agent translation pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The Acknowledgments mention Google Cloud compute credits but do not quantify the total compute used for experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "UniTranslator, powered by compact open-source LLMs, achieves performance that rivals and sometimes surpasses GPT-4 in code translation.",
       "evidence": "Table 1 shows UniTranslator (labeled 'TransCode') outperforming GPT-4 in some cases: Java translation 80.6% vs 81.3% (close), but the column labeled 'TransCode' appears to be a different system from the reference [Puri et al., 2021]. UniTranslator's own results are not clearly separated from TransCode in Table 1.",
       "supported": "weak"
     },
     {
       "claim": "UniTranslator outperforms SolMover in Solidity-to-Move translation with 61.6% vs 47.5% success rate.",
       "evidence": "Table 2 shows UniTranslator achieving 61.6% success rate on 734 tasks vs SolMover's 47.5%. However, no statistical tests, confidence intervals, or details about what constitutes 'success' are provided.",
       "supported": "moderate"
     },
     {
       "claim": "UniTranslator shows competitive CodeBLEU performance on XLCoST benchmark, surpassing existing models in several translation-to-Python tasks.",
       "evidence": "Table 3 shows UniTranslator outperforming on C++ to Python (79.38 vs 75.42) and JavaScript to Python (80.07 vs 73.05), but substantially underperforming on C# to Python (54.31 vs 74.83), Java to Python (64.27 vs 74.39), and PHP to Python (53.22 vs 71.11). The claim of 'competitive performance' is contradicted by 3 out of 5 cases showing significant underperformance.",
       "supported": "weak"
     },
     {
       "claim": "The multi-agent collaborative approach with NLI grounding mitigates code artifacts and hallucinations.",
       "evidence": "No direct evidence is presented. The paper describes the NLI mechanism architecturally (Section 2.1) but provides no quantitative measurement of hallucination reduction or comparison with/without NLI grounding.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "UniTranslator proposes a multi-agent framework for code translation using compact LLMs orchestrated by a Director LLM, with NLI-based fact checking and compiler feedback loops. Preliminary evaluations on CodeNet, Avatar, and XLCoST benchmarks show mixed results: the system outperforms baselines in some language pairs (e.g., C++ to Python, JavaScript to Python, Solidity to Move) but substantially underperforms in others (C# to Python, PHP to Python). The paper is explicitly described as presenting 'preliminary' results.",
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Co-author Sam Blackshear is from Mysten Labs (the company behind the Sui blockchain and Move language). The paper is funded by the Sui Foundation. A key evaluation (Table 2) tests Solidity-to-Move translation, directly relevant to the Sui ecosystem. This conflict is not acknowledged anywhere in the paper."
     },
     {
       "flag": "Selective presentation of results",
       "detail": "Table 3 shows UniTranslator scoring 54.31, 64.27, and 53.22 on three translation tasks where SteloCoder achieves 74.83, 74.39, and 71.11 respectively — losses of 20+ CodeBLEU points. These substantial underperformances are not discussed. The abstract and conclusion focus exclusively on cases where UniTranslator performs well."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single point estimates with no error bars, confidence intervals, significance tests, or information about number of runs. It is impossible to assess whether observed differences are meaningful or due to random variation."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "Benchmarks from 2021-2022 (CodeNet, Avatar, XLCoST) are used with 2024-era LLMs. No analysis of whether the models have seen these benchmarks during training. Solutions to these problems may be in the training data."
     },
     {
       "flag": "Missing ablation study for multi-component system",
       "detail": "UniTranslator has at least 5 major components (Director LLM, concept agent, NLI checker, compiler feedback, agent quorum) but no ablation study shows which components contribute to performance. It is unknown whether the multi-agent approach adds value over a single LLM with compiler feedback."
     },
     {
       "flag": "Ambiguous Table 1 attribution",
       "detail": "Table 1 lists 'TransCode' with a citation to [Puri et al., 2021] (the CodeNet paper) as a column. It is unclear whether 'TransCode' refers to UniTranslator or to a baseline system. The paper states UniTranslator uses Gemma family models but does not clearly identify which column represents UniTranslator's results."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The abstract claims UniTranslator 'achieves a level of accuracy and efficiency that rivals larger, monolithic models' and 'mitigates common issues such as code artifacts and hallucinations.' The actual results show mixed performance and no evidence for hallucination mitigation."
     }
   ],
   "cited_papers": [
     {
       "title": "Lost in translation: A study of bugs introduced by large language models while translating code",
       "authors": ["Rangeet Pan", "Ali Reza Ibrahimzada", "Rahul Krishna"],
       "year": 2024,
       "relevance": "Empirical study of bugs introduced by LLMs during code translation, directly relevant to understanding LLM code generation quality."
     },
     {
       "title": "Attention, Compilation, and Solver-based Symbolic Analysis are All You Need",
       "authors": ["Prithwish Jana", "Piyush Jha", "Haoyang Ju"],
       "year": 2023,
       "arxiv_id": "2306.06755",
       "relevance": "Combines LLMs with compilation and symbolic analysis for code tasks, relevant to LLM-assisted programming methodology."
     },
     {
       "title": "Teaching Machines to Code: Smart Contract Translation with LLMs",
       "authors": ["Rabimba Karanjai", "Lei Xu", "Weidong Shi"],
       "year": 2024,
       "arxiv_id": "2403.09740",
       "relevance": "Prior work on LLM-based smart contract translation (SolMover), directly relevant to agentic code generation evaluation."
     },
     {
       "title": "Unsupervised translation of programming languages",
       "authors": ["Baptiste Roziere", "Marie-Anne Lachaux", "Lowik Chanussot", "Guillaume Lample"],
       "year": 2020,
       "relevance": "Foundational work on neural code translation published at NeurIPS, relevant to code generation methodology."
     },
     {
       "title": "XLCoST: A Benchmark Dataset for Cross-lingual Code Intelligence",
       "authors": ["Ming Zhu", "Aneesh Jain", "Karthik Suresh"],
       "year": 2022,
       "arxiv_id": "2206.08474",
       "relevance": "Benchmark dataset for cross-lingual code evaluation, relevant to understanding evaluation methodology in code translation."
     },
     {
       "title": "Understanding the Performance and Estimating the Cost of LLM Fine-Tuning",
       "authors": ["Yuchen Xia", "Jiho Kim", "Yuhan Chen"],
       "year": 2024,
       "arxiv_id": "2408.04693",
       "relevance": "Study of LLM fine-tuning cost and performance, relevant to practical cost assessment of LLM-based tools."
     },
     {
       "title": "SteloCoder: a Decoder-Only LLM for Multi-Language to Python Code Translation",
       "authors": ["Jialing Pan", "Adrien Sadé", "Jin Kim"],
       "year": 2023,
       "arxiv_id": "2310.15539",
       "relevance": "LLM for multi-language code translation used as a baseline, relevant to code generation evaluation methodology."
     },
     {
       "title": "Project CodeNet: A large-scale AI for code dataset for learning a diversity of coding tasks",
       "authors": ["Ruchir Puri", "David S Kung", "Geert Janssen"],
       "year": 2021,
       "arxiv_id": "2105.12655",
       "relevance": "Large-scale code dataset used as evaluation benchmark, relevant to understanding benchmarks for code AI systems."
     },
     {
       "title": "TRUE: Re-evaluating factual consistency evaluation",
       "authors": ["Or Honovich", "Roee Aharoni", "Jonathan Herzig"],
       "year": 2022,
       "arxiv_id": "2204.04991",
       "relevance": "NLI-based factual consistency evaluation method adapted for code verification in this paper, relevant to LLM output verification methodology."
     },
     {
       "title": "CARGO: AI-guided dependency analysis for migrating monolithic applications to microservices architecture",
       "authors": ["Vikram Nitin", "Shubhi Asthana", "Baishakhi Ray", "Rahul Krishna"],
       "year": 2022,
       "relevance": "AI-guided code migration tool, relevant to LLM-assisted software engineering and code transformation."
     }
   ]
 }