ai-research-survey

scan.json (31799B)

---

 {
   "paper": {
     "title": "RepoTransBench: A Real-World Multilingual Benchmark for Repository-Level Code Translation",
     "authors": [
       "Yanli Wang",
       "Yanlin Wang",
       "Suiquan Wang",
       "Daya Guo",
       "Jiachi Chen",
       "John Grundy",
       "Xilin Liu",
       "Yuchi Ma",
       "Mingzhi Mao",
       "Hongyu Zhang",
       "Zibin Zheng"
     ],
     "year": 2024,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2412.17744",
     "doi": "10.1109/TSE.2025.3645056"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Repository-level code translation remains highly challenging, with the best method (RepoTransAgent with Claude or GPT-4.1) achieving only 32.8% success rate across 1,897 samples and 13 language pairs. Translation difficulty exhibits a strong directional asymmetry: static-to-dynamic translations achieve 45-63% success while dynamic-to-static translations typically score below 10%. Repository complexity across all dimensions (cross-file dependencies, code length, structural complexity) inversely correlates with translation success. Different LLMs show unexpected specialization patterns for specific language pairs, likely reflecting training data composition.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We provide the code and data at https://github.com/DeepSoftwareAnalytics/RepoTransBench' and reference [90] links to the same repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (1,897 repository samples with test suites) is released at the same GitHub repository mentioned in the abstract."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section V.D mentions using Docker as a sandboxed execution environment and bridging network for dependency installation, but no requirements.txt, Dockerfile, or detailed library version specifications are provided in the paper for reproducing the experimental setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides no step-by-step instructions for reproducing the experiments. A GitHub link is provided but the paper itself contains no README-style reproduction guide or commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables IV and V report only point estimates (e.g., 32.8% SR) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims various methods and models outperform others (e.g., 'RepoTransAgent consistently outperforms baseline approaches') based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance numbers with baseline context throughout, e.g., 'RepoTransAgent achieves 32.8% SR compared to 0.0% for TranslationOnly and 11.3% for ErrorFeedback' and 'improvements of up to 21.5% over the error feedback method,' providing sufficient context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 1,897 samples across 13 language pairs, but there is no justification for why this number was chosen, no power analysis, and no discussion of whether the per-pair sample sizes (some as small as 64 for Matlab) are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single experimental runs. No standard deviations, variance across runs, or spread measures are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares RepoTransAgent against two baselines: TranslationOnly (direct translation without feedback) and ErrorFeedback (translation with error feedback iteration), evaluated across all 8 LLMs (Table IV)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 8 backbone LLMs are all from 2025 (Table III) including Claude-Sonnet-4, GPT-4.1, DeepSeek-Chat, and Qwen3-235B, representing state-of-the-art models at the time of evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "RepoTransAgent has 5 core capabilities (ReadFile, CreateFile, ExecuteCommand, SearchContent, Finished) but no ablation study is conducted to show which components contribute most to performance. The progressive comparison (TranslationOnly → ErrorFeedback → RepoTransAgent) does not constitute a component-level ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are defined and used throughout: SR (Success Rate), CR (Compilation Rate), APR (Average Pass Rate), and AMPR (Average Module Pass Rate), as described in Section V.C."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated through test suite execution. No human evaluation of translation quality is conducted, though this could provide insights into partial successes and code readability."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any portion of the benchmark was used during agent development or prompt engineering. The paper does not describe a dev/test split or whether the reported results are on data that was used for any tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table V provides detailed breakdowns across all 13 translation pairs for each of the 8 LLMs. Figure 4 also breaks down performance by repository complexity dimensions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.D provides a detailed error analysis with 5 error categories (E1-E5): Configuration File Issues, Limited Understanding Ability Issues, Incomplete Generation Issues, Language Feature Issues, and Encoding Issues, each with concrete examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: TranslationOnly achieves 0% success across most models; DeepSeek-Reasoner performs 'dramatically worse' than DeepSeek-Chat (1.2% vs 22.5%); reasoning-focused models show 'no clear advantages'; dynamic-to-static translation scores below 10%."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims best-performing method achieves 32.8% (supported by Table IV), dynamic-to-static below 10% vs static-to-dynamic at 45-63% (supported by Table V), and that RepoTransAgent outperforms baselines (supported by Table IV showing improvements across all models)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RepoTransAgent improves translation via its reasoning-action framework, but the comparison is confounded by compute budget—RepoTransAgent likely uses more API calls and iterations than TranslationOnly or ErrorFeedback. No ablation isolates which agent components drive improvement, and compute is not controlled across methods."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded: the paper specifies 13 translation pairs, 7 languages, and 8 specific LLMs. Section VII explicitly notes results may not generalize to less common or domain-specific languages, and that language selection was based on TIOBE rankings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses several alternative explanations: LLM generation variability could affect reproducibility, test suites may not capture all edge cases, repository filtering may bias toward well-maintained projects, and the lack of fine-tuning may impact performance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures test passage rates as a proxy for functional correctness and explicitly acknowledges the gap: 'passing all test cases provides strong evidence of functional correctness, it may not capture all edge cases or guarantee complete semantic equivalence' (Section VII)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table III lists model names like 'Claude-Sonnet-4', 'GPT-4.1', 'Gemini-2.5-Flash-Lite', and 'o3-mini' with release dates and context windows, but no specific API versions or snapshot dates are provided. These are marketing names that can change behavior across API updates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 3 shows only a prompt template with placeholders: 'You are a code translator. Translate {source_language} to {target_language} for project {project_name}...' The actual fill values and complete prompts are not provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling/generation hyperparameters are reported for any of the 8 LLMs used. These settings significantly affect output quality."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes the RepoTransAgent architecture in detail: 5 core actions (ReadFile, CreateFile, ExecuteCommand, SearchContent, Finished), the ReAct-based reasoning-action workflow, and the iterative translation/validation cycle with concrete execution examples."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III documents the full pipeline: data collection from GitHub with star ranking, rule-based filtering (Algorithm 1 specifying language dominance, popularity threshold, and package exclusion criteria), and multi-agent test suite construction with four specialized agents."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threats to Validity' provides a dedicated section covering both internal and external threats with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses specific threats: the rapidly evolving LLM landscape, no fine-tuning applied to models, specific language pair selection based on TIOBE rankings, LLM generation variability affecting reproducibility, and that test suites may not capture all edge cases."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: results cover 13 translation pairs across 7 languages, 'may not generalize to less common languages or domain-specific languages,' and the filtering criteria favor well-maintained repositories, which 'is intentional to ensure benchmark quality' but may exclude experimental projects."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data including repository samples and test suites is released at https://github.com/DeepSoftwareAnalytics/RepoTransBench, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A describes collecting repositories from GitHub, ranking by star count, filtering by the top 7 TIOBE languages plus Rust and Matlab, and conducting a developer questionnaire to identify practical translation demands."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "21 professional developers filled a questionnaire to determine language pair demands, but the paper does not describe how these developers were recruited, from which organizations, or whether this introduces selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: GitHub collection → star-based ranking → rule-based filtering (Algorithm 1 with 3 specific rules) → multi-agent test suite construction (4 agents described in Section III.C). Table II reports final statistics: 970 projects yielding 1,897 samples."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states: 'This work is supported by CCF-Huawei Populus Grove Fund CCF-HuaweiSE202403.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sun Yat-sen University, Monash University, Huawei Cloud Computing Technologies Co., Ltd, and Chongqing University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The CCF-Huawei fund supports the research, but the paper evaluates general-purpose LLMs (from OpenAI, Anthropic, Google, DeepSeek, Alibaba) rather than any Huawei product specifically. Huawei does not have a direct financial stake in which model performs best on the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper despite Huawei co-authorship and Huawei funding."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table III provides model release dates but not training data cutoff dates. Since the benchmark uses existing GitHub repositories, the training cutoff is critical for assessing whether models may have seen benchmark code during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of existing public GitHub repositories (with >50 stars), which are highly likely to be included in LLM training corpora. No analysis of potential train/test overlap is conducted or discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses popular GitHub repositories that were almost certainly available online before the models' training cutoffs. No contamination analysis, canary strings, or temporal analysis is performed despite this being a critical validity concern."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper's evaluation is an automated benchmark evaluation with no human participants. The 21-person developer questionnaire was for benchmark design only."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main evaluation. The developer survey for language pair selection is a minor design input, not a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main evaluation."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main evaluation."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main evaluation."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main evaluation."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, wall-clock time, or per-example costs are reported despite evaluating 8 LLMs across 1,897 translation tasks using potentially expensive API calls."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or total API spend is reported. The experiments involve extensive LLM API usage across 8 models and 1,897 tasks but the cost is entirely unquantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many experimental runs were conducted per configuration. Section VII acknowledges 'inherent randomness in their outputs' as a threat but does not report multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The agent's configuration and any tuning decisions for prompts or parameters are not documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how the agent's configuration or prompts were selected. No validation set or selection process is described for the agent design choices."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 8 models, 3 methods, and 13 translation pairs without any multiple comparison correction or statistical testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own RepoTransAgent framework against baselines they implemented (TranslationOnly, ErrorFeedback) without acknowledging the inherent bias of evaluating one's own system or having independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RepoTransAgent uses an iterative multi-step reasoning process that almost certainly requires more API calls and compute than the simpler TranslationOnly and ErrorFeedback baselines. This compute difference is never quantified or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly discusses construct validity: 'passing all test cases provides strong evidence of functional correctness, it may not capture all edge cases or guarantee complete semantic equivalence.' They also argue execution-based metrics are superior to similarity-based metrics used in prior work."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "When comparing different backbone LLMs, the same RepoTransAgent framework is used consistently (Table V), controlling for the scaffolding variable. The paper also evaluates each model under all three methods (TranslationOnly, ErrorFeedback, RepoTransAgent)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses existing GitHub repositories that were publicly available before the models' training periods. No analysis of whether models may have seen the source repositories' code during training is conducted."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The agent can read source files and execute commands, but there is no analysis of whether this provides advantages beyond what a real translation scenario would offer."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the GitHub repositories in the benchmark (or similar code) appeared in the LLMs' training data. Popular repositories with >50 stars are especially likely to be in training corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap, decontamination) is applied despite using public GitHub repositories as the benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "The best-performing method achieves only 32.8% success rate on repository-level code translation.",
       "evidence": "Table IV shows RepoTransAgent with Claude and GPT-4.1 both achieving 32.8% SR, the highest across all method-model combinations.",
       "supported": "strong"
     },
     {
       "claim": "RepoTransAgent consistently outperforms baseline approaches across all evaluated backbone models.",
       "evidence": "Table IV shows RepoTransAgent outperforming TranslationOnly and ErrorFeedback for every LLM tested. For Claude: 32.8% vs 0.0% and 11.3%. For GPT-4.1: 32.8% vs 0.0% and 15.6%.",
       "supported": "moderate"
     },
     {
       "claim": "Translation from statically-typed to dynamically-typed languages achieves substantially higher success rates (45-63%) compared to the reverse direction (below 10%).",
       "evidence": "Table V shows C→Python at 61.5%, C++→Python at 63.0%, Java→Python at 45.9% for Claude, while Python→Java is 5.8%, Python→C++ is 3.5%, Python→Rust is 11.7%.",
       "supported": "strong"
     },
     {
       "claim": "Repository complexity inversely correlates with translation success across all dimensions.",
       "evidence": "Figure 4 shows successful translations consistently have lower median values for cross-file dependencies, intra-file dependencies, lines of code, functions, and classes compared to failed translations.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning-focused models do not show clear advantages over standard counterparts for repository-level translation.",
       "evidence": "Table IV shows Qwen3-think at 19.1% vs Qwen3 at 16.9% (modest improvement), and DeepSeek-Reasoner dramatically worse at 1.2% vs DeepSeek-Chat at 22.5%.",
       "supported": "moderate"
     },
     {
       "claim": "Different LLM backbones demonstrate specialized advantages for specific translation pairs.",
       "evidence": "Table V shows o3-mini achieving 67.2% on C→Rust, outperforming Claude (47.5%) and GPT-4.1 (47.5%) on the same pair, while Claude excels at translations to Python.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results across Tables IV and V are single-point estimates with no uncertainty quantification. LLM outputs are inherently stochastic, and the paper acknowledges this as a threat but does not mitigate it with multiple runs."
     },
     {
       "flag": "Critical contamination risk unaddressed",
       "detail": "The benchmark uses popular GitHub repositories (>50 stars) that are almost certainly in the training data of the LLMs being evaluated. No contamination analysis or decontamination is performed. Models may have seen the source code during training, inflating results for some language pairs."
     },
     {
       "flag": "Compute confound between methods",
       "detail": "RepoTransAgent uses an iterative multi-step reasoning process with multiple API calls, while TranslationOnly uses a single call. The improvement could partly be attributed to more compute/iterations rather than the agent architecture, but compute usage is never reported or controlled."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors evaluate their own RepoTransAgent against baselines they themselves implemented (TranslationOnly, ErrorFeedback). The baseline implementations may be suboptimal, and no independent evaluation or comparison against other agent frameworks (e.g., TransAgent [80]) on the same benchmark is provided."
     },
     {
       "flag": "Small developer survey for language pair justification",
       "detail": "Only 21 professional developers were surveyed to justify the choice of 13 translation pairs, with no description of how these developers were recruited. This thin justification is presented as evidence of 'real-world developer needs.'"
     }
   ],
   "cited_papers": [
     {
       "title": "Lost in translation: A study of bugs introduced by large language models while translating code",
       "authors": ["R. Pan", "A. R. Ibrahimzada", "R. Krishna"],
       "year": 2024,
       "relevance": "Directly related prior work showing LLMs struggle with repository-level code translation, achieving only 8.1% success rate with GPT-4."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Codex model, foundational for evaluating LLM code capabilities."
     },
     {
       "title": "Exploring and unleashing the power of large language models in automated code translation",
       "authors": ["Z. Yang", "F. Liu", "Z. Yu"],
       "year": 2024,
       "relevance": "UniTrans framework for LLM-based code translation at function level, a key baseline approach."
     },
     {
       "title": "Unsupervised translation of programming languages",
       "authors": ["M.-A. Lachaux", "B. Roziere", "L. Chanussot"],
       "year": 2020,
       "arxiv_id": "2006.03511",
       "relevance": "TransCoder: seminal work on unsupervised neural code translation with the TransCoder-test benchmark."
     },
     {
       "title": "TransAgent: An LLM-based multi-agent system for code translation",
       "authors": ["Z. Yuan", "W. Chen", "H. Wang"],
       "year": 2024,
       "arxiv_id": "2409.19894",
       "relevance": "Multi-agent system for code translation, directly comparable agent approach to RepoTransAgent."
     },
     {
       "title": "AlphaTrans: A neuro-symbolic compositional approach for repository-level code translation and validation",
       "authors": ["A. R. Ibrahimzada", "K. Ke", "M. Pawagi"],
       "year": 2025,
       "relevance": "Neuro-symbolic approach for repository-level translation that decomposes repos into fragments, a directly competing method."
     },
     {
       "title": "Rectifier: Code translation with corrector via LLMs",
       "authors": ["X. Yin", "C. Ni", "T. N. Nguyen"],
       "year": 2024,
       "arxiv_id": "2407.07472",
       "relevance": "LLM-based error correction approach for code translation, relevant to iterative refinement methods."
     },
     {
       "title": "CodeNet: A large-scale AI for code dataset for learning a diversity of coding tasks",
       "authors": ["R. Puri", "D. S. Kung", "G. Janssen"],
       "year": 2021,
       "arxiv_id": "2105.12655",
       "relevance": "Large-scale code dataset including file-level translation benchmarks, a predecessor benchmark."
     },
     {
       "title": "SpecTra: Enhancing the code translation ability of language models by generating multi-modal specifications",
       "authors": ["V. Nitin", "B. Ray"],
       "year": 2024,
       "arxiv_id": "2405.18574",
       "relevance": "Specification-enhanced LLM code translation approach, relevant technique for improving translation quality."
     },
     {
       "title": "TRACY: Benchmarking execution efficiency of LLM-based code translation",
       "authors": ["Z. Gong", "Z. Sun", "D. Huang"],
       "year": 2025,
       "arxiv_id": "2508.11468",
       "relevance": "Benchmark for evaluating execution efficiency of code translations, complementary benchmark perspective."
     },
     {
       "title": "CodeTransOcean: A comprehensive multilingual benchmark for code translation",
       "authors": ["W. Yan", "Y. Tian", "Y. Li"],
       "year": 2023,
       "arxiv_id": "2310.04951",
       "relevance": "Multilingual file-level code translation benchmark, a direct predecessor to RepoTransBench at coarser granularity."
     },
     {
       "title": "Vert: Verified equivalent Rust transpilation with few-shot learning",
       "authors": ["A. Z. Yang", "Y. Takashima", "B. Paulsen"],
       "year": 2024,
       "arxiv_id": "2404.18852",
       "relevance": "LLM-based code translation to Rust with formal verification guarantees, relevant to code translation safety."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Benchmark and agent framework are directly usable for evaluating and performing repository-level code translation, a real developer need."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms expected difficulty of repository-level translation; the static-to-dynamic asymmetry finding is interesting but not paradigm-shifting."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper is about code translation quality."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward benchmark evaluation."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data released on GitHub; practitioners could run the benchmark or agent framework on their own repositories."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Academic authors from Sun Yat-sen University and Monash; evaluates well-known LLMs (GPT-4.1, Claude) but the paper itself is not from a famous lab."
     }
   }
 }