ai-research-survey

scan.json (28727B)

---

 {
   "paper": {
     "title": "Across Programming Language Silos: A Study on Cross-Lingual Retrieval-augmented Code Generation",
     "authors": [
       "Qiming Zhu",
       "Jialun Cao",
       "Xuanang Chen",
       "Yaojie Lu",
       "Hongyu Lin",
       "Xianpei Han",
       "Le Sun",
       "Shing-Chi Cheung"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.03535"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section X (Data Availability) provides a URL: https://anonymous.4open.science/r/Cross-Lingual-RACG-0F3C. The paper states 'We released the artifact and all experiment data' at this link. Note: this is an anonymous submission link, so long-term accessibility is uncertain."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The constructed 13,910-instance Multilingual Code Dataset Expansion is released at the same artifact URL (Section X). HumanEval-X is a publicly available dataset used for the parallel multilingual subset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions specific model names and greedy decoding settings (Section II.E), but provides no requirements.txt, Dockerfile, or library version list. There is insufficient detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references the artifact URL but provides no step-by-step reproduction instructions within the paper itself. No README commands or scripted replication pipeline are described in the text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only point estimates (Pass@K values) across all tables. No confidence intervals, error bars, or standard errors are provided for any main result. Standard deviations of per-language scores appear in some tables but represent cross-language spread, not uncertainty over runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims throughout (e.g., 'Java demonstrates superior cross-lingual utility over Python', 'multi-lingual LLMs achieve greater improvements compared to mono-lingual LLMs') but applies no statistical significance tests. All comparisons are based on direct number comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The schema states that 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper's tables consistently show both baseline and enhanced absolute Pass@K values alongside percentage improvements (e.g., Table II: baseline 55.28 → 95.85, '73% up'). This provides the magnitude of effect with full baseline context. While no formal effect size metric like Cohen's d is used, the schema explicitly allows 'percentage improvement with baseline context' as sufficient."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains approximately 1,000 instances per language (Section II.C), but no power analysis or explicit justification for this sample size is provided. The paper states it was an extension of existing resources, not a principled sample size determination."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Experiments use greedy decoding (temperature=0.0) producing deterministic single-run outputs (Section II.E). No variance across runs is reported because only one run is conducted. While averaging across 5 models is performed, no spread measures (std dev, IQR) are provided for the aggregated results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently compares against 'Baseline (without RAG/injection)' for all experiments across Tables II-VI, IX. Retrieval strategies are compared against each other (BM25, BGE-large-en-v1.5, CodeRankEmbed) in Table X."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include the no-RAG baseline (which represents current LLM capability), and CodeRankEmbed (2024 arXiv) is used as the state-of-the-art retriever. The models tested (Qwen2.5-Coder-7B, DeepSeek-Coder-6.7B) are from 2024, which is contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper conducts ablation-style experiments comparing Doc vs. Doc w/o NL settings (removing NL comments from corpus), comparing knowledge injection vs. full RACG pipeline, and comparing mono-lingual vs. multi-lingual LLMs. These systematically isolate components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pass@K for generation evaluation and Precision@K and Recall@K for retrieval evaluation (Section II.D, Table X). Multiple metrics are reported across different experimental aspects."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation using automated test case execution (Pass@K). Human evaluation of the generated code outputs is not applicable given the use of executable unit tests as ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The study uses a retrieval corpus of reference solutions and evaluates generation on separate test queries with verified unit tests. The dataset construction (Section II.C) describes that each instance has test cases for evaluation, distinct from the retrieval corpus documents."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "All result tables provide per-programming-language breakdowns (e.g., Tables III, V, VI show results for all 13 PLs separately). The paper explicitly analyzes language-specific performance variations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes including: PHP and Scala corpus degrading performance (Table V), Python mono-lingual LLMs suffering native language degradation with cross-lingual RACG, and Perl corpus causing widespread performance drops for mono-lingual LLMs (Table V). Section VIII also acknowledges limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple negative results: cross-lingual RACG can degrade performance for mono-lingual LLMs on their native language (e.g., -4.57% for Python in Table V), some corpora (PHP, Scala) cause net-negative effects, and Go corpus produces widespread degradation for mono-lingual LLMs (Table V)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The four key insights in the abstract (effectiveness, inequality, robustness, specialization) are all supported by empirical results in the paper. Tables II-X provide quantitative evidence for each claim. The abstract does not make claims beyond what the results show."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims primarily through controlled experiments: the knowledge injection setting (Section II.B.1) explicitly isolates the retrieval variable using an oracle retriever, and adversarial attacks (RQ3) directly manipulate corpus documents to measure causal effects on generation. These are adequate for the causal inferences drawn."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only models up to 7B parameters but draws broader conclusions about 'multi-lingual RACG systems' and 'reliable multi-lingual code assistants.' Section VIII acknowledges model size as a threat but the abstract and conclusion still make broad claims. The conclusions about Java vs. Python utility may not generalize to larger models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents one interpretation for most findings without systematically considering alternatives. For example, the claim that 'multi-lingual LLMs rely on internal knowledge to resist adversarial perturbations' is asserted without ruling out alternatives (e.g., distributional shift). Section VIII focuses on experimental limitations rather than alternative explanations for results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section II.E specifies models by exact names: CodeLlama-7B-Instruct, Deepseek-Coder-6.7B-Instruct, Qwen2.5-Coder-7B-Instruct, Phi-1, Phi-1.5, and CodeRankEmbed for retrieval. These are sufficiently specific model identifiers (matching HuggingFace/arxiv model names)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section II.E states 'we use a unified prompt template for LLMs, following the design in [19] and [43]' but no actual prompt text is provided in the paper or appendix. Readers cannot reconstruct the exact prompts sent to the models without reading the referenced papers."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section II.E states 'we implement greedy decoding with the temperature 0.0' and fixes the retrieval window size to 3 (top-K=3). A fixed random seed of 42 is also reported for perturbation experiments. Key inference hyperparameters are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RACG pipeline is described in Section II: a retriever (CodeRankEmbed) retrieves top-K documents from a corpus, which are injected into a prompt for the generation LLM. Figure 1 illustrates the pipeline construction and four experimental settings. The scaffold is sufficiently described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C describes the dataset construction process: using HumanEval-X (164 problems), extending from existing sources [43][44], generating reference solutions using Qwen2-72B-Instruct-GPTQ-Int4 with RACG, verifying through unit testing across 5 iterations. The pipeline and per-language counts are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII (Threats to Validity) is a dedicated section discussing methodological limitations, including model representativeness bias and deterministic perturbation outcomes from fixed random seeds."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII identifies two specific threats: (1) the models tested (none exceeding 7B parameters) may not represent broader LLM landscape, and (2) fixed random seed (42) could limit perturbation diversity. Both are specific to this study and include mitigation measures attempted."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section VIII acknowledges model size limitations, the paper does not explicitly state what the results do NOT show (e.g., it does not state that conclusions may not apply to larger models or commercial APIs). The conclusion broadly claims to 'establish foundational insights for designing more powerful and safer code intelligence' without bounding these claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The artifact URL (Section X) includes 'all experiment data' including the constructed 13,910-instance dataset. Raw dataset instances can be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C describes how the dataset was constructed: starting from HumanEval-X and extensions of existing datasets [43][44], generating reference solutions using Qwen2-72B-Instruct-GPTQ-Int4 via multi-lingual RACG, and verifying through unit testing over 5 iterations. Collection procedure is clear."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or crowd-sourced recruitment. The data source is existing benchmark datasets (HumanEval-X, McEval, MultiPL-E) plus LLM-generated solutions verified by unit tests."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C documents the full pipeline: starting datasets → LLM generation of missing reference solutions → unit test verification over 5 iterations → final 13,910 instances. Per-language counts are provided (e.g., Python: 1181, Kotlin: 1071, etc.)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure appears in the paper. There is no mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Chinese Information Processing Laboratory (Institute of Software, CAS), University of Chinese Academy of Sciences, and Hong Kong University of Science and Technology. No obvious conflict with any evaluated model vendor."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The schema says 'NA if unfunded.' Sonnet marked applies=false reasoning that no funding is disclosed making it unassessable. However, the authors are at major academic institutions (Chinese Academy of Sciences, HKUST) where research is typically funded. The absence of a funding disclosure does not mean the work is unfunded — it means we cannot verify funder independence. Since it is not clearly unfunded solo independent work, applies=true, answer=false is more appropriate."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears in the paper. The absence of disclosure means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses HumanEval-X (based on HumanEval, published 2021) as a benchmark and evaluates models like CodeLlama and Qwen2.5-Coder without stating their training data cutoff dates. The training cutoff is not mentioned for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval-X is used as a benchmark (164 problems for the parallel dataset, Section II.C). All evaluated models were likely trained on data post-2021, which may include HumanEval problems. No discussion of potential train/test overlap appears in the paper."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and the models evaluated (CodeLlama-2023, DeepSeek-Coder-2024, Qwen2.5-Coder-2024) have training cutoffs after HumanEval's release. The paper does not address whether HumanEval problems appear in model training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-based code generation systems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study with no human subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is not an experimental study with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Blinding is not applicable to automated benchmark evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report inference costs, API costs, tokens consumed, or wall-clock time for running the experiments. Given the scale (13,910 instances × multiple models × multiple settings), this is a significant omission."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget are stated. The paper used Qwen2-72B-Instruct-GPTQ-Int4 for dataset generation and multiple 7B models for evaluation, but no compute costs are quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Multi-lingual RACG significantly enhances multi-lingual code LLM generation, achieving Pass@K mean scores of 95.85% (+73%) for multi-lingual LLMs and 89.45% (+345%) for mono-lingual LLMs in same-language knowledge injection.",
       "evidence": "Table II shows knowledge injection results on Parallel Multilingual Code Dataset for both multi-lingual and mono-lingual LLMs across 5 programming languages.",
       "supported": "strong"
     },
     {
       "claim": "Java demonstrates superior cross-lingual utility over Python as a retrieval corpus in RACG, with Java→others average improvement of +16.08%/+13.35% compared to Python→others: +8.29%/+6.30%.",
       "evidence": "Tables IV, V, VI show cross-lingual RACG performance across all language pairs for both multi-lingual and mono-lingual LLMs. Section IV explicitly states the Java vs Python comparison.",
       "supported": "strong"
     },
     {
       "claim": "Adversarial attacks significantly degrade mono-lingual RACG (Pass@K dropping by up to 87% for Python and 88% for Java), but cross-lingual RACG demonstrates inherent robustness with mitigated impacts.",
       "evidence": "Tables VIII and IX show adversarial attack performance degradation for multi-lingual and mono-lingual LLMs across all perturbation types and language pairs.",
       "supported": "strong"
     },
     {
       "claim": "Domain-specific code retrievers (CodeRankEmbed) achieve approximately 91.60% Precision@5 and 88.04% Recall@20, substantially outperforming general text embeddings (BGE-large: ~56%, ~46%) and sparse retrieval (BM25: ~6.57%, ~4.79%).",
       "evidence": "Table X directly compares three retrieval strategies on the Multilingual Code Dataset Expansion in Doc w/o NL setting.",
       "supported": "strong"
     },
     {
       "claim": "Perturbed code documentation can paradoxically improve generation quality in cross-lingual scenarios, with 3,520 cases where syntax perturbations enabled correct code generation that correct documentation or internal knowledge could not.",
       "evidence": "Sections V and Figure 2 present Venn diagram analysis of 3,520 cases where perturbed documentation helped but correct documentation did not. The analysis is descriptive without statistical testing.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "This study constructs a 13,910-instance multilingual code dataset spanning 13 programming languages and evaluates cross-lingual retrieval-augmented code generation (RACG). Key findings are: (1) same-language corpus significantly enhances generation, with oracle injection reaching 95%+ Pass@1 for multi-lingual LLMs; (2) Java outperforms Python as a cross-lingual retrieval corpus despite Python's dominance in training data; (3) adversarial attacks severely degrade mono-lingual RACG but have mitigated effects in cross-lingual settings; and (4) domain-specific code retrievers vastly outperform general-purpose and sparse retrievers (91% vs. 56% vs. 6.57% precision).",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (e.g., Java outperforms Python, multi-lingual LLMs are more robust than mono-lingual LLMs) are made based solely on comparing point estimates without any statistical significance tests. Given the relatively small parallel dataset (164 HumanEval-X problems), the differences may not be statistically meaningful."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The paper uses HumanEval-X (derived from HumanEval, published 2021) as a core evaluation benchmark, but all tested models (CodeLlama-2023, DeepSeek-Coder-2024, Qwen2.5-Coder-2024) have training cutoffs after HumanEval's release. The paper does not discuss the risk that test problems appeared in training data, which could inflate performance estimates."
     },
     {
       "flag": "Single-run deterministic results, no variance",
       "detail": "All experiments use greedy decoding (temperature=0.0) producing single deterministic outputs. While this is reproducible, reporting no variance across any experimental condition makes it impossible to assess result stability or whether differences are within noise thresholds."
     },
     {
       "flag": "Dataset generated by LLM rather than human-verified",
       "detail": "The Multilingual Code Dataset Expansion (13,910 instances) uses Qwen2-72B-Instruct-GPTQ-Int4 to generate reference solutions, verified only by unit tests. LLM-generated reference solutions may introduce systematic biases, and unit test passage does not guarantee correctness for all edge cases."
     },
     {
       "flag": "Generalization beyond 7B models not bounded",
       "detail": "All generation models tested are 7B-parameter instruction-tuned models. The paper makes broad conclusions about 'multi-lingual RACG systems' and 'code assistants' without acknowledging that findings may not transfer to larger models (GPT-4, Claude, Gemini) or API-based commercial systems."
     },
     {
       "flag": "No funding disclosed",
       "detail": "The paper contains no acknowledgments section and no funding disclosure. This makes it impossible to assess potential conflicts of interest, though the academic affiliations suggest no obvious commercial conflict."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": [
         "M. Chen",
         "J. Tworek",
         "H. Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational HumanEval benchmark paper introducing Pass@K metric used throughout this study."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": [
         "B. Roziere",
         "J. Gehring",
         "F. Gloeckle"
       ],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the evaluated multi-lingual code LLMs (CodeLlama-7B-Instruct) used in the benchmark evaluation."
     },
     {
       "title": "CodeRAG-bench: Can retrieval augment code generation?",
       "authors": [
         "Z. Z. Wang",
         "A. Asai",
         "X. V. Yu"
       ],
       "year": 2025,
       "relevance": "Most closely related prior work on RACG benchmarking, limited to Python; this paper extends to 13 languages."
     },
     {
       "title": "Deepseek-coder: When the large language model meets programming – the rise of code intelligence",
       "authors": [
         "D. Guo",
         "Q. Zhu",
         "D. Yang"
       ],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Evaluated as one of the multi-lingual code LLMs (DeepSeek-Coder-6.7B-Instruct) in the benchmark study."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": [
         "B. Hui",
         "J. Yang",
         "Z. Cui"
       ],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Evaluated as one of the multi-lingual code LLMs (Qwen2.5-Coder-7B-Instruct) and used for dataset generation."
     },
     {
       "title": "Codegeex: A pre-trained model for code generation with multilingual benchmarking on humaneval-x",
       "authors": [
         "Q. Zheng",
         "X. Xia",
         "X. Zou"
       ],
       "year": 2023,
       "relevance": "Introduces HumanEval-X, the parallel multilingual benchmark used for the controlled knowledge injection experiments in this paper."
     },
     {
       "title": "Multipl-e: a scalable and polyglot approach to benchmarking neural code generation",
       "authors": [
         "F. Cassano",
         "J. Gouwar",
         "D. Nguyen"
       ],
       "year": 2023,
       "relevance": "Multi-lingual code generation benchmark relevant to evaluating cross-lingual code generation capabilities."
     },
     {
       "title": "Mceval: Massively multilingual code evaluation",
       "authors": [
         "L. Chai",
         "S. Liu",
         "J. Yang"
       ],
       "year": 2024,
       "arxiv_id": "2406.07436",
       "relevance": "One of the source datasets used to construct the Multilingual Code Dataset Expansion spanning 13 programming languages."
     },
     {
       "title": "PoisonedRAG: Knowledge corruption attacks to retrieval-augmented generation of large language models",
       "authors": [
         "W. Zou",
         "R. Geng",
         "B. Wang"
       ],
       "year": 2024,
       "arxiv_id": "2402.07867",
       "relevance": "Prior work on adversarial attacks against RAG systems that motivates the adversarial attack experiments in this paper."
     },
     {
       "title": "Retrieval augmented code generation and summarization",
       "authors": [
         "M. R. Parvez",
         "W. Ahmad",
         "S. Chakraborty"
       ],
       "year": 2021,
       "relevance": "Classic RACG paper (REDCODER) whose document construction methodology is followed in this study."
     },
     {
       "title": "An empirical study of retrieval-augmented code generation: Challenges and opportunities",
       "authors": [
         "Z. Yang",
         "S. Chen",
         "C. Gao"
       ],
       "year": 2025,
       "relevance": "Related empirical study of RACG relevant to the survey scope on LLM-based software engineering."
     },
     {
       "title": "How should we build a benchmark? Revisiting 274 code-related benchmarks for LLMs",
       "authors": [
         "J. Cao",
         "Y.-K. Chan",
         "Z. Ling"
       ],
       "year": 2025,
       "arxiv_id": "2501.10711",
       "relevance": "Systematic review of code benchmarks for LLMs, directly relevant to evaluating the quality of code generation benchmarks in the survey."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": [
         "C. E. Jimenez",
         "J. Yang",
         "A. Wettig"
       ],
       "year": 2024,
       "relevance": "Major benchmark for LLM code generation on real-world software engineering tasks, relevant context for the survey."
     }
   ]
 }