ai-research-survey

scan.json (24306B)

---

 {
   "paper": {
     "title": "Enhancing Code Generation for Low-Resource Languages: No Silver Bullet",
     "authors": ["Alessandro Giagnorio", "Alberto Martin-Lopez", "Gabriele Bavota"],
     "year": 2025,
     "venue": "IEEE International Conference on Program Comprehension",
     "arxiv_id": "2501.19085",
     "doi": "10.1109/ICPC66645.2025.00058"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Modern LLMs handle some 'low-resource' languages (Julia, Lua) nearly as well as high-resource ones, but R and Racket show major performance gaps (pass@1 of 7-33% vs 31-75% for Java/Python). Fine-tuning helps smaller models (1B) but hurts larger ones (33B), likely due to insufficient data for weight updates. In-context learning with translation examples is a 'safe bet' that consistently improves performance across all model sizes, though no single technique is universally best.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided via Zenodo (ref [27], https://doi.org/10.5281/zenodo.13128630)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses publicly available MultiPL-E benchmark [19] and MultiPL-T datasets [2]. The replication package is released via Zenodo."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (NVIDIA A30/A40/A100 GPUs) but no requirements.txt, Dockerfile, or detailed software dependency listing is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The replication package is referenced but no README or commands are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only point estimates of pass@1 are reported in Tables I and III. No confidence intervals or error bars are shown."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "McNemar's test is used for pairwise comparisons of dichotomous results, with Benjamini-Hochberg correction for multiple comparisons (Section III-D)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Odds Ratios are reported alongside significance tests (Table II and Section IV-D), providing effect size magnitude for all comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the choice of n=50 repetitions beyond citing prior work that 'this rate appears to stabilize at n=20'."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only average pass@1 rates are reported. No standard deviations, interquartile ranges, or spread measures across the 50 repetitions are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each model's baseline (out-of-the-box) performance is compared against all techniques (Table III white rows)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek Coder and Code Llama were reasonable at submission time, but GitHub Copilot is a black box with unknown model version. More critically, newer models (e.g., GPT-4, Claude) are not included despite being available."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically compares five techniques (three in-context learning variants, fine-tuning, pre-training+fine-tuning) across models and sizes, effectively serving as an ablation of technique components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only pass@1 is used as the evaluation metric. No additional metrics (e.g., pass@10, CodeBLEU, syntactic correctness rate) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors manually analyzed a sample of generated programs to understand failure reasons (Section III-E): 'We analyzed a sample of the generated programs to understand the reasons behind the performance gap.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "MultiPL-E HumanEval is used as the test set, separate from the MultiPL-T fine-tuning data. The 157/161 programs used for evaluation are distinct from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model, per model size, per language, and per technique in Tables I and III."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Specific failure examples are analyzed: R returning null instead of empty list, vector instead of list (Fig. 1), Julia push vs push! API errors (Section III-E)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that fine-tuning worsens performance for DeepSeek Coder 33B (ORs of 1.64 and 1.42 for degradation), translation rules sometimes hurts, and pre-training doesn't consistently help."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about fine-tuning helping small models, in-context learning being a safe bet, and large models degrading with fine-tuning are all supported by Tables I and III."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims ('fine-tuning [...] helps in substantially boosting performance', 'possibly due to the fact that even a small dataset is sufficient') but the study design is observational across models — confounds between model architecture, training data, and size are not controlled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title says 'No Silver Bullet' and the paper consistently qualifies findings by model size and language. The threats to validity explicitly acknowledges findings 'may not generalize to other settings' (Section V)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for performance gaps: language similarity to high-resource languages, programming paradigm differences, domain-specificity of R, repository size vs count (Section III-E)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims match measurement granularity: the paper measures pass@1 on HumanEval and reports it as pass@1 on HumanEval, without inflating to broader 'code quality' or 'developer productivity' claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Model families and sizes are specified (DeepSeek Coder 1B/7B/33B, Code Llama 7B/13B) but no specific version snapshots or dates. Copilot version is completely unspecified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates are shown in Listings 1-3 with structural detail. Full prompts are stated to be in the replication package [27]. The few-shot examples and translation rules are described with enough detail."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature (0.2), learning rates (2×10⁻⁵ for DeepSeek, 5×10⁻⁵ for Code Llama), optimizers (AdamW), schedulers, max sequence lengths (1024/2048/3072), mixed precision (bfloat16), and epochs (3) are all reported (Sections III-C, IV-B)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly for code generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents how fine-tuning datasets were constructed from Cassano et al.'s data, the matching process for pre-training pairs (name+docstring matching with quality filtering), and filtering from 161→157 common programs (Sections III-C, IV-A)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' provides a dedicated discussion of construct, internal, and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: pass@k metric bias and mitigation via n=50, no hyperparameter tuning performed, training limited to 3 epochs which may cap results, specific prompt choices may be suboptimal (Section V)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "External validity explicitly states: 'We decided to focus our study on four low-resource languages...Our findings may not generalize to other settings' and notes the specific models and sizes tested (Section V)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Replication package released via Zenodo [27] (doi: 10.5281/zenodo.13128630) containing experimental results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are well-described: MultiPL-E benchmark with 157 common programs, MultiPL-T datasets (37,592 R functions, 40,489 Racket functions), and the matching process for pre-training pairs (Sections III-C, IV-A)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a standard public benchmark (MultiPL-E/HumanEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from 164 HumanEval → 161 MultiPL-E → 157 common programs is documented with reasons for each filtering step. Fine-tuning dataset construction and pre-training pair matching are documented with counts (Section IV-A.5)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII acknowledges 'the financial support of the Swiss National Science Foundation for the PARSED project (SNF Project No. 219294)' and CHOOSE sponsorship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Software Institute, USI Università della Svizzera italiana, Switzerland. No conflict with evaluated tools."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Swiss National Science Foundation is an independent government funding agency with no stake in whether fine-tuning or in-context learning performs better."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used. DeepSeek Coder and Code Llama training data temporal boundaries are not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval problems appeared in the training data of any evaluated model."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021. All models were trained after 2021 and could have seen these problems. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times are reported despite running 50 repetitions × 157-161 problems × 6 models × multiple techniques."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (NVIDIA A30/A40/A100 GPUs) but no GPU hours, training time, or total compute budget is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 50 repetitions (via temperature=0.2 sampling) but no seed sensitivity analysis or variance across seeds is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'We compute pass@1 with n = 50 repetitions' (Section III-C)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states 'we did not perform hyperparameter tuning' (Section V) and used default configurations. No search budget reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "For fine-tuned models, 'we evaluate each epoch on the MultiPL-E benchmark and only report the best model's results' — selection on the evaluation benchmark, which is documented (Section IV-C). All epoch results in replication package."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Benjamini-Hochberg procedure is applied to adjust p-values for multiple comparisons (Sections III-D, IV-C)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors devised the translation examples and translation rules prompts and compare them against existing techniques without acknowledging author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Fine-tuning vs in-context learning have vastly different compute costs but performance is not compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval is used without discussing whether it adequately represents real-world code generation tasks for low-resource languages. The paper notes R is used for data analysis but HumanEval tests general programming — this construct validity gap is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; models are directly prompted."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) predates all models' training. No discussion of temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup leaks information (e.g., function signatures providing too much structure)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether fine-tuning data (MultiPL-T) shares structural similarities with test data (MultiPL-E HumanEval), despite both being derived from similar Python function collections."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Modern LLMs show a major performance gap between high-resource (Java, Python) and some low-resource languages (R, Racket), with pass@1 gaps of 24-39 percentage points.",
       "evidence": "Table I shows pass@1 ranges of 7-33% for R/Racket vs 31-75% for Java/Python across all 6 models. All differences statistically significant with McNemar's test (Section III-E).",
       "supported": "strong"
     },
     {
       "claim": "Julia and Lua are not as 'low-resource' as previously characterized — performance is close to high-resource languages.",
       "evidence": "Table I shows Julia/Lua pass@1 of 19-61% vs Java/Python 31-75%. Average gap is 10.9%/20.8% for Julia and 3.6%/13.5% for Lua vs Java/Python (Section III-E).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning is best for small models (1B) but degrades performance for large models (33B).",
       "evidence": "Table III: DeepSeek 1B improves from 7.0% to 18.4% on Racket with fine-tuning. DeepSeek 33B degrades from 30.2% to 25.3% on R and 32.5% to 28.0% on Racket (ORs 1.64 and 1.42).",
       "supported": "strong"
     },
     {
       "claim": "In-context learning with translation examples is a 'safe bet' that always improves performance for models ≥7B.",
       "evidence": "Table III shows consistent improvement for all models ≥7B across both languages, with ORs between 1.28 and 2.27 (Section IV-D).",
       "supported": "strong"
     },
     {
       "claim": "The pre-training task (code translation) does not provide relevant additional benefit over fine-tuning alone.",
       "evidence": "Fine-tuning only works better than pre-training+fine-tuning for 3/5 models on R and 4/5 on Racket. Average pass@1 difference is 0.3% (Section IV-D).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "HumanEval was published in 2021 and all models were trained after that. The benchmark solutions could be in training data, inflating absolute numbers. While this affects all models equally for relative comparisons, it undermines claims about absolute performance levels."
     },
     {
       "flag": "Single benchmark",
       "detail": "All conclusions rest on HumanEval (157 problems). MBPP was available but excluded. Results may not generalize to other code generation tasks or benchmarks."
     },
     {
       "flag": "No variance reporting",
       "detail": "Despite 50 repetitions, no standard deviations or confidence intervals are reported. Statistical tests are performed but the reader cannot assess result stability."
     },
     {
       "flag": "Best-epoch selection on test set",
       "detail": "Fine-tuned models are evaluated each epoch and 'only report the best model's results' on the MultiPL-E benchmark — this is effectively selecting on the test set, which inflates fine-tuning results."
     }
   ],
   "cited_papers": [
     {
       "title": "Knowledge transfer from high-resource to low-resource programming languages for code LLMs",
       "authors": ["F. Cassano", "J. Gouwar", "F. Lucchetti"],
       "year": 2024,
       "relevance": "Introduces MultiPL-T framework for fine-tuning LLMs on low-resource languages via automated code translation."
     },
     {
       "title": "Multi-lingual evaluation of code generation models",
       "authors": ["B. Athiwaratkun", "S. K. Gouda"],
       "year": 2023,
       "relevance": "Studies few-shot learning for multi-lingual code generation including out-of-domain languages."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["D. Guo", "Q. Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the main model families evaluated; state-of-the-art code generation LLM."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["B. Roziere", "J. Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Second main model family evaluated; open-source code generation LLM."
     },
     {
       "title": "MultiPL-E: A scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["F. Cassano", "J. Gouwar"],
       "year": 2023,
       "relevance": "The benchmark used for evaluation — translates HumanEval/MBPP to 18 languages including low-resource ones."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational to code generation evaluation."
     },
     {
       "title": "Measuring the impact of programming language distribution",
       "authors": ["G. Orlanski", "K. Xiao"],
       "year": 2023,
       "relevance": "Shows balanced language distribution in training reduces performance disparities among languages."
     },
     {
       "title": "StarCoder: May the source be with you",
       "authors": ["R. Li", "L. B. Allal"],
       "year": 2023,
       "relevance": "Major open-source code LLM used in the MultiPL-T pipeline for generating training data."
     },
     {
       "title": "A survey on LLM-based code generation for low-resource and domain-specific programming languages",
       "authors": ["S. Joel", "J. J. Wu", "F. H. Fard"],
       "year": 2024,
       "arxiv_id": "2410.03981",
       "relevance": "Directly relevant survey highlighting scarcity of benchmarks for niche languages and need for advanced techniques."
     },
     {
       "title": "On the transferability of pre-trained language models for low-resource programming languages",
       "authors": ["F. Chen", "F. Fard", "D. Lo", "T. Bryksin"],
       "year": 2022,
       "relevance": "Studies fine-tuning on similar languages to boost low-resource performance, directly builds on this approach."
     }
   ]
 }