ai-research-survey

scan-v4.json (28732B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Enhancing Code Generation for Low-Resource Languages: No Silver Bullet",
     "authors": [
       "Alessandro Giagnorio",
       "Alberto Martin-Lopez",
       "Gabriele Bavota"
     ],
     "year": 2025,
     "venue": "IEEE International Conference on Program Comprehension",
     "arxiv_id": "2501.19085",
     "doi": "10.1109/ICPC66645.2025.00058"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about fine-tuning helping small models, in-context learning being a safe bet, and large models degrading with fine-tuning are all supported by Tables I and III.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims ('fine-tuning [...] helps in substantially boosting performance', 'possibly due to the fact that even a small dataset is sufficient') but the study design is observational across models — confounds between model architecture, training data, and size are not controlled.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title says 'No Silver Bullet' and the paper consistently qualifies findings by model size and language. The threats to validity explicitly acknowledges findings 'may not generalize to other settings' (Section V).",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for performance gaps: language similarity to high-resource languages, programming paradigm differences, domain-specificity of R, repository size vs count (Section III-E).",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims match measurement granularity: the paper measures pass@1 on HumanEval and reports it as pass@1 on HumanEval, without inflating to broader 'code quality' or 'developer productivity' claims.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' provides a dedicated discussion of construct, internal, and external validity threats.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: pass@k metric bias and mitigation via n=50, no hyperparameter tuning performed, training limited to 3 epochs which may cap results, specific prompt choices may be suboptimal (Section V).",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "External validity explicitly states: 'We decided to focus our study on four low-resource languages...Our findings may not generalize to other settings' and notes the specific models and sizes tested (Section V).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII acknowledges 'the financial support of the Swiss National Science Foundation for the PARSED project (SNF Project No. 219294)' and CHOOSE sponsorship.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Software Institute, USI Università della Svizzera italiana, Switzerland. No conflict with evaluated tools.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Swiss National Science Foundation is an independent government funding agency with no stake in whether fine-tuning or in-context learning performs better.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'Low-resource language' is explicitly defined as niche languages characterized by scarcity of training data; 'high-resource' is defined by contrast; pass@k is explained with formula and repetition procedure.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames its contribution as a comparative empirical study of five techniques (three in-context learning, two fine-tuning) for boosting LLM code generation on low-resource languages across six models.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section II explicitly positions the work relative to Cassano et al., Athiwaratkun et al., Van Dam et al., and others, explaining what this study adds (comparative analysis, model size dimension) that prior work lacked.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "A replication package is provided via Zenodo (ref [27], https://doi.org/10.5281/zenodo.13128630).",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The study uses publicly available MultiPL-E benchmark [19] and MultiPL-T datasets [2]. The replication package is released via Zenodo.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Hardware is mentioned (NVIDIA A30/A40/A100 GPUs) but no requirements.txt, Dockerfile, or detailed software dependency listing is provided in the paper.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided in the paper. The replication package is referenced but no README or commands are described.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Only point estimates of pass@1 are reported in Tables I and III. No confidence intervals or error bars are shown.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "McNemar's test is used for pairwise comparisons of dichotomous results, with Benjamini-Hochberg correction for multiple comparisons (Section III-D).",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Odds Ratios are reported alongside significance tests (Table II and Section IV-D), providing effect size magnitude for all comparisons.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No power analysis or justification for the choice of n=50 repetitions beyond citing prior work that 'this rate appears to stabilize at n=20'.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only average pass@1 rates are reported. No standard deviations, interquartile ranges, or spread measures across the 50 repetitions are provided.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Each model's baseline (out-of-the-box) performance is compared against all techniques (Table III white rows).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": false,
           "justification": "DeepSeek Coder and Code Llama were reasonable at submission time, but GitHub Copilot is a black box with unknown model version. More critically, newer models (e.g., GPT-4, Claude) are not included despite being available.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The study systematically compares five techniques (three in-context learning variants, fine-tuning, pre-training+fine-tuning) across models and sizes, effectively serving as an ablation of technique components.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": false,
           "justification": "Only pass@1 is used as the evaluation metric. No additional metrics (e.g., pass@10, CodeBLEU, syntactic correctness rate) are reported.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "The authors manually analyzed a sample of generated programs to understand failure reasons (Section III-E): 'We analyzed a sample of the generated programs to understand the reasons behind the performance gap.'",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "MultiPL-E HumanEval is used as the test set, separate from the MultiPL-T fine-tuning data. The 157/161 programs used for evaluation are distinct from training data.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down per model, per model size, per language, and per technique in Tables I and III.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Specific failure examples are analyzed: R returning null instead of empty list, vector instead of list (Fig. 1), Julia push vs push! API errors (Section III-E).",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports that fine-tuning worsens performance for DeepSeek Coder 33B (ORs of 1.64 and 1.42 for degradation), translation rules sometimes hurts, and pre-training doesn't consistently help.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Model families and sizes are specified (DeepSeek Coder 1B/7B/33B, Code Llama 7B/13B) but no specific version snapshots or dates. Copilot version is completely unspecified.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Prompt templates are shown in Listings 1-3 with structural detail. Full prompts are stated to be in the replication package [27]. The few-shot examples and translation rules are described with enough detail.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Temperature (0.2), learning rates (2×10⁻⁵ for DeepSeek, 5×10⁻⁵ for Code Llama), optimizers (AdamW), schedulers, max sequence lengths (1024/2048/3072), mixed precision (bfloat16), and epochs (3) are all reported (Sections III-C, IV-B).",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. Models are prompted directly for code generation.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "The paper documents how fine-tuning datasets were constructed from Cassano et al.'s data, the matching process for pre-training pairs (name+docstring matching with quality filtering), and filtering from 161→157 common programs (Sections III-C, IV-A).",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Replication package released via Zenodo [27] (doi: 10.5281/zenodo.13128630) containing experimental results.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data sources are well-described: MultiPL-E benchmark with 157 common programs, MultiPL-T datasets (37,592 R functions, 40,489 Racket functions), and the matching process for pre-training pairs (Sections III-C, IV-A).",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data source is a standard public benchmark (MultiPL-E/HumanEval).",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline from 164 HumanEval → 161 MultiPL-E → 157 common programs is documented with reasons for each filtering step. Fine-tuning dataset construction and pre-training pair matching are documented with counts (Section IV-A.5).",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates are stated for any of the models used. DeepSeek Coder and Code Llama training data temporal boundaries are not discussed.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether HumanEval problems appeared in the training data of any evaluated model.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "HumanEval was published in 2021. All models were trained after 2021 and could have seen these problems. This contamination risk is not discussed.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference costs, API costs, or wall-clock times are reported despite running 50 repetitions × 157-161 problems × 6 models × multiple techniques.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware is mentioned (NVIDIA A30/A40/A100 GPUs) but no GPU hours, training time, or total compute budget is stated.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "Results are averaged over 50 repetitions (via temperature=0.2 sampling) but no seed sensitivity analysis or variance across seeds is reported.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": true,
           "justification": "Explicitly stated: 'We compute pass@1 with n = 50 repetitions' (Section III-C).",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "The paper explicitly states 'we did not perform hyperparameter tuning' (Section V) and used default configurations. No search budget reported.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "For fine-tuned models, 'we evaluate each epoch on the MultiPL-E benchmark and only report the best model's results' — selection on the evaluation benchmark, which is documented (Section IV-C). All epoch results in replication package.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": true,
           "justification": "Benjamini-Hochberg procedure is applied to adjust p-values for multiple comparisons (Sections III-D, IV-C).",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors devised the translation examples and translation rules prompts and compare them against existing techniques without acknowledging author-evaluation bias.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "Fine-tuning vs in-context learning have vastly different compute costs but performance is not compared at matched compute budgets.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "HumanEval is used without discussing whether it adequately represents real-world code generation tasks for low-resource languages. The paper notes R is used for data analysis but HumanEval tests general programming — this construct validity gap is not addressed.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is involved; models are directly prompted.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "HumanEval (2021) predates all models' training. No discussion of temporal leakage.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether evaluation setup leaks information (e.g., function signatures providing too much structure).",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether fine-tuning data (MultiPL-T) shares structural similarities with test data (MultiPL-E HumanEval), despite both being derived from similar Python function collections.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No concrete leakage detection or prevention methods are applied.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Modern LLMs achieve near-high-resource performance on Julia and Lua, with performance gaps often below 10pp from Python/Java",
       "evidence": "Table I: DeepSeek 33B scores 43.3% Julia vs 57.8% Java (14.5pp gap) and 53.8% Lua; Copilot achieves 61.4% Lua vs 61.7% Python",
       "supported": "strong"
     },
     {
       "claim": "R and Racket remain problematic low-resource languages with pass@1 gaps of 24-39pp below Python",
       "evidence": "Table I: average pass@1 for R is 23.1% vs Python 57.3%; for Racket 18.7% vs Python 57.3%; all differences statistically significant",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning is best for small models (~1B parameters) but hurts very large models (~33B parameters)",
       "evidence": "Table III: DeepSeek 1B improves from 7.0% to 18.4% on Racket with pre-training+fine-tuning; DeepSeek 33B drops from 30.2% to 25.3% on R after fine-tuning (OR=1.64)",
       "supported": "strong"
     },
     {
       "claim": "In-context learning with translation examples is a 'safe bet' — consistently improves performance across all model sizes and both languages",
       "evidence": "Table III: translation examples improves over baseline for all 5 models on R (excluding 1B) and in-context learning is the best family for 33B; OR range 1.28-2.27 for significant differences",
       "supported": "moderate"
     },
     {
       "claim": "Pre-training on code translation before fine-tuning provides no consistent additional benefit over fine-tuning alone",
       "evidence": "Fine-tuning-only outperforms pre-training+fine-tuning for 3/5 models on R and 4/5 on Racket; average pass@1 difference is 19.92% vs 19.62% across models on R",
       "supported": "moderate"
     },
     {
       "claim": "No single technique is best across all combinations of model size and language",
       "evidence": "Table III shows mixed results: for 7B models, in-context learning wins on R but fine-tuning wins on Racket; the paper title reflects this 'no silver bullet' conclusion",
       "supported": "strong"
     },
     {
       "claim": "GitHub Copilot substantially benefits from few-shot in-context learning on R (+8.4% pass@1, OR=1.94)",
       "evidence": "Table III: Copilot baseline 32.7% R, few-shot 41.1% R; stated as statistically significant with OR=1.94",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Modern LLMs have effectively closed the gap for some nominally 'low-resource' languages (Julia, Lua), achieving pass@1 rates within 10-15pp of Python/Java, suggesting data volume is not the only determinant of LLM performance. For R and Racket, performance gaps of 25-40pp below Python persist, motivating enhancement techniques. Among five evaluated strategies, no single approach dominates: fine-tuning with limited data helps small models (1B) substantially but degrades performance for the largest model (33B), while in-context learning with translation examples consistently improves all model sizes and is the recommended 'safe bet.' The pre-training-on-translation step before fine-tuning provides no reliable additional benefit.",
   "red_flags": [
     {
       "flag": "No variance reported",
       "detail": "Tables I and III report only average pass@1 across 50 repetitions with no standard deviations or confidence intervals, making it impossible to assess the reliability of small performance differences."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "GitHub Copilot version is explicitly 'unknown'; DeepSeek Coder and Code Llama are identified only by arXiv citation without checkpoint hashes or snapshot dates, undermining reproducibility."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HumanEval (2021) and MBPP were published before training of the evaluated models; the possibility that test problems were in training data is not discussed."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only pass@1 is reported; no complementary metrics (compilation rate, pass@5, partial correctness) are used, limiting insight into how techniques affect different aspects of code quality."
     }
   ],
   "cited_papers": [
     {
       "title": "Knowledge transfer from high-resource to low-resource programming languages for code LLMs (MultiPL-T)",
       "relevance": "Provides the fine-tuning datasets used in this study and the framework for low-resource language benchmarking"
     },
     {
       "title": "MultiPL-E: A scalable and polyglot approach to benchmarking neural code generation",
       "relevance": "Provides the primary evaluation benchmark (HumanEval translated to 18 languages) used throughout"
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "relevance": "One of the two open-source model families evaluated across four size variants (1B, 7B, 33B)"
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "relevance": "Second open-source model family evaluated (7B and 13B variants)"
     },
     {
       "title": "Multi-lingual evaluation of code generation models",
       "relevance": "Prior work on few-shot learning for low-resource languages that this study replicates and extends"
     },
     {
       "title": "Measuring the impact of programming language distribution",
       "relevance": "Establishes that training data distribution across languages affects model performance disparities"
     },
     {
       "title": "On the transferability of pre-trained language models for low-resource programming languages",
       "relevance": "Prior work showing multilingual fine-tuning benefits for low-resource languages including Ruby"
     },
     {
       "title": "A survey on LLM-based code generation for low-resource and domain-specific programming languages",
       "relevance": "Contemporary survey that contextualizes the importance of this research direction"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly actionable guidance for developers using LLMs with niche languages: use in-context learning with translation examples as a safe default, fine-tune only for small models."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that Julia and Lua are no longer practically low-resource for modern LLMs challenges prior characterizations and the assumption that data volume is the primary driver."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns raised."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild contrarian angle in the 'no silver bullet' title, but no significant controversy with prior work."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Reproducible with public benchmarks and open-source models, but requires significant GPU compute for fine-tuning experiments."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "USI is a reputable but not top-brand research institution; GitHub Copilot evaluation adds recognition but is only one of six models."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }