ai-research-survey

scan-v5.json (26149B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Enhancing Code Generation for Low-Resource Languages: No Silver Bullet",
     "authors": [
       "Alessandro Giagnorio",
       "Alberto Martin-Lopez",
       "Gabriele Bavota"
     ],
     "year": 2025,
     "venue": "IEEE International Conference on Program Comprehension",
     "arxiv_id": "2501.19085",
     "doi": "10.1109/ICPC66645.2025.00058"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims (fine-tuning best for small models, ICL scales with size, large models degrade with fine-tuning) are directly supported by Table III with statistical comparisons.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims like 'fine-tuning improves small models' and 'ICL boosts performance' are supported by controlled experiments comparing techniques against a baseline using the same benchmark, models, and languages; McNemar's tests with ORs quantify the differences.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly limits findings to R, Racket, 6 models, and 4 sizes in the threats section, stating 'Our findings may not generalize to other settings'; the title 'No Silver Bullet' itself signals bounded scope.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The authors discuss alternative reasons for the performance gap (language similarity, domain of use, programming paradigm) and speculate on why fine-tuning hurts large models (insufficient data to update weights) and why small models struggle with ICL (limited ability to interpret complex prompts).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses pass@1 on unit tests as a direct measure of functional code correctness, which matches what is claimed; no conflation with broader productivity or quality proxies.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' is dedicated to limitations, covering construct, internal, and external validity separately.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: no hyperparameter tuning (resource constraint), prompt sensitivity for ICL techniques, training limited to 3 epochs potentially capping fine-tuning gains, and restriction to 4 low-resource languages and 6 models.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The external validity section explicitly bounds results to 'four low-resource languages (Julia, Lua, R and Racket), one closed-source tool (GitHub Copilot), two open source models (DeepSeek Coder and Code Llama) and four model sizes.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments disclose Swiss National Science Foundation funding for the PARSED project (SNF Project No. 219294) and CHOOSE sponsorship for conference travel.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors list affiliation with Software Institute – USI Università della Svizzera italiana, Switzerland; none are affiliated with the evaluated tools (DeepSeek, Meta, GitHub).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Swiss National Science Foundation is a government research funder with no stake in the evaluated commercial or open-source tools.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper; absence of declaration = NO.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Low-resource languages are explicitly defined as 'niche programming languages characterized by the scarcity of training data'; pass@k is defined with its computation procedure; in-context learning and fine-tuning are described.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it contributes a comparative empirical study of five techniques across six LLMs for code generation on low-resource languages, filling a gap in previous work that studied techniques in isolation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section II engages substantively with Cassano et al., Athiwaratkun et al., Van Dam et al., and Orlanski et al., explaining why their approaches are reused, extended, or not applicable, not merely listing them.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "A replication package is released at https://doi.org/10.5281/zenodo.13128630 (reference [27]); Zenodo is a persistent archival repository, not a promise of future release.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The study uses MultiPL-E benchmark and MultiPL-T datasets, both publicly available from Cassano et al.; no proprietary datasets were created.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper specifies GPU hardware (A30/A40/A100) and bfloat16 precision but does not provide Python version, PyTorch/CUDA/Transformers library versions, or a Dockerfile/requirements.txt.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The training procedure is described in narrative form (Section IV.B), but the paper does not provide step-by-step reproduction instructions; the replication package may contain more but its contents are not enumerated.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Tables I and III report only mean pass@1 scores across 50 repetitions; no confidence intervals or standard deviations are reported around those averages.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "McNemar's test is used for all pairwise comparisons with Benjamini-Hochberg p-value correction for multiple comparisons; all comparisons are run with 157×50 = 7,850 observations.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Odds Ratios (OR) are reported throughout, e.g., 'the odds of generating a correct program in Java are about 5 times higher than in Julia' (OR=5.93), with full OR tables (Tables II and III).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": true,
           "justification": "The paper justifies n=50 repetitions by citing Cassano et al.'s finding that pass@1 'appears to stabilize at n=20', and uses k=1 with temperature 0.2 consistent with prior work.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only average pass@1 rates are reported in Tables I and III; variance or standard deviation across the 50 repetitions is not shown.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table III explicitly includes baseline rows for each model (model used out-of-the-box on the low-resource language) against which all five techniques are compared.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "DeepSeek Coder (2024), Code Llama (2023), and GitHub Copilot are all state-of-the-art at the time of publication; no suspiciously old or weak baselines.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Pre-training + fine-tuning vs. fine-tuning only serves as an ablation of the pre-training component; the paper explicitly compares these and finds pre-training adds no consistent benefit.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": false,
           "justification": "Only pass@1 is used as the evaluation metric; no alternative metrics such as pass@k (k>1), CodeBLEU, or manual code quality rating are reported.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Evaluation is fully automated via unit tests (pass@1); human evaluation of generated code quality is not applicable to this benchmark-based study.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Fine-tuning uses MultiPL-T datasets; evaluation is on MultiPL-E (HumanEval translations), a separate benchmark not used during training.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by model family, model size (1B/7B/13B/33B), technique (5 + baseline), and language (R vs. Racket) across Table III.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Figure 1 and Listing 3 show concrete failure cases with analysis: R generation fails on edge cases (null vs. empty list, vector vs. list return type), and failure reasons are categorized.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "DeepSeek Coder 33B performance degrades after fine-tuning (ORs of 1.64 for R and 1.42 for Racket against baseline); translation rules worsens baseline for 4/6 models in Racket.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Models are identified by marketing names (DeepSeek Coder 1B/7B/33B instruct, Code Llama 7B/13B instruct, GitHub Copilot) without specific snapshot dates or checkpoint identifiers.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Listing 1 and Listing 2 provide actual prompt templates for translation examples and translation rules; Listing 3 shows a fully instantiated prompt with real content.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Learning rates (2×10⁻⁵ for DeepSeek, 5×10⁻⁵ for Code Llama), optimizers (AdamW), schedulers, max sequence lengths (1024/2048/3072), temperature (0.2), batch size, precision (bfloat16), and epochs (3) are all reported.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; models are invoked directly via prompts without multi-step orchestration.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section IV.A.5 details the process for building the code translation pre-training dataset: matching Python functions to translations, docstring alignment, exclusion of ambiguous matches; fine-tuning dataset construction (combining D, S, B) is also described.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The Zenodo replication package (https://doi.org/10.5281/zenodo.13128630) is referenced and includes at minimum all statistical test results (162 tests); this is a persistent public archive.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The construction of fine-tuning and pre-training datasets from MultiPL-T is described in detail, including the matching procedure, exclusion criteria, and resulting dataset sizes (22,796 R pairs, 25,390 Racket pairs).",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; study uses standard public benchmarks with no recruitment.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from dataset construction (MultiPL-T → pre-training/fine-tuning sets) through training (Section IV.B) to evaluation (MultiPL-E, pass@1 with 50 reps) is described sequentially.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff is stated for any of the six models; for Code Llama the paper notes that 'the complete list of programming languages used for its training is not publicly available.'",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether HumanEval problems (the basis for MultiPL-E) may have appeared in model training data, despite these benchmarks predating all tested models.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "HumanEval was published in 2021 and all tested models were trained after this date; potential benchmark contamination is not addressed anywhere in the paper.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Inference cost is mentioned qualitatively ('extremely expensive' for 70B models) as a reason for exclusion, but no actual latency or cost figures are reported for the evaluated models.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "The HPC cluster is described (NVIDIA A30/A40/A100 GPUs) and training cost for 33B is called 'extremely high', but no GPU-hours, wall-clock time, or dollar cost is reported.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "There is a significant performance gap between high-resource (Python, Java) and low-resource (R, Racket) languages across all six tested LLMs.",
       "evidence": "Table I shows pass@1 for R ranging 7.0–32.9% and Racket 7.0–33.1% vs. Java 30.6–58.1% and Python 33.7–74.9%; Table II reports statistically significant ORs for all high- vs. low-resource comparisons.",
       "supported": "strong"
     },
     {
       "claim": "Julia and Lua behave more like high-resource languages than like R and Racket despite all four being classified as low-resource in prior work.",
       "evidence": "Pass@1 for Julia/Lua (19.2–61.4%) is far closer to Java/Python than to R/Racket (7.0–33.1%); ORs for Java vs. Julia/Lua (0.72–19.34) are substantially smaller than for Java vs. R/Racket (4.05–251.59).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning is the best technique for the smallest model (1B), substantially outperforming in-context learning approaches.",
       "evidence": "For DeepSeek Coder 1B, fine-tuning achieves 16.7%/18.1% and pre-training+FT 16.0%/18.4% on R/Racket vs. baseline 13.9%/7.0%; ORs vs. ICL techniques range 1.45–14.01.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning degrades performance of large models (33B) on low-resource languages, likely due to insufficient training data.",
       "evidence": "DeepSeek Coder 33B drops from baseline 30.2%/32.5% to 25.3%/28.0% (FT) and 25.8%/26.8% (pre+FT) on R/Racket; ORs of 1.64/1.42 vs. baseline are statistically significant.",
       "supported": "moderate"
     },
     {
       "claim": "In-context learning with translation examples is a safe bet that consistently improves performance across all model sizes (excluding 1B) and languages.",
       "evidence": "Translation examples improves over baseline for all 5 non-1B model configurations in both R and Racket (Table III), with ORs 1.28–2.27; few-shot worsens Code Llama 13B on Racket.",
       "supported": "moderate"
     },
     {
       "claim": "No single technique is universally best across all combinations of model size and language (no silver bullet).",
       "evidence": "Table III shows different techniques winning for different model-size-language combinations: FT wins for 1B, mixed for 7B/13B, ICL wins for 33B; no technique achieves best performance in all 10 cells.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "No single technique dominates for improving LLM code generation on low-resource languages (R, Racket). Fine-tuning works best for the smallest models (~1B) but actually degrades performance for the largest (33B), likely because the scarce training data cannot effectively update their parameters. In-context learning with translation examples is the most reliable technique — it always improves over baseline for models ≥7B and is cheap to apply. Julia and Lua, though classified as low-resource in prior work, now perform close to high-resource languages with modern LLMs, suggesting the amount of GitHub repositories is a poor proxy for LLM performance on a language.",
   "red_flags": [
     {
       "flag": "No variance reported",
       "detail": "Tables report only average pass@1 across 50 repetitions; no standard deviation or confidence intervals are shown, making it impossible to assess reliability of small differences (e.g., +0.9% for Code Llama 7B in Racket with translation examples)."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HumanEval (2021) and its MultiPL-E translations predate all tested models; potential contamination of test problems in training data is never discussed, which could inflate all reported pass@1 scores."
     },
     {
       "flag": "Model versions not pinned",
       "detail": "No snapshot dates or checkpoint hashes are given for DeepSeek Coder, Code Llama, or Copilot; as instruct models receive periodic updates, exact reproduction requires guessing which version was used."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only pass@1 is reported; no pass@k (k>1) or other metrics are used, limiting understanding of model behavior on harder problems."
     },
     {
       "flag": "Copilot asymmetry",
       "detail": "GitHub Copilot cannot be fine-tuned, so only 3 of 5 techniques are evaluated for it, making cross-technique comparisons structurally asymmetric for the most commercially important model."
     }
   ],
   "cited_papers": [
     {
       "title": "Knowledge transfer from high-resource to low-resource programming languages for code LLMs (MultiPL-T)",
       "relevance": "Primary source of fine-tuning datasets (37,592 R and 40,489 Racket functions) and prior work on fine-tuning for low-resource languages"
     },
     {
       "title": "MultiPL-E: A scalable and polyglot approach to benchmarking neural code generation",
       "relevance": "The benchmark used for all evaluations; translates HumanEval to 18 languages including R and Racket"
     },
     {
       "title": "Multi-lingual evaluation of code generation models",
       "relevance": "Prior work on few-shot learning for out-of-domain languages; few-shot technique replicated in this study"
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming — the rise of code intelligence",
       "relevance": "Primary open-source model family evaluated (1B, 7B, 33B)"
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "relevance": "Second open-source model family evaluated (7B, 13B)"
     },
     {
       "title": "Measuring the impact of programming language distribution",
       "relevance": "Prior work treating low-resource as a data distribution issue; their approach (balanced training across 14 languages) not applicable here due to no control over pretraining"
     },
     {
       "title": "On the transferability of pre-trained language models for low-resource programming languages",
       "relevance": "Prior work on fine-tuning for similar low-resource languages; motivates the multilingual pretraining approach"
     },
     {
       "title": "A survey on LLM-based code generation for low-resource and domain-specific programming languages",
       "relevance": "Recent survey highlighting scarcity of benchmarks for niche languages; contextualizes this study's contribution"
     },
     {
       "title": "Evaluating large language models trained on code (HumanEval)",
       "relevance": "Original benchmark whose problems form the basis of MultiPL-E used in all evaluations"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Gives actionable guidance to practitioners using R/Racket with LLMs: use ICL with translation examples for large models, fine-tune only for small ones."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that fine-tuning always helps — large models actually get worse, and Julia/Lua turn out not to be meaningfully low-resource anymore."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk concerns raised; purely a performance benchmarking study."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild tension with prior work that reported uniformly positive fine-tuning results; the degradation finding at 33B is notable but not framed controversially."
     },
     "demo_ability": {
       "score": 2,
       "justification": "All techniques (especially ICL with translation examples) can be tried immediately with any API-accessible LLM and the public MultiPL-E benchmark."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "USI is a respected Swiss research university but not a famous AI lab; evaluates GitHub Copilot which adds some brand recognition."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }