ai-research-survey

scan.json (21349B)

---

 {
   "paper": {
     "title": "A Survey on LLM-based Code Generation for Low-Resource and Domain-Specific Programming Languages",
     "authors": ["Sathvik Joel", "Jie JW Wu", "Fatemeh Fard"],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology (TOSEM)",
     "arxiv_id": "2410.03981"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "Systematic review of 111 papers on LLM-based code generation for low-resource and domain-specific programming languages. Found that fine-tuning and prompting are the most prevalent improvement methods, LLaMA family is the most popular base model for fine-tuning, and there is a significant lack of standardized benchmarks and evaluation metrics for most LRPLs and DSLs. The survey categorizes evaluation techniques into four types (automatic, user-centric, domain-specific, manual) and improvement methods into six main groups.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository to organize surveyed papers: https://github.com/jie-jw-wu/Survey-CodeLLM4LowResource-DSL, mentioned in the abstract and introduction."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository organizes the papers of this survey. The list of 111 filtered papers and their categorizations constitute the data of this SLR."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. This is a survey paper, but analysis scripts or tools used for screening could have been documented."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for replicating the systematic search and screening process beyond the methodology description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not run experiments or report quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental comparisons requiring significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted; this is a literature review."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experimental sample sizes to justify; the paper pool of 111 is determined by the search methodology."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs to report variance over."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly compares this survey against prior related surveys (Fan et al., Hou et al., Watson et al., Zan et al., etc.) and explains how this work differs from and extends them."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The related surveys compared against are from 2023-2024, which are contemporary given the 2024 publication date."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental evaluation requiring multiple metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; human evaluation of system outputs is not applicable."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experimental evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides extensive breakdowns: by language (LRPL vs DSL), by technique category (6 main groups in Table 6), by evaluation metric type (Tables 3-4), by venue (Figure 2), and by dataset type (Tables 11-14)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses challenges and limitations extensively, including evaluation challenges, data scarcity issues, and where current approaches fail for LRPLs/DSLs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The survey reports negative findings such as LLMs struggling with COQ syntax, low performance on LRPLs compared to HRPLs (Figure 1), and cases requiring 117 rounds of iteration to produce correct code."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims about filtering 111 papers from 27,000, identifying four evaluation techniques, six improvement method groups, and lack of standard benchmarks are all supported by the detailed analysis in Sections 4-7."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a systematic literature review that reports findings from surveyed papers rather than making causal claims of its own."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Threats to Validity) explicitly bounds generalization, noting database selection limitations, the exclusion of SQL, and the distinction between LRPLs and DSLs within the survey scope."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey/taxonomy paper presenting no empirical results of its own that would need alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no measurements of its own; no proxy-outcome gap to address."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use any LLM models in its own methodology."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting used in the survey methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted requiring hyperparameter reporting."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides detailed documentation of the paper selection pipeline: Table 1 shows keyword groups, Table 2 shows filtering counts at each iteration (27,330 → 506 → 204 → 192 → 189 → 75 + 36 snowballing = 111), and Section 3.4-3.5 describe eligibility criteria and screening process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Threats to Validity' provides a dedicated discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: database selection limitations, exclusion of SQL, the difference between LRPLs and DSLs within the survey, and incomplete backward snowballing for highly-cited papers (>300 citations). Also notes the extended 2025 search may have missed some papers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: LLMs defined as ≥1B parameters (Section 3.3), time period 2020-2024 (Section 3.3), SQL explicitly excluded (Section 3.4), and exclusion criteria listed in Section 3.4."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository (https://github.com/jie-jw-wu/Survey-CodeLLM4LowResource-DSL) organizes the surveyed papers, making the raw paper list verifiable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the search strategy in detail: four databases (arXiv, IEEE Xplore, Web of Science, ACM DL), keyword groups (Table 1), search period (Jan 2020 - May 2024), and snowballing approach."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is published papers identified through systematic search."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 documents the full pipeline from 27,330 initial papers through five iterations to the final 111, with counts at each stage separated by database. The screening process is described in Section 3.5 with four distinct iterations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This research is supported by a grant from the Natural Sciences and Engineering Research Council of Canada RGPIN-2019-05175.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: IIT Madras (Joel) and University of British Columbia (Wu, Fard). No commercial product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC is a Canadian government research council with no financial stake in the survey's findings about LLM code generation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The survey follows a structured systematic review protocol adapted from Rodriguez et al. [148], with a four-iteration screening process, explicit keyword groups (Table 1), defined databases, eligibility criteria (Section 3.4), and counts at each filtering stage (Table 2). While not explicitly PRISMA, it follows an established SLR methodology."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its 111 source papers. All papers that pass the inclusion criteria are treated equally regardless of their rigor. No quality scoring rubric or risk-of-bias assessment is applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not discuss publication bias. There is no consideration of whether published papers skew toward positive results for LLM code generation, no funnel plots, and no acknowledgment that negative results may be underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "111 relevant papers were filtered from over 27,000 published studies from 2020-2024 on LLM code generation for LRPLs and DSLs.",
       "evidence": "Table 2 shows the filtering pipeline: 27,330 initial papers across 4 databases, reduced through 5 iterations to 75 unique papers, plus 36 from snowballing, totaling 111. An additional 5 papers from 2024-2025 were added in an expanded search.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning is the most prevalent technique for improving LLM performance on LRPLs/DSLs, used in 48 papers.",
       "evidence": "Figure 3(a) shows the frequency distribution of techniques, with fine-tuning at 48, followed by prompting strategies (25) and pre-training (22).",
       "supported": "strong"
     },
     {
       "claim": "The LLaMA family is the most popular base model for fine-tuning in LRPL/DSL code generation.",
       "evidence": "Figure 3(b) shows LLaMA at 14 instances, followed by DeepSeek (10) and StarCoder (9).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuned smaller models can match or outperform GPT-3.5/GPT-4 on LRPL/DSL tasks.",
       "evidence": "Table 8 provides 12 examples where fine-tuned models outperform GPT-3.5 or GPT-4, including DeepSeek-FT solving 5/148 FIMO problems vs GPT-4's 0/148, and CodeV achieving 78.1% vs GPT-4's 60.0% on VerilogEval-Machine.",
       "supported": "strong"
     },
     {
       "claim": "There is a lack of standardized benchmarks and evaluation metrics for most LRPLs and DSLs.",
       "evidence": "Section 7.2 discusses the lack of language-specific benchmarks for LRPLs compared to HRPLs (which have APPS, CodeContests, SWE-bench, etc.), and notes that many DSL researchers must create custom evaluation datasets.",
       "supported": "moderate"
     },
     {
       "claim": "60.4% of surveyed papers use proprietary models in their research.",
       "evidence": "Figure 3(c) shows the distribution with 59.5% using proprietary models vs 40.5% using only open-source models.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of source papers",
       "detail": "The survey treats all 111 papers equally without assessing their methodological quality. This means findings from rigorous studies are given equal weight to findings from methodologically weak papers, potentially laundering low-quality results."
     },
     {
       "flag": "No publication bias discussion",
       "detail": "The survey does not consider whether published papers on LLM code generation for LRPLs/DSLs skew toward positive results. Negative findings (LLMs failing on certain languages) may be underrepresented."
     },
     {
       "flag": "Primary screening by single author",
       "detail": "The initial title screening of all 27,333 papers was done by the first author alone. While second and third authors reviewed categorizations and disagreements were discussed, the initial filtering represents a single-rater bottleneck."
     },
     {
       "flag": "Incomplete backward snowballing",
       "detail": "Section 3.5 notes that backward snowballing was skipped for papers with over 300 citations, potentially missing relevant work citing foundational papers."
     }
   ],
   "cited_papers": [
     {
       "title": "MultiPL-E: A Scalable and Polyglot Approach to Benchmarking Neural Code Generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "relevance": "Key multilingual code generation benchmark covering low-resource languages, widely used as evaluation standard."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li"],
       "year": 2023,
       "relevance": "Major open-source code LLM trained on 86 programming languages, frequently used as base model for fine-tuning."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Most popular base model family for fine-tuning on LRPL/DSL code generation tasks."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo"],
       "year": 2024,
       "relevance": "Second most popular base model for LRPL/DSL fine-tuning with strong coding benchmark performance."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational for code generation evaluation."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Major benchmark for real-world software engineering tasks, cited as example of sophisticated evaluation frameworks."
     },
     {
       "title": "Large Language Models for Software Engineering: Survey and Open Problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mark Harman"],
       "year": 2023,
       "arxiv_id": "2310.03533",
       "relevance": "Related survey on LLMs for SE that this paper positions against as not covering LRPLs/DSLs."
     },
     {
       "title": "A Survey on Large Language Models for Software Engineering",
       "authors": ["Quanjun Zhang"],
       "year": 2023,
       "arxiv_id": "2312.15223",
       "relevance": "Related survey on LLMs for SE; this paper fills the gap left by its lack of LRPL/DSL coverage."
     },
     {
       "title": "Knowledge transfer from high-resource to low-resource programming languages for code llms",
       "authors": ["Federico Cassano"],
       "year": 2024,
       "relevance": "Demonstrates knowledge transfer techniques (MultiPL-T) to improve LLM performance on LRPLs."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key study on AI coding tool productivity impact, relevant to understanding the practical stakes of LRPL/DSL support."
     },
     {
       "title": "IRCoder: Intermediate Representations Make Language Models Robust Multilingual Code Generators",
       "authors": ["Indraneil Paul", "Goran Glavaš"],
       "year": 2024,
       "relevance": "Novel cross-lingual transfer approach using LLVM IR for improving code generation across languages including LRPLs."
     },
     {
       "title": "VerilogEval: Evaluating large language models for Verilog code generation",
       "authors": ["Mingjie Liu", "Nathaniel Pinckney"],
       "year": 2023,
       "relevance": "Standard benchmark for Verilog DSL code generation, widely adopted in the hardware design LLM community."
     }
   ]
 }