ai-research-survey

scan.json (25734B)

---

 {
   "paper": {
     "title": "A Survey on LLM-based Code Generation for Low-Resource and Domain-Specific Programming Languages",
     "authors": ["Sathvik Joel", "Jie JW Wu", "Fatemeh Fard"],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology (TOSEM)",
     "arxiv_id": "2410.03981",
     "doi": "10.48550/arXiv.2410.03981"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository for organizing surveyed papers: 'A GitHub repository was created to organize the papers of this survey at https://github.com/jie-jw-wu/Survey-CodeLLM4LowResource-DSL.' This is a curated list rather than analysis code, but it is a released artifact."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "While the GitHub repository organizes the paper list, the extracted survey data (categorizations, technique classifications, detailed extraction spreadsheets) is only available in the paper's tables. No downloadable structured dataset of the survey's extracted information is mentioned."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a survey, there is no computational environment to specify, but any analysis scripts or data processing tools used are not described."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the search strategy and screening process are described, there are no instructions for replicating the full extraction and categorization process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not run experiments or report quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no statistical comparisons of its own. All reported statistics are counts and proportions of categorized papers."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not conduct experiments requiring effect size reporting."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — the paper corpus size is determined by the search strategy, not by statistical power requirements."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental runs to report variance across."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 'Related Surveys' compares against multiple prior surveys [71, 89, 179, 192, 198, 200, 203, 207] and explicitly discusses how their work differs from and extends these prior reviews."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared surveys are recent (2023-2024), including Hou et al. [89], Zhang et al. [200], and Zheng et al. [207], representing the current state of the survey landscape."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate a system with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not applicable to a systematic literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no train/test split applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides extensive breakdowns: by language type (LRPL vs DSL, 51 vs 59 papers), by technique category (Figure 3a, Table 6 with 6 main categories), by venue (Figure 2 waffle chart), by year, and by model family (Figure 3b)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Challenges and Opportunities' extensively discusses limitations and failure modes of current approaches, including evaluation challenges, data scarcity issues, and methodological limitations specific to LRPLs and DSLs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The survey reports negative findings: LLMs perform poorly on LRPLs (Figure 1), lack of standardized benchmarks, DSL-specific challenges that remain unsolved, and that iterative feedback often requires extensive human guidance even with advanced models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 111 papers filtered from 27,000+ (Table 2), four evaluation techniques identified (Section 4), six method categories (Table 6), and dataset curation approaches classified (Section 6). All claims match the paper content."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a descriptive survey that categorizes and summarizes existing work. It does not make its own causal claims — when it discusses what 'improves' performance, it is reporting claims from the surveyed papers."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The survey explicitly bounds its scope: LLMs above 1 billion parameters (Section 3.7), code generation specifically (not code analysis), LRPLs and DSLs only, papers from 2020-2024, SQL explicitly excluded (Section 3.4). Section 8 discusses limitations of database selection."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure survey/taxonomy that presents no empirical results requiring alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no measurements of its own. It reviews other papers' measurements but makes no proxy-to-outcome claims itself."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use any LLM models in its own methodology."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full screening pipeline: four-iteration filtering approach (title screening → abstract screening → preliminary content review → final full-text review), with explicit inclusion/exclusion criteria (Section 3.4), search keywords (Table 1), and paper counts at each stage (Table 2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Threats to Validity' provides a dedicated discussion of the study's limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: database selection limitations, exclusion of SQL, inability to conduct backward snowballing for highly-cited papers (>300 citations), potential for missed recent papers in the expanded search, and the distinction between LRPL and DSL challenges that may affect interpretation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what it does not cover: SQL is excluded (Section 3.4), models under 1B parameters are generally excluded (Section 3.7), code analysis and translation are out of scope, and the time boundary (2020-2024, expanded to July 2025) is clearly stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The GitHub repository organizes the paper list, but the raw extraction data (categorization decisions, detailed coding sheets, inter-rater disagreement records) is not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 provides detailed search strategy: four databases (ArXiv, IEEE Xplore, Web of Science, ACM Digital Library), date range (Jan 2020 - May 2024), complete keyword groups (Table 1), and the Boolean logic connecting them."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how papers were 'recruited' for the review: systematic search across four databases, manual initial search on arXiv, forward and backward snowballing (Section 3.5), with explicit inclusion/exclusion criteria (Section 3.4)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 documents the full pipeline with counts: 27,333 initial → 506 (Iter 1) → 204 (Iter 2) → 192 (Iter 3) → 189 (Iter 4) → 75 (combined/deduped) + 36 (snowballing) = 111 final papers. Each iteration's purpose is described in Section 3.5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This research is supported by a grant from the Natural Sciences and Engineering Research Council of Canada RGPIN-2019-05175.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Indian Institute of Technology Madras and University of British Columbia. Neither institution has a product being evaluated in the survey."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC (Natural Sciences and Engineering Research Council of Canada) is a government funding agency with no financial interest in the survey's conclusions about LLM code generation for LRPLs/DSLs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate a pre-trained model on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate a pre-trained model on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate a pre-trained model on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The survey follows a structured SLR protocol adapted from Rodriguez et al. [148], with a systematic search strategy using reproducible keyword queries (Table 1), explicit database selection, four-iteration screening process (Section 3.5), defined inclusion/exclusion criteria (Section 3.4), and forward/backward snowballing. It also uses the Goal Question Metric (GQM) approach [7] for defining research objectives."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its 111 included papers. It categorizes papers by topic, technique, and language but treats all papers equally regardless of their methodological rigor. No quality scoring rubric or risk-of-bias assessment is applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not discuss publication bias. There is no consideration of whether the surveyed literature skews toward positive results, no funnel plots, and no acknowledgment that papers reporting successful LLM applications may be overrepresented. Section 8 discusses database selection threats but not publication bias."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs show significantly lower performance on low-resource programming languages compared to high-resource languages on MultiPL-E benchmark.",
       "evidence": "Figure 1 shows a heatmap comparing model performance across HRPLs and LRPLs, demonstrating consistently higher scores for Python and Java compared to LRPLs like R, Julia, and D across 13 models.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning is the most prevalent technique for enhancing LLM performance in LRPL/DSL code generation, used in 48 papers.",
       "evidence": "Figure 3(a) and Table 6 show fine-tuning in 48 papers, followed by prompting strategies (25) and pre-training (22).",
       "supported": "strong"
     },
     {
       "claim": "The LLaMA family is the most popular base model for fine-tuning in LRPL/DSL code generation.",
       "evidence": "Figure 3(b) shows LLaMA used in 14 instances, followed by DeepSeek (10) and StarCoder (9). The paper attributes this to earlier release date, open-source availability, range of model sizes, and LoRA compatibility.",
       "supported": "strong"
     },
     {
       "claim": "Smaller fine-tuned models can match or outperform GPT-3.5/GPT-4 on specialized LRPL/DSL tasks.",
       "evidence": "Table 8 shows 12 examples across Lean, Verilog, PowerShell, CQL, and other languages where fine-tuned models (e.g., DeepSeek-FT, CodeV, BetterV) match or surpass GPT-3.5/GPT-4.",
       "supported": "moderate"
     },
     {
       "claim": "There is a lack of standardized benchmark datasets and evaluation metrics for most LRPLs and DSLs.",
       "evidence": "Section 7 discusses this gap extensively. Many DSLs lack standardized benchmarks (Section 4.3), and researchers frequently create their own evaluation datasets. Only Verilog has standard benchmarks (VerilogEval, RTLLM).",
       "supported": "strong"
     },
     {
       "claim": "Research output in LRPL/DSL code generation has increased dramatically, from 1 paper in 2020 to 49 papers in just the first half of 2024.",
       "evidence": "Section 3.6 states: '1 and 4 papers in 2020 and 2021, respectively, followed by a notable increase to 13 publications in 2022, 44 papers in 2023, and reaching a zenith of 49 publications just in the first half of 2024.'",
       "supported": "strong"
     },
     {
       "claim": "Approximately 60% of surveyed papers use proprietary models in their research.",
       "evidence": "Figure 3(c) shows 59.5% use proprietary models versus 40.5% using only open-source models.",
       "supported": "strong"
     },
     {
       "claim": "Iterative feedback approaches significantly improve code quality and correctness for DSLs with available validators.",
       "evidence": "Section 5.2 and Table 10 summarize 11 iterative feedback approaches across Verilog, XDL, PDDL, PLC, and Verus, reporting improvements in compilation rates and functional correctness. E.g., Yao et al. improved automated proofs from 3/20 to 14/20 with iterative feedback.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This systematic literature review of 111 papers (from 27,000+ screened) identifies fine-tuning (48 papers) and prompting (25 papers) as the dominant strategies for improving LLM code generation in low-resource and domain-specific programming languages, with LLaMA being the most popular base model. The survey reveals that while smaller fine-tuned models can match or exceed GPT-3.5/GPT-4 on specialized tasks, there remains a critical lack of standardized benchmarks and evaluation metrics for most LRPLs and DSLs. DSLs face additional challenges requiring domain-specific evaluation infrastructure, specialized validators, and expert judgment that general-purpose metrics cannot capture. Research output in this area has grown exponentially from 1 paper in 2020 to 49 in the first half of 2024.",
   "red_flags": [
     {
       "flag": "No quality assessment of source papers",
       "detail": "The survey treats all 111 papers equally regardless of their methodological quality. There is no quality scoring rubric, risk-of-bias assessment, or structured evaluation of included studies. This means findings from rigorous peer-reviewed work carry the same weight as weaker arXiv preprints."
     },
     {
       "flag": "High proportion of non-peer-reviewed sources",
       "detail": "46 out of 111 papers (41%) are arXiv preprints (Figure 2). While the paper justifies including arXiv due to the rapid pace of the field, this high proportion of unreviewed work could affect the reliability of the survey's conclusions. Six technical reports were explicitly included despite exclusion criteria."
     },
     {
       "flag": "No discussion of publication bias",
       "detail": "The survey does not consider whether published papers skew toward positive results about LLM capabilities. Papers showing LLMs failing at LRPL/DSL tasks may be less likely to be published, potentially biasing the landscape toward optimistic findings about improvement techniques."
     },
     {
       "flag": "Selective reporting of improvement claims",
       "detail": "Tables 7 and 8 showcase successful fine-tuning improvements and cases where fine-tuned models beat GPT-3.5/GPT-4, but the survey does not systematically report or quantify how often fine-tuning fails or provides marginal improvement. The presentation selectively highlights success stories."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen et al."],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational for evaluating LLM code generation capabilities."
     },
     {
       "title": "MultiPL-E: A Scalable and Polyglot Approach to Benchmarking Neural Code Generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "relevance": "Key multilingual benchmark for evaluating LLM code generation across low-resource programming languages including Bash, Lua, R, Rust, and others."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li et al."],
       "year": 2023,
       "relevance": "Major open-source code LLM trained on 86 programming languages, widely used as baseline and fine-tuning base for LRPL/DSL code generation."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière et al."],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Most popular base model family for fine-tuning in LRPL/DSL code generation according to this survey."
     },
     {
       "title": "Knowledge transfer from high-resource to low-resource programming languages for code llms",
       "authors": ["Federico Cassano", "John Gouwar", "Francesca Lucchetti"],
       "year": 2024,
       "relevance": "Directly addresses the core topic of transferring LLM capabilities to low-resource languages using synthetic data generation."
     },
     {
       "title": "VerilogEval: Evaluating large language models for Verilog code generation",
       "authors": ["Mingjie Liu", "Nathaniel Pinckney", "Brucek Khailany", "Haoxing Ren"],
       "year": 2023,
       "relevance": "Standard benchmark for evaluating LLM code generation in Verilog, a key DSL in hardware design."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM software engineering capabilities on real-world code tasks."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key study on developer productivity with AI coding assistants, relevant to understanding the motivation for LRPL/DSL support."
     },
     {
       "title": "Grammar prompting for domain-specific language generation with large language models",
       "authors": ["Bailin Wang", "Zi Wang", "Xuezhi Wang"],
       "year": 2023,
       "relevance": "Novel prompting technique for DSL generation that augments examples with specialized grammars, effective across diverse DSLs."
     },
     {
       "title": "DocPrompting: Generating Code by Retrieving the Docs",
       "authors": ["Shuyan Zhou", "Uri Alon", "Frank F. Xu", "Graham Neubig"],
       "year": 2023,
       "relevance": "RAG approach for code generation using documentation retrieval, applied to Bash and other languages."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou et al."],
       "year": 2024,
       "arxiv_id": "2308.10620",
       "relevance": "Major related survey on LLMs for software engineering that this paper positions against and extends."
     },
     {
       "title": "Magicoder: empowering code generation with OSS-INSTRUCT",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2024,
       "relevance": "Demonstrates synthetic data generation approach for improving code generation including in low-resource languages."
     },
     {
       "title": "IRCoder: Intermediate Representations Make Language Models Robust Multilingual Code Generators",
       "authors": ["Indraneil Paul", "Goran Glavaš", "Iryna Gurevych"],
       "year": 2024,
       "relevance": "Uses LLVM IR as cross-lingual bridge to improve code generation for low-resource languages like D, Ruby, and Swift."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J. Hu et al."],
       "year": 2022,
       "relevance": "Parameter-efficient fine-tuning method widely used for adapting LLMs to LRPL/DSL code generation with limited compute."
     }
   ]
 }