ai-research-survey

scan.json (29669B)

---

 {
   "paper": {
     "title": "TigerCoder: A Novel Suite of LLMs for Code Generation in Bangla",
     "authors": [
       "Nishat Raihan",
       "Antonios Anastasopoulos",
       "Marcos Zampieri"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2509.09101",
     "doi": "10.48550/arXiv.2509.09101"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TigerCoder, the first dedicated Bangla code-generation LLM family (1B and 9B parameters), achieves 11-13% Pass@1 improvements over prior models on Python benchmarks, with larger gains on other programming languages. Machine-translating Bangla prompts to English does not improve code generation performance, contradicting a common assumption. Most existing LLMs exhibit large performance drops (20-60 percentage points) when coding prompts are in Bangla rather than English, with TigerLLM being the only prior model maintaining parity. The combined use of self-instruct, synthetic, and translated instruction datasets yields the best fine-tuning results.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository link provided in abstract footnote: https://github.com/mraihan-gmu/TigerCoder/. The paper states 'We open-source all resources to advance further Bangla LLM research.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims open-source release of all three Bangla-Code-Instruct datasets (SI, Syn, TE — 300K total) and the MBPP-Bangla benchmark (974 problems) via the same GitHub repository. Licensed under CC-BY-SA-4.0 per Appendix D."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (NVIDIA A100 40GB, 80GB RAM, Google Colab) and Python 3.13.0 is stated for code validation, but no requirements.txt, Dockerfile, or library version listing is provided. The fine-tuning framework is not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described. Appendix D mentions 'a Colab notebook offers a plug-and-play demo' but no reproduction scripts or README commands for replicating main experiments are provided in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3, 4, and 7 are point estimates with no confidence intervals, error bars, or uncertainty measures. Pass@K values are reported as single numbers."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'significant 11-18% performance gains' but no statistical significance tests (p-values, t-tests, bootstrap tests) are used. All comparative claims are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 includes explicit Δ columns showing improvement over baseline (e.g., TigerCoder 9B: +0.11 P@1 on mHumanEval, +0.13 on MBPP). Baseline values are available in Table 3 for context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes (100K per subset, 974 benchmark problems). The choice of 974 MBPP problems is inherited from the original benchmark but no power analysis or sample size reasoning is given for any evaluation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Table 6 states Seed=42, suggesting single-run results with no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines in Tables 3-4: proprietary models (GPT-3.5, GPT-4o-mini, Gemini Flash 2.5), multilingual open-source (LLaMA 3.2, Gemma 3, Pangea, Phi-4), and Bangla-specific models (Titu-LLM, Bong-LLaMA, Bangla-LLaMA, Bangla-Gemma, TigerLLM)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent models: Gemma 3 (2025), Gemini Flash 2.5 (2024), GPT-4o-mini (2024), LLaMA 3.2 (2024). These represent the current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides a thorough ablation study testing all 7 combinations of the three instruction datasets (SI, Syn, TE) for both 1B and 9B model sizes, showing individual and synergistic contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three variations of Pass@K are used (K=1, 10, 100) across two benchmarks (mHumanEval-Bangla, MBPP-Bangla) and five programming languages."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model-generated code outputs. All evaluation is automated via test case execution (Pass@K). Human involvement was limited to benchmark creation (translation and verification), not system output evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation benchmarks (MBPP-Bangla, mHumanEval-Bangla) are entirely separate from the training data (Bangla-Code-Instruct). The fine-tuning data consists of instruction-code pairs, while the test benchmarks are independently curated problem sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language in Appendix E (C++, Java, JavaScript, Ruby in addition to Python), by benchmark (mHumanEval vs MBPP), and by dataset combination (Figure 2). MBPP-Bangla also has five topical classes, though per-class results are not reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows specific code keyword mistranslation examples causing failures in the MT approach. Section 5 provides analysis of why machine translation produces poor results. However, no error analysis of where TigerCoder itself fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 5) is entirely a negative result: machine translation does not help Bangla code generation. The Syn dataset alone being less effective than SI or TE is also reported in Figure 2."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'significant 11-18% performance gains at Pass@1' but Table 7 shows the 9B model achieves 11% and 13% at Pass@1 (range: 11-13%, not 11-18%). The 18% gain only appears at Pass@100 (MBPP, 9B model). The 1B model's Pass@1 gains are only 5%. The abstract conflates Pass@1 and higher-K metrics."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that curated datasets improve performance. The ablation study (Figure 2) provides controlled single-variable manipulation of dataset combinations, and the fine-tuning comparison against the TigerLLM base model is a reasonable before/after controlled intervention."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion extends to 'other low-resource languages' ('an effective and replicable blueprint for the future of efficient, high-performance LLM development for Bangla and other low-resource languages') but results are only on Bangla. The title is appropriately scoped, but the conclusion overclaims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. The strong TigerLLM base model's contribution vs. the fine-tuning data is not disentangled — would these datasets help other base models equally? Potential overfitting to MBPP-style tasks is not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@K on code generation benchmarks and claims code generation capability. The measurements match the claims at appropriate granularity — they don't overframe Pass@K as 'developer productivity' or 'software quality.' The scope is clearly code generation on benchmarks."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT-4o-mini', 'GPT-3.5', 'Gemini-Flash 2.5', 'Claude-3.5-Sonnet'. No API version strings, snapshot dates, or specific model identifiers (e.g., 'gpt-4o-mini-2024-07-18') are provided for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided for model evaluation or data generation. Appendix B describes generation prompts in natural language ('Provide only the Python code', 'Ensure the code is self-contained') but does not reproduce the full prompt templates used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides comprehensive fine-tuning hyperparameters for both 1B and 9B models: learning rate, batch size, gradient accumulation, epochs, optimizer, scheduler, warmup, precision. However, inference/generation parameters (temperature, top-p, number of samples n for Pass@K) are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on code generation tasks via standard inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data curation is extensively documented: Section 3 and Appendix D for MBPP-Bangla (5-step pipeline), Appendix A for SI subset (seed prompts, self-instruction loop, filtering), Appendix B for Syn (BERTScore filtering), Appendix C for TE (MT + quality filtering with CometKiwi/BERTScore thresholds)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the conclusion, discussing scope constraints of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section contains only generic future work statements: 'expanding dataset diversity and task complexity', 'investigating larger model architectures and additional programming languages.' No specific threats like benchmark contamination risk, single-seed fragility, or selection bias in the benchmark adaptation process are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations mention the work 'focuses on Bangla code generation using 1B and 9B parameter models' but do not explicitly state what the results do NOT show. No statements about what claims are NOT being made or what populations/settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets (Bangla-Code-Instruct: 300K pairs, MBPP-Bangla: 974 problems) are claimed to be released via GitHub under CC-BY-SA-4.0. The JSONL format with task ID, prompts, reference codes, and test cases is specified in Appendix D."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: MBPP-Bangla curation (Section 3, Appendix D), SI subset (5000 expert seeds → self-instruction, Appendix A), Syn subset (GPT-4o + Claude 3.5 generation, Appendix B), TE subset (Evol-Instruct translation, Appendix C)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Translator/verifier qualifications are stated (native Bangla speakers, TOEFL >100, polyglot programmer) but how these individuals were recruited or selected is not described. The relationship to the authors (e.g., co-authors, lab members, hired annotators) is not disclosed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Each dataset's pipeline is documented with filtering criteria and stages: SI (cosine similarity <0.95 filter, syntax+execution validation, Appendix A), Syn (BERTScore ≥0.7 diversity filter, syntax check, Appendix B), TE (CometKiwi QE >0.85 + BERTScore F1 >0.95, Appendix C). Counts at each stage are implied by the final 100K per subset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers appear anywhere in the paper. The authors are at George Mason University but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are at George Mason University, Fairfax, VA, USA. They are not evaluating a commercial product they are affiliated with."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making it impossible to assess funder independence. The use of GPT-4o and Claude 3.5 Sonnet APIs for data generation implies some financial expenditure, but the source of these funds is not stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper. The authors evaluate their own models (TigerLLM, TigerCoder) but no declaration regarding related IP or commercial interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any evaluated model (GPT-3.5, GPT-4o-mini, Gemini Flash, LLaMA 3.2, etc.) or for the TigerLLM base model. This is critical since MBPP was published in 2021."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MBPP-Bangla is derived from MBPP (2021), and the English version of these problems could appear in the training data of all evaluated models. The Bangla translation may partially mitigate this but this is never discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP was published in 2021 and is widely available online. All evaluated models were trained after 2021 and could have seen these problems in English. The paper uses this benchmark without any contamination analysis or acknowledgment of this risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Translators and verifiers are part of the research team creating the benchmark, not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The paper has an ethical considerations section focused on translation quality and open-source release, not IRB matters."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Translator qualifications are described (native Bangla, TOEFL >100) but these are annotators, not study subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency, tokens consumed, or cost per example are reported. The paper proposes models for practical use but provides no cost information for inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states: 'We conduct finetuning on a single NVIDIA A100 (40GB) through Google Colab, supported by 80GB RAM and 256GB storage. The process completes in approximately 96 hours.' Hardware and training time are quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 6 reports Seed=42 for both models, indicating a single seed. No seed sensitivity analysis or results across multiple seeds are provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Seed=42 in Table 6 implies a single training run, but this is not confirmed. The number of samples per problem for Pass@K computation is also not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Table 6 caption says 'Empirically selected hyperparameters' implying some search was conducted, but no search budget, number of configurations tried, or search method is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 reports all 7 dataset combinations for both model sizes, transparently showing which combinations work best. The final model uses all three datasets (SI+Syn+TE) based on these results."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across models, datasets, benchmarks, and programming languages, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own models (TigerLLM, TigerCoder) against baselines. No acknowledgment that self-evaluation may introduce bias, and no independent evaluation is mentioned."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper highlights that TigerCoder 1B 'surpasses systems 27× larger' but does not report performance as a function of compute budget. No compute-normalized comparisons or efficiency curves are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Pass@K on MBPP-Bangla and mHumanEval-Bangla actually measures real-world Bangla code generation capability. The benchmarks test basic-to-intermediate programming problems, which may not represent practical coding needs."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are evaluated via direct inference on code generation benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. MBPP was published in 2021 and all evaluated models were trained after 2021. The English problems could be memorized by models, and the Bangla translation does not guarantee protection if models can cross-lingually transfer."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether evaluation setup provides information beyond what would be available in real Bangla code generation use cases."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The fine-tuning data (Bangla-Code-Instruct-TE) derives from Evol-Instruct, and the test benchmarks derive from MBPP. Whether there is topical overlap between Evol-Instruct and MBPP is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, decontamination pipeline, n-gram overlap analysis, or membership inference tests."
       }
     }
   },
   "claims": [
     {
       "claim": "TigerCoder achieves 11-18% Pass@1 improvements over existing multilingual and general-purpose Bangla LLMs",
       "evidence": "Table 7 shows 9B model gains of Δ=0.11 (mHumanEval P@1) and Δ=0.13 (MBPP P@1). The 1B model gains are only Δ=0.05 at P@1. The 18% figure appears only at P@100 (MBPP 9B: Δ=0.18), not at P@1 as the abstract claims.",
       "supported": "weak"
     },
     {
       "claim": "Machine-translating Bangla prompts to English does not improve code generation performance",
       "evidence": "Table 4 shows MT variants produce similar or worse results compared to direct Bangla prompts across all models and benchmarks. Figure 1 provides qualitative analysis of keyword mistranslation causing failures.",
       "supported": "moderate"
     },
     {
       "claim": "Most LLMs exhibit significant performance drops when coding prompts are in Bangla rather than English",
       "evidence": "Table 3 shows consistent drops across all models except TigerLLM: LLaMA drops from 0.73 to 0.15 on mHumanEval P@1, Phi-4 from 0.79 to 0.10, GPT-3.5 from 0.79 to 0.56.",
       "supported": "strong"
     },
     {
       "claim": "Curated high-quality datasets empower smaller models to overcome low-resource limitations, challenging the notion that scale alone drives performance",
       "evidence": "Table 7 shows TigerCoder 1B (with curated data) outperforms Gemma-3 27B on Bangla benchmarks. Figure 2's ablation shows dataset combination effects.",
       "supported": "moderate"
     },
     {
       "claim": "Combined use of all three instruction datasets (SI+Syn+TE) yields the best fine-tuning results",
       "evidence": "Figure 2 shows all-three combination achieves top scores across both benchmarks and model sizes, with 9B reaching 0.82 P@1 on MBPP-Bangla.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Abstract overclaiming",
       "detail": "The abstract claims '11-18% performance gains at Pass@1' but the 18% gain is at Pass@100, not Pass@1. The 1B model achieves only 5% at Pass@1. The actual Pass@1 range for the 9B model is 11-13%."
     },
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results are from a single seed (seed=42) with no variance, confidence intervals, or repeated runs. Pass@K values can vary significantly with different random seeds and generation temperatures, neither of which are explored."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MBPP was published in 2021 and is widely available. All evaluated models were trained after this date and could have memorized the English problems. Translation to Bangla does not eliminate contamination risk via cross-lingual transfer, and this is never discussed."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors evaluate their own models (TigerLLM and TigerCoder) against external baselines. TigerLLM is presented as the strongest prior baseline. No independent evaluation or acknowledgment of self-evaluation bias."
     },
     {
       "flag": "Missing inference parameters",
       "detail": "Temperature, top-p, number of samples per problem (n), and other generation parameters for Pass@K evaluation are not reported. These critically affect Pass@K scores, especially at higher K values."
     },
     {
       "flag": "Generalization overclaim",
       "detail": "The conclusion claims results provide 'an effective and replicable blueprint for Bangla and other low-resource languages' but only Bangla was tested. No evidence that the approach transfers to other languages."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark, foundational for code generation evaluation including this paper's mHumanEval-Bangla adaptation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark, the direct source of this paper's MBPP-Bangla benchmark."
     },
     {
       "title": "mHumanEval - a multilingual benchmark to evaluate large language models for code generation",
       "authors": ["Nishat Raihan", "Antonios Anastasopoulos", "Marcos Zampieri"],
       "year": 2025,
       "relevance": "Prior work by same authors creating multilingual code generation benchmarks; provides mHumanEval-Bangla used as evaluation benchmark."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "State-of-the-art code LLM demonstrating advances in code generation capabilities."
     },
     {
       "title": "Cross-lingual generalization through multitask finetuning",
       "authors": ["Niklas Muennighoff", "Alex Wang", "Alena Fenogenova"],
       "year": 2023,
       "relevance": "Documents performance gaps in multilingual code generation, showing models perform poorly on non-English code tasks."
     },
     {
       "title": "Self-instruct: Aligning language models with self-generated instructions",
       "authors": ["Yizhong Wang", "Yeganeh Kordi", "Swaroop Mishra"],
       "year": 2023,
       "relevance": "Core methodology used for creating the Bangla-Code-Instruct-SI dataset; key technique for LLM data curation."
     },
     {
       "title": "WizardLM: Empowering large language models to follow complex instructions",
       "authors": ["Canwen Xu", "Ruqing Wang", "Yeyun Gong"],
       "year": 2023,
       "arxiv_id": "2304.12244",
       "relevance": "Source of Evol-Instruct dataset used as basis for the Bangla-Code-Instruct-TE translated subset."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Abhimanyu Dubey", "Abhinav Jauhri"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "LLaMA 3 model evaluated as baseline; major open-source multilingual LLM with minimal Bangla support."
     },
     {
       "title": "Challenges to using large language models in code generation and repair",
       "authors": ["Liliana Pasquale", "Antonino Sabetta", "Marcelo d'Amorim"],
       "year": 2025,
       "relevance": "Discusses limitations and challenges of LLMs for code generation in software engineering."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "TigerCoder claims to challenge the 'scale alone drives performance' thesis from this foundational work."
     },
     {
       "title": "BabelCode: LLM as a polyglot programmer",
       "authors": ["Tianyi Wang", "Yang Ye", "Panupong Pasupat"],
       "year": 2023,
       "arxiv_id": "2303.03845",
       "relevance": "Multilingual code generation evaluation framework relevant to the multi-PL evaluation in this work."
     }
   ]
 }