ai-research-survey

scan.json (24743B)

---

 {
   "paper": {
     "title": "BanglaForge: LLM Collaboration with Self-Refinement for Bangla Code Generation",
     "authors": [
       "Mahir Labib Dihan",
       "Sadif Ahmed",
       "Md Nafiu Rahman"
     ],
     "year": 2025,
     "venue": "BLP-2025 (Workshop on Bangla Language Processing)"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The paper references the BLP-2025 starter kit URL for the dataset but does not provide a link to the BanglaForge implementation itself."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset comes from the BLP-2025 shared task and is publicly available via the official starter kit at https://noshinulfat.github.io/blp25_code_generation_task/#/get-started. External benchmarks mHumanEval-Bangla and MBPP-Bangla are also publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. Appendix B mentions generation parameters (temperature, top_p, max_new_tokens) but does not describe the software environment, library versions, or hardware used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the methodology is described in detail and prompts are given in the appendix, there are no explicit instructions for running the pipeline end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single-point Pass@1 percentages (e.g., 84.00%, 95.5%) with no confidence intervals, error bars, or uncertainty estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., ablation components cause performance drops) but uses no statistical significance tests. Differences are simply compared as raw percentage points."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout. For example, the ablation study in Table 11 shows the full model at 95.5% vs. without translation at 73.6% (a 21.9 percentage point drop), without feedback loop at 69.8%, etc. These provide sufficient context for understanding effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset sizes are stated (Trial: 74, Dev: 400, Test: 500 in Table 1) but no justification is given for why these sizes are adequate for the claims made. No power analysis or acknowledgment of sample size limitations."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or any spread measure is reported. Results appear to be from single runs. There is no mention of multiple runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares multiple models including Gemma-1B, GPT-OSS-20B, DeepSeek-R1-Llama-70B, Gemini-2.0-Flash, and Lg Exaone Deep 32B under various configurations, serving as baselines for comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary models such as DeepSeek-R1-Llama-70B (2025), Gemini-2.0-Flash (2024), Gemini-2.5-Pro (2025), and Lg Exaone Deep 32B (2024). These are recent and competitive models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E contains a thorough ablation study with 8 variants (Tables 4-11): removing translation, glossary, feedback loop, reviewer, RAG, varying iteration count M, and varying k. Each isolates a single component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 accuracy is reported. No other metrics such as Pass@k (for k>1), compilation rate, token efficiency, or latency are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is performed. All evaluation is automated via unit test execution (Pass@1). Given claims about code 'robustness' and 'stylistic reliability,' human evaluation of code quality would have been relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates development (400 problems) and test (500 problems) sets. Development set results are used for model selection and ablations; final results are reported on the held-out test set (Section 5.3, Table 2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only overall Pass@1 accuracy is reported. No per-category, per-difficulty, or per-problem-type breakdowns are provided despite the dataset containing diverse problem types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G (Failure Cases and Dataset Limitations) discusses specific failure modes including semantic translation errors (e.g., 'even' translated as 'এমনকি' instead of 'জোড়'), incorrect terminology (e.g., 'Map' translated as geographic map), and loss of context/intent."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports several configurations that hurt performance (e.g., removing translation drops to 73.6%, removing feedback loop drops to 69.8%). Table 10 shows that increasing k beyond 5 slightly decreases performance (94.7% at k=7 vs. 95.5% at k=5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'Pass@1 accuracy of 84.00%' on the BLP-2025 benchmark, which matches the test set result in Table 2 (Gemini-2.5-Pro with RAG: 84.00%). The abstract's claims about 'effectiveness of retrieval, model collaboration, and self-refinement' are supported by the ablation study."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation studies (e.g., 'removing the translation stage reduces Pass@1 accuracy by nearly 22 percent'). The ablation design uses controlled single-variable manipulation, which is adequate for these within-system component contribution claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion claim effectiveness for 'low-resource Bangla code generation' broadly, but results are only on the BLP-2025 benchmark which consists of machine-translated English problems. The paper's title and framing suggest generality to Bangla code generation, but the dataset is machine-translated MBPP/HumanEval, not natively Bangla programming tasks. The Limitations section (Section 7) acknowledges this partially but the title and abstract do not bound the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, it does not consider whether the improvement from translation is simply because the models were trained on English code and thus English prompts are inherently easier, rather than the translation component being novel. No threats-to-validity or confounds discussion."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referenced by marketing names only: 'Gemini-2.0-Flash', 'Gemini-2.5-Pro', 'DeepSeek-R1-Llama-70B', 'Lg Exaone Deep 32B'. No API versions, snapshot dates, or specific model IDs are provided. The references list 'Accessed: 2025-10-05' for model pages but not specific model version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix C (Figures 3-9): system prompts for coder, reviewer, and translator models, main prompt templates, failed attempt feedback templates, and few-shot example templates. While these contain placeholders, the placeholder semantics are clearly defined and the fill values come from the dataset (instructions, tests, etc.)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states: 'temperature = 0.7, top_p = 0.9, and max_new_tokens = 1024. Each query generated n = 1 output sample per decoding pass.' Additional parameters: k=5 for retrieval, M=5 for maximum refinement iterations."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding (translation -> retrieval -> coder -> reviewer -> feedback loop) is described in detail in Section 4 and Algorithm 1 (Appendix F). The workflow includes tool descriptions, the retry logic (max M=5 iterations), and the feedback mechanism with error categories (Table 3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the dataset structure, splits (Table 1: Trial 74, Dev 400, Test 500), and the JSON format of each entry. Section 4.1 describes function prototype normalization. Section 4.2 explains how the retrieval database is constructed (Trial set for Dev experiments, Trial+Dev for Test experiments)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Limitations') provides a dedicated discussion of five specific limitations: translation quality dependency, TF-IDF retrieval limitations, assumption of well-structured input, fixed iteration limits, and machine-translated dataset limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section discusses threats specific to this study: 'the system relies heavily on high-quality bilingual translation; inaccuracies in Bangla-to-English mapping or glossary coverage can propagate errors,' and 'since the dataset itself originates from machine-translated English sources, true Bangla-native problem framing and linguistic diversity remain under-represented.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While limitations are discussed, there is no explicit statement bounding claims (e.g., 'our results do not generalize to natively-written Bangla problems' or 'we have not tested on languages other than Bangla'). The title claims 'Bangla Code Generation' broadly without qualifying that this is only tested on machine-translated benchmark problems."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark dataset is publicly available via the BLP-2025 starter kit, but the authors' experimental outputs (generated code, per-problem results, logs of refinement iterations) are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data source: 'Our dataset comes from the Bangla Code Generation shared task (Task 2) at BLP-2025.' The dataset structure (id, Bangla instruction, response, test_list) and splits are clearly documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study uses publicly available benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline from input (Bangla instruction + unit tests) through translation, retrieval, code generation, review, and refinement is documented in Section 4 and Algorithm 1. The retrieval database construction is also described (Section 4.2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are from the Department of Computer Science and Engineering, Bangladesh University of Engineering and Technology (BUET)."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The authors are university researchers with no apparent corporate affiliation that would create a conflict regarding the evaluated models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses multiple pre-trained LLMs (Gemini-2.5-Pro, Lg Exaone Deep 32B, etc.) on code generation benchmarks derived from HumanEval and MBPP, but does not state the training data cutoff dates for any of the models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The benchmarks are Bangla translations of HumanEval and MBPP, which are well-known English benchmarks. The models may have seen the English originals during pre-training, which could affect performance since the pipeline translates Bangla back to English. This contamination vector is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP are well-known benchmarks published before the training cutoffs of the models used. The Bangla translations may provide partial decontamination, but the pipeline explicitly translates back to English before generating code, potentially re-introducing contamination. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, tokens consumed, or wall-clock time is reported. The system calls multiple LLMs (translator, coder, reviewer) with up to 5 refinement iterations per problem, which could be expensive, but cost is never mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, hardware specifications, GPU hours, or total API spend is reported. The paper does not mention what hardware or infrastructure was used to run the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "BanglaForge achieves 84.00% Pass@1 accuracy on the BLP-2025 Bangla Code Generation benchmark test set.",
       "evidence": "Table 2 shows Gemini-2.5-Pro with RAG (Trial+Dev), 5 examples, translation, and 1 unit test achieves 84.00% Pass@1 on the test set.",
       "supported": "strong"
     },
     {
       "claim": "English translation is the most impactful component, with removal causing a 21.9 percentage point drop in accuracy.",
       "evidence": "Table 4 (Appendix E.1) shows Full Model at 95.5% vs. Bangla Only at 73.6% on the development set using Lg Exaone Deep 32B.",
       "supported": "moderate"
     },
     {
       "claim": "The feedback loop is critical, with removal causing a 25.7 percentage point drop.",
       "evidence": "Table 6 (Appendix E.3) shows Full Model at 95.5% vs. Without Feedback Loop at 69.8% on the development set.",
       "supported": "moderate"
     },
     {
       "claim": "Retrieval augmentation provides consistent improvement over manual few-shot prompting.",
       "evidence": "Table 9 (Appendix E.6) shows RAG at 95.5% vs. Manual Few-shot at 94.2%, a 1.3 percentage point improvement on the development set.",
       "supported": "weak"
     },
     {
       "claim": "The Reviewer LLM improves robustness and coverage, with removal causing a 5.1 percentage point drop.",
       "evidence": "Table 7 (Appendix E.4) shows Full Model at 95.5% vs. Without Reviewer at 90.4% on the development set.",
       "supported": "moderate"
     },
     {
       "claim": "The controlled translation glossary improves accuracy by over 7 percentage points.",
       "evidence": "Table 5 (Appendix E.2) shows With Glossary at 95.5% vs. Without Glossary at 88.2% on the development set.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BanglaForge is a retrieval-augmented dual-LLM framework for generating Python code from Bangla natural language descriptions that achieves 84.00% Pass@1 on the BLP-2025 test set. The system combines Bangla-to-English translation, TF-IDF-based example retrieval, a coder-reviewer architecture, and iterative self-refinement with execution feedback. Ablation studies on the development set show that English translation and the feedback loop are the two most impactful components, each contributing over 20 percentage points to performance. The results suggest that current LLMs still struggle with direct Bangla code generation and benefit substantially from translation-mediated approaches.",
   "red_flags": [
     {
       "flag": "No variance or multiple runs",
       "detail": "All results appear to be from single runs with no error bars, standard deviations, or repeated experiments. Given that LLM outputs are stochastic (temperature=0.7), results could vary substantially across runs."
     },
     {
       "flag": "Contamination risk via back-translation",
       "detail": "The benchmark consists of machine-translated HumanEval and MBPP problems. The pipeline translates Bangla instructions back to English before code generation. The models may have seen the original English HumanEval/MBPP problems during pre-training, which could inflate performance. This contamination vector is never discussed."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only Pass@1 is reported. Additional metrics like Pass@k, compilation success rate, or code quality measures would provide a more complete picture of system performance."
     },
     {
       "flag": "Ablation on dev set only, final results use different model",
       "detail": "All ablation experiments use Lg Exaone Deep 32B on the development set, but the final test set result (84.00%) uses Gemini-2.5-Pro. It is unclear whether the component contributions observed with one model transfer to the other. The Lg Exaone Deep 32B test set result (80.60%) is lower than many ablated dev set configurations."
     },
     {
       "flag": "No cost reporting despite multi-LLM pipeline",
       "detail": "The system calls three separate LLMs (translator, coder, reviewer) with up to 5 refinement iterations per problem, but no cost, latency, or resource usage is reported. This makes practical applicability assessment impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the foundational code generation benchmark from which the Bangla evaluation dataset is derived."
     },
     {
       "title": "mHumanEval - a multilingual benchmark to evaluate large language models for code generation",
       "authors": ["Nishat Raihan", "Antonios Anastasopoulos", "Marcos Zampieri"],
       "year": 2025,
       "relevance": "Multilingual extension of HumanEval used as one of the evaluation benchmarks in this paper; directly relevant to cross-lingual code generation evaluation."
     },
     {
       "title": "TigerCoder: A Novel Suite of LLMs for Code Generation in Bangla",
       "authors": ["Nishat Raihan", "Antonios Anastasopoulos", "Marcos Zampieri"],
       "year": 2025,
       "arxiv_id": "2509.09101",
       "relevance": "Introduces Bangla-specific code LLMs and the MBPP-Bangla benchmark used in this paper's evaluation."
     },
     {
       "title": "BenLLMEval: A comprehensive evaluation into the potentials and pitfalls of large language models on Bengali NLP",
       "authors": ["Mohsinul Kabir", "Mohammed Saidul Islam", "Md Tahmid Rahman Laskar"],
       "year": 2023,
       "arxiv_id": "2309.13173",
       "relevance": "Comprehensive evaluation of LLMs on Bangla NLP tasks revealing performance gaps, relevant to understanding LLM capabilities in low-resource languages."
     },
     {
       "title": "DeepSeek-R1: Reasoning models built on llama-70b",
       "authors": ["DeepSeek AI"],
       "year": 2025,
       "relevance": "One of the baseline models evaluated in the paper for Bangla code generation."
     },
     {
       "title": "Dense passage retrieval for open-domain question answering",
       "authors": ["Vladimir Karpukhin", "Barlas Oguz", "Sewon Min"],
       "year": 2020,
       "relevance": "Foundational work on dense retrieval that the paper compares against in justifying its TF-IDF retrieval approach."
     },
     {
       "title": "BanglaBERT: Language model pretraining and benchmarks for low-resource language understanding evaluation in Bangla",
       "authors": ["Abhik Bhattacharjee", "Tahmid Hasan", "Wasi Uddin Ahmad"],
       "year": 2022,
       "relevance": "Key work on Bangla-specific language model pretraining, relevant to understanding low-resource language model capabilities."
     },
     {
       "title": "BEnQA: A question answering and reasoning benchmark for Bengali and English",
       "authors": ["Sheikh Shafayat", "Quamran Hasan H. M.", "Minhajur Rahman Chowdhury Mahim"],
       "year": 2024,
       "arxiv_id": "2403.10900",
       "relevance": "Parallel Bengali-English QA benchmark showing that cross-lingual approaches improve Bengali performance, relevant to the translation-mediated approach used here."
     }
   ]
 }