ai-research-survey

scan.json (28748B)

---

 {
   "paper": {
     "title": "NALA_MAINZ at BLP-2025 Task 2: A Multi-agent Approach for Bangla Instruction to Python Code Generation",
     "authors": [
       "Hossain Shaikh Saadi",
       "Faria Alam",
       "Mario Sanz-Guerrero",
       "Minh Duc Bui",
       "Manuel Mager",
       "Katharina von der Wense"
     ],
     "year": 2025,
     "venue": "BLP-2025 Workshop (arXiv preprint)",
     "arxiv_id": "2511.16787",
     "doi": "10.48550/arXiv.2511.16787"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "A two-stage multi-agent pipeline (code generation + selective debugging) wins BLP-2025 Bangla→Python code generation with GPT-5 at 95.4% Pass@1. The debugger stage provides the largest gains (GPT-5 jumps from 64.6% to 95.4%), driven primarily by exposing the model to unit tests and error traces. External test case augmentation is critical—without it, Pass@1 drops to 86.0%. Translation from Bangla to English generally hurts or has mixed effects, suggesting the models handle Bangla instructions adequately.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We make our code publicly available' with a GitHub link provided in footnote 1 (https://github.com/shaikhsaadi999/blp25_code_genneration)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The development and test datasets are provided by the shared task organizers (Raihan et al., 2025a,c), and the external dataset is the publicly available Austin et al. (2021). Standard public datasets used without modification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment specifications are mentioned in the paper. Only the proprietary API providers are named."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository is linked but the paper itself contains no instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 95.4, 64.6, 82.60) with no confidence intervals, error bars, or uncertainty measures anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-5 'outperforms' other models and that Stage 2 'consistently boosts' performance based solely on comparing raw numbers. No statistical significance tests (t-tests, bootstrap, etc.) are used."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: 'for GPT-5, the increment is 47.67%' (from 64.6 to 95.4), and similar for other models (Section 6). The effect of external data is also quantified: 86.00 → 95.4."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset sizes (400 dev, 500 test) are dictated by the shared task organizers. No justification is given for whether these sizes are adequate for the claims being made, nor is any power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviations, variance across seeds, or multiple-run statistics are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four proprietary models are compared (GPT-5, GPT-4.1, Claude Sonnet 4, Gemini-2.5-Flash) across both pipeline stages, and Stage 1 (no debugging) serves as a baseline for Stage 2 (with debugging)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All models compared are contemporary and competitive: GPT-5, GPT-4.1, Claude Sonnet 4, and Gemini-2.5-Flash are all current-generation proprietary models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations are presented: Stage 1 vs Stage 2 (effect of debugging), with vs without external data (86.00 vs 95.4), generated vs external test cases (84.00 vs 95.4), and with vs without Bangla→English translation (Figure 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 is used as the evaluation metric. Error rate is also reported but is just 100 - Pass@1, not an independent metric. No other metrics (e.g., code quality, efficiency, readability) are measured."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via pytest unit test pass/fail. No human evaluation of code quality, readability, or correctness beyond test passing is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The final evaluation uses hidden unit tests on Codabench (Section 6, 'Overfitting to the unit tests'). The dev set (400 samples) and test set (500 samples) are clearly separated, and the shared task uses hidden tests for final scoring."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate Pass@1 scores are reported. No breakdown by problem difficulty, instruction type, or error category is provided. The paper notes some codes 'cannot handle edge cases' but doesn't quantify which categories fail."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes generically that 'generated codes sometimes cannot handle edge cases, or they are not generalizing well' (Section 6) but provides no specific failure examples, error analysis, or qualitative examination of where the system breaks down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Gemini-2.5-Flash shows minimal improvement with debugging (only 13.68% gain), translation hurts GPT-5 performance in both stages, and generated test cases (84.00) underperform external tests (95.4). The dev→test performance gap (99.8→95.4) is also honestly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Pass@1 of 95.4 and first place, both supported by Table 2 and the shared task results. The claim of a 'simple but effective' multi-agent pipeline is supported by the two-stage architecture and results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('structured feedback consistently boosts model performance', 'targeted, test-driven refinement substantially improves code synthesis') are supported by controlled ablation studies: removing the debugging stage, removing external data, and comparing translation effects. These are single-variable manipulations within the pipeline."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested setting. The title specifies 'BLP-2025 Task 2' and 'Bangla Instruction to Python Code Generation.' The conclusion says 'code synthesis in an underserved language,' appropriately scoped to the shared task context."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses overfitting to unit tests as one alternative explanation, but does not consider other factors: whether GPT-5's superiority is due to better Bangla training data, whether the improvement from Stage 2 is simply due to having more context (tests) rather than the multi-agent architecture, or whether the external dataset advantage reflects data leakage rather than generalization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@1 (passing unit tests) and claims Pass@1 scores. It does not overclaim beyond the measurement — no broader claims about 'code quality' or 'developer productivity.' The proxy matches the claim granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT-5', 'GPT-4.1', 'Gemini-2.5-Flash', 'Claude-Sonnet-4' (Section 5). No API versions, snapshot dates, or model IDs are provided. Model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix A for all agents: coding agent user prompt, debugger agent system and user prompts, and test generation agent system and user prompts. The fill values are data-instance variables ({spec.name}, {spec.instruction_bn}, {code}, etc.) that are deterministic from the dataset."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only 'reasoning effort' settings are mentioned (low for coding agent, high for debugger, Section 5). No temperature, top-p, max tokens, or other sampling parameters are reported for any of the four proprietary APIs."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-agent pipeline is described in detail in Section 5 with Algorithm 1 and Figure 1: code generation → test execution → selective debugging with error trace extraction. The feedback mechanism, retry logic (max 2 attempts in Stage 1), and agent roles are all specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data handling is documented: dev set composition (400 instructions, 3 unit tests each), test set (500 instructions, 1 unit test), external dataset matching by function name (480/500 matched), and how non-overlapping external tests are appended (Section 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present at the end of the paper with substantive discussion of reproducibility concerns, reliance on external data, and model opacity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: proprietary-only models limiting reproducibility, 10pp score drop without external data enrichment, and inability to attribute performance differences due to opaque training data. These are specific to this study, not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'focuses exclusively on proprietary models... does not evaluate any open-source models,' dependence on external dataset acknowledged with quantified impact (10pp), and 'we lack transparency into their training data' stated as a boundary on interpretability of results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The shared task input data is available through organizers, but the raw experimental outputs (generated code, intermediate debugging results, model responses) are not released for independent verification. Results depend on proprietary API outputs that cannot be exactly reproduced."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are well described in Section 4: organizer-provided development set (400 instructions, 3 unit tests each), test set (500 instructions, 1 unit test), and external dataset (Austin et al. 2021) with function name matching yielding 480 matches."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses shared task benchmark data and publicly available datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: function name matching against external dataset (480/500), non-overlapping test extraction and appending, two-stage processing with Algorithm 1 specifying how failed codes are forwarded to the debugger."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgement section: 'This work was supported by the Carl Zeiss Foundation through the TOPML and MAINCE projects (grant numbers P2021-02-014 and P2022-08-009I).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Johannes Gutenberg University Mainz, Saarland University, University of Colorado Boulder. Authors are academic researchers with no disclosed affiliation to the proprietary model providers (OpenAI, Google, Anthropic)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Carl Zeiss Foundation is a general research funder with no financial stake in the outcome of a Bangla code generation shared task or the relative performance of proprietary LLM APIs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper. Absence of a declaration is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four proprietary models used (GPT-5, GPT-4.1, Claude Sonnet 4, Gemini-2.5-Flash). This is critical since the models may have been trained on similar programming problems."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the BLP-2025 shared task problems, the Austin et al. (2021) dataset, or similar programming problems appeared in any model's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Austin et al. (2021) dataset was published before all models' training cutoffs and is likely in their training data. The BLP-2025 task adapts existing problems to Bangla. Neither contamination vector is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of code generation models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, wall-clock time, or cost per example reported despite using four different proprietary APIs across two stages with multiple attempts per sample."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or resource usage is stated anywhere in the paper."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. All results appear to be from single runs with no variation assessment."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many times the pipeline was executed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The 'reasoning effort' settings (low/high) appear chosen without documented exploration of alternatives."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'For our primary submission, we use GPT-5 since it is the best-performing model in the dev set' (Section 5). Model selection is based on development set performance, not test set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple model comparisons are made across both stages without any statistical tests at all, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. The authors designed the pipeline and evaluated it themselves without acknowledging this potential bias or involving independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis despite comparing models of likely very different cost profiles (GPT-5 vs Gemini-2.5-Flash). GPT-5 wins on accuracy but the cost trade-off is never discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Pass@1 on unit tests actually measures Bangla instruction-to-code generation ability, or whether the benchmark has construct validity issues (e.g., whether solutions that pass tests are actually correct for the Bangla instructions)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All four models are evaluated within the same two-stage pipeline with identical prompts, test augmentation, and debugging workflow. The scaffold is held constant across model comparisons, controlling for this confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The Austin et al. (2021) external dataset was published years before the models' training, meaning solutions to those problems likely appear in training data. This is never addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The debugger stage provides unit tests and error traces as input, giving the model substantial information about expected behavior. While this is by design, the paper does not discuss whether this constitutes feature leakage relative to a real-world deployment scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the shared task problems are independent of examples in the models' training data, or whether the Austin et al. (2021) external test cases create overlap between training and evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination is mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "The multi-agent pipeline achieves first place in BLP-2025 Task 2 with Pass@1 of 95.4%",
       "evidence": "Table 2 shows GPT-5 achieves 95.4 Pass@1 in Stage 2. The paper states this secured first place on the official leaderboard (Section 1, Section 7).",
       "supported": "strong"
     },
     {
       "claim": "The selective debugging stage (Stage 2) dramatically improves performance over code generation alone (Stage 1)",
       "evidence": "Tables 1-2: GPT-5 jumps 64.6→95.4 (+47.67%), GPT-4.1 58.0→82.6 (+41.37%), Claude Sonnet 4 58.2→79.0 (+35.73%), Gemini-2.5-Flash 52.6→59.8 (+13.68%).",
       "supported": "moderate"
     },
     {
       "claim": "External test case augmentation is critical, improving GPT-5 from 86.00 to 95.4 Pass@1",
       "evidence": "Section 6 ('Effect of external data'): 'without the external unit test for GPT-5, our proposed pipeline achieves an 86.00 Pass@1 score'.",
       "supported": "moderate"
     },
     {
       "claim": "Generated test cases (84.00 Pass@1) underperform external test cases (95.4) but still substantially improve over Stage 1 (64.6)",
       "evidence": "Section 6 ('Effect of generated unit tests'): GPT-5 with generated tests achieves 84.00, lower than 95.4 with external tests but much higher than 64.60 without tests.",
       "supported": "moderate"
     },
     {
       "claim": "Translation from Bangla to English has mixed effects and generally does not help GPT-5",
       "evidence": "Figure 2 and Section 6 ('Effect of translation'): GPT-5 Stage 1 drops 64.6→60.8, Stage 2 drops 95.8→94.8 with translation. Other models show mixed results (some improve in Stage 2).",
       "supported": "moderate"
     },
     {
       "claim": "The system overfits to provided unit tests, with performance dropping from 99.8 (dev, all tests visible) to 95.4 (test, hidden tests)",
       "evidence": "Section 6 ('Overfitting to the unit tests'): dev phase achieves 99.8 Pass@1 with all test cases visible; test phase with hidden tests drops to 95.4.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests for any comparative claims",
       "detail": "All claims of model superiority and component effectiveness are based on comparing raw Pass@1 numbers without significance tests, confidence intervals, or multiple-run variance. With only point estimates, the reported differences could be within noise."
     },
     {
       "flag": "Single-run results with no variance assessment",
       "detail": "All experiments appear to be single runs. LLM outputs are stochastic — the same pipeline could produce different results on a second run. Without multiple runs, the stability of the 95.4% result is unknown."
     },
     {
       "flag": "Contamination risk from Austin et al. (2021) dataset",
       "detail": "The external dataset (Austin et al. 2021) was published 4+ years before these models were trained. Solutions to these problems are very likely in the training data. The 480 external test cases may be testing memorization rather than Bangla comprehension."
     },
     {
       "flag": "Proprietary-only evaluation limits reproducibility",
       "detail": "All four models are proprietary APIs with opaque training data and changing behavior. Results cannot be exactly reproduced, and the paper acknowledges this in its Limitations section."
     },
     {
       "flag": "No cost analysis despite multi-stage API usage",
       "detail": "The pipeline calls proprietary APIs multiple times per sample (generation + retry + debugging). GPT-5 with high reasoning effort is expensive. No cost data is provided, making practical deployment assessment impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and Pass@k metrics that are the standard for code generation evaluation, directly relevant to benchmark methodology."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Provides the external dataset (MBPP) used for test case augmentation; foundational LLM code generation benchmark."
     },
     {
       "title": "A survey on code generation with LLM-based agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jiaru Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Recent survey on LLM-based agent code generation approaches, directly relevant to the survey scope."
     },
     {
       "title": "Large language models for code generation: A comprehensive survey of challenges, techniques, evaluation, and applications",
       "authors": ["Nam Huynh", "Beiyu Lin"],
       "year": 2025,
       "arxiv_id": "2503.01245",
       "relevance": "Comprehensive survey on LLM code generation covering evaluation methods and challenges."
     },
     {
       "title": "A survey on large language models for code generation",
       "authors": ["Juyong Jiang", "Fan Wang", "Jiasi Shen"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey on LLM code generation documenting the English-centric bias in benchmarks and systems."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for a major open-source code generation model, relevant to code LLM evaluation."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code generation model; relevant to the landscape of code LLMs being evaluated."
     },
     {
       "title": "mHumanEval - a multilingual benchmark to evaluate large language models for code generation",
       "authors": ["Nishat Raihan", "Antonios Anastasopoulos", "Marcos Zampieri"],
       "year": 2025,
       "relevance": "Multilingual code generation benchmark directly relevant to evaluating LLMs on non-English code generation tasks."
     },
     {
       "title": "MConala: A benchmark for code generation from multiple natural languages",
       "authors": ["Zhiruo Wang", "Grace Cuenca", "Shuyan Zhou"],
       "year": 2023,
       "arxiv_id": "2203.08388",
       "relevance": "Multilingual code generation benchmark highlighting the English-centric limitation of existing evaluations."
     },
     {
       "title": "Qwen3 technical report",
       "authors": ["An Yang", "Anfeng Li", "Baosong Yang"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Technical report for a major LLM family; relevant to the landscape of models used for code generation."
     }
   ]
 }