ai-research-survey

scan.json (28864B)

---

 {
   "paper": {
     "title": "Test-Driven Development for Code Generation",
     "authors": ["Noble Saji Mathews", "Meiyappan Nagappan"],
     "year": 2024,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2402.13521",
     "doi": "10.1145/3691620.3695527"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Providing test cases alongside problem statements to LLMs improves code generation correctness by 9.15–29.57% across MBPP, HumanEval, and a curated CodeChef dataset, with less performant models (Llama 3) benefiting more than stronger ones (GPT-4). Remediation loops that feed test failures back to the LLM add an additional 5.26–9.02% improvement. Problem difficulty inversely correlates with TDD effectiveness, and solutions generated with test information also perform well against unseen private tests, suggesting the improvement is not mere test overfitting.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided at OSF (https://osf.io/e3jy6/) which includes 'code scripts for the experiments' as stated in Section 3.2."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The primary datasets (MBPP, HumanEval via EvalPlus) are publicly available. The CodeChef dataset is curated from a public platform. The replication package includes 'output and runtime details from TGen for all cases we experimented with' (Section 3.2)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions GPT-4 Turbo v1106, Llama 3 70B Instruct, temperature 0, and seed 1106, but provides no requirements.txt, Dockerfile, or detailed environment specification for reproducing the Python execution environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The replication package contains scripts and output data, but the paper itself includes no step-by-step reproduction instructions, README description, or 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimate percentages (e.g., '80.5% of MBPP problems') with no confidence intervals or error bars anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'including tests contributes to solving an additional 12.0%' are made by comparing raw percentages with no statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 and the text report percentage improvements with baseline context (e.g., MBPP: 69.67% → 82.45% → 87.71%, improvement of +12.78% and +5.26%), allowing readers to assess magnitude in context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes. The 399 MBPP and 164 HumanEval problems are inherited from existing benchmarks; the 1100 CodeChef problems (100 per difficulty level) are chosen without power analysis or justification for why 100 per level is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments use a single seed (1106) and temperature 0 with no variance, standard deviation, or spread measure reported across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares: (1) baseline code generation with problem statement alone, (2) with tests added, (3) with tests plus remediation loop. Figure 4, Table 1, and Table 3 present these comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 Turbo v1106 and Llama 3 70B Instruct were among the most capable models at the time of publication. The baseline is the same model without TDD intervention, which is the appropriate comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper progressively adds components: no tests → tests → tests + remediation, measuring each component's contribution. Section 5.2 also varies the number of tests (Figure 7), serving as an ablation of test quantity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The sole evaluation metric is pass rate (fraction of problems solved correctly). No secondary metrics such as code quality, code length, readability, or execution efficiency are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via PyTest pass/fail. While the authors manually inspect specific cases for qualitative insights (Sections 4.2–4.4), there is no systematic human evaluation of the generated code."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "EvalPlus private tests (35x for MBPP, 80x for HumanEval) are used as held-out evaluation that the LLM never sees. CodeChef uses the platform's private test suites. Results validated against these are reported in Table 1."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (MBPP, HumanEval, CodeChef), by strategy (solved without tests, needs tests, needs remediation), and by difficulty level (Figure 5, Figure 6, Table 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 extensively categorizes unsolved problems: misunderstanding core logic, data structure handling, logical errors, input/output format issues, and performance bottlenecks, with specific problem IDs cited."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: remediation has diminishing returns beyond 3–4 iterations (Section 4.3), adding more tests can cause regressions due to 'lost in the middle' (Section 5.2), and 47.18% of CodeChef problems remain unsolved (Section 5.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'including test cases leads to higher success in solving programming challenges' which is supported by the consistent improvements across MBPP (+12.78%), HumanEval (+9.15%), and CodeChef (+3.09%) shown in Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims 'providing LLMs with tests enhances code generation outcomes.' The experimental design holds the model constant and varies only the input (with/without tests), which is adequate controlled single-variable manipulation for this causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'we therefore advocate for the widespread adoption of test-driven methodologies to maximize the benefits of LLMs in code generation' — a broad generalization from experiments on two models, Python-only benchmarks, and function/file-level tasks. The title 'Test-Driven Development for Code Generation' is similarly unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider that adding tests simply adds more input context (any additional context might help), or that tests may leak structural information about the expected solution. Section 7 discusses generic threats (prompt sensitivity, benchmark limitations) but not alternative explanations for why the improvements occur."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass rate on test suites and claims to measure 'correctness' of code generation. The proxy (test-passing) closely matches the claimed outcome (correctness), with no broader framing that would create a proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'GPT-4 Turbo v1106' (Section 3.2), 'GPT-3.5 Turbo v1106' (Section 5.3), and 'Meta Llama 3 70B Instruct' (Section 5.3)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for the Coder Agent is shown in Figure 2 and the Remediation Agent configuration in Figure 3, including the multi-step reasoning instructions and dataset-specific guidelines."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states: 'We use a seed value of 1106 and a temperature of 0 in our experiments.' Remediation loop parameters are also specified: 5 iterations max, 3 repeated failures."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The TGen framework is described in detail in Section 3.2 with Figure 1 showing the full pipeline: input phase, LLM engine, coder and remediation agents, verifier, and remediation loop with specific stopping criteria."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Dataset preparation is documented: MBPP uses the EvalPlus sanitized subset (399 of 427 problems), HumanEval uses 164 problems with EvalPlus enrichment, CodeChef uses 100 most popular problems per difficulty level scraped November 2023. Sample test cases were removed from problem statements for the baseline experiment."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated discussion of limitations spanning prompt engineering, benchmark limitations, non-deterministic problems, and model variability."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 includes study-specific threats: test assumptions may not hold for non-deterministic problems (they excluded such cases), the drop in baseline performance vs. leaderboards due to removed sample tests, and that EvalPlus test quality was used to address known benchmark weaknesses."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper notes benchmarks 'do not capture the full complexity' of real-world problems (Section 7), it does not explicitly state what the results do NOT show — e.g., no statement that results don't generalize to repository-level tasks, non-Python languages, or models beyond GPT-4/Llama 3."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at OSF includes 'output and runtime details from TGen for all cases we experimented with' (Section 3.2), providing raw experimental outputs for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "MBPP and HumanEval datasets are described with their EvalPlus variants (Section 3.1). CodeChef collection is described: 100 most popular problems per difficulty level, popularity defined as accepted solutions count, scraped November 2023 (Section 5.1.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (MBPP, HumanEval) and a curated dataset from a public platform (CodeChef) with selection criteria described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in Section 3.2: problem statements and tests as input → LLM generation → PyTest verification → optional remediation loop. The sequential evaluation process (solved without tests → needs tests → needs remediation → unsolved) is clearly described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. No acknowledgments section mentioning grants or sponsors is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are affiliated with University of Waterloo, Canada, clearly listed in the paper header. They evaluate GPT-4 (OpenAI) and Llama 3 (Meta) with no affiliation to either company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed. The paper evaluates commercial products (GPT-4, Llama 3) but there is no evidence of industry funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Neither GPT-4 Turbo's nor Llama 3's training data cutoff dates are stated. This is relevant because MBPP (2021) and HumanEval (2021) predate both models' training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MBPP or HumanEval problems appeared in GPT-4's or Llama 3's training data, despite both benchmarks being widely available online since 2021."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP and HumanEval were published in 2021, well before GPT-4 and Llama 3 training. The paper uses these benchmarks without any discussion of contamination risk, which could inflate baseline performance and reduce the measured improvement from TDD."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve automated LLM evaluation on coding benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM code generation on programming benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or cost per example are reported despite extensive GPT-4 API usage across hundreds of problems with multiple iterations each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or wall-clock time is reported for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "A single seed (1106) with temperature 0 is used for all experiments. No sensitivity analysis across multiple seeds is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state the number of experimental runs per problem. The use of fixed seed and temperature 0 implies single runs but this is not explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Temperature (0), seed (1106), max iterations (5), and repeated failure threshold (3) appear to be chosen without reported search or justification."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The configuration choices (temperature 0, seed 1106, 5 iteration max, 3 repeated failure limit) are stated but not justified with systematic comparison. The remediation limits are partially justified by observing 'diminishing returns' but no formal analysis is provided."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own TGen framework without acknowledging potential author-evaluation bias. No independent evaluation or discussion of this bias is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "TGen with remediation makes multiple LLM calls per problem (up to 10+ round trips for 5 remediation iterations) compared to a single call for the baseline, yet this compute difference is never quantified or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses that 'existing work in the literature has questioned the accuracy of function-level benchmarks for evaluating LLMs' and the paper addresses this by also using CodeChef for file-level generation and noting benchmarks 'do not capture the full complexity and variability of real-world programming problems' (Section 7)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The TGen scaffold is held constant across conditions (the variable is test inclusion, not scaffolding). When comparing GPT-4 vs Llama 3, the same TGen pipeline is used for both, controlling the scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP (2021) and HumanEval (2021) were published years before GPT-4 and Llama 3 were trained. No discussion of whether these benchmarks and their solutions were in the training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether providing test cases as input leaks structural information about the expected solution beyond what would be available in a realistic TDD scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MBPP and HumanEval problems (or solutions similar to them) appear in the training corpora of GPT-4 or Llama 3."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is employed (no canary strings, membership inference, temporal splits, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "Providing test cases improves LLM code generation correctness by 9.15–29.57% across benchmarks after private test validation.",
       "evidence": "Table 1 shows MBPP: +12.78%, HumanEval: +9.15%, CodeChef: +3.09% for GPT-4. Table 3 shows MBPP: +29.57%, HumanEval: +13.41% for Llama 3. All validated against private/held-out tests.",
       "supported": "moderate"
     },
     {
       "claim": "Remediation loops that feed test failures back to the LLM provide additional improvement of 5.26–9.02%.",
       "evidence": "Table 1 shows remediation adds +5.26% (MBPP), +5.49% (HumanEval), +4.18% (CodeChef) for GPT-4. Table 3 shows +9.02% (MBPP), +8.54% (HumanEval) for Llama 3.",
       "supported": "moderate"
     },
     {
       "claim": "Less performant models benefit more from TDD — Llama 3 shows almost double the improvement of GPT-4.",
       "evidence": "Section 5.3: Llama 3 total improvement 38.60% (MBPP) and 21.95% (HumanEval) vs GPT-4's 18.04% and 14.64% respectively (Tables 1 and 3).",
       "supported": "moderate"
     },
     {
       "claim": "Problem difficulty inversely correlates with TDD effectiveness — simpler problems benefit more from tests while harder problems remain unsolved.",
       "evidence": "Figure 5 and Figure 6 show success rates sharply decrease with difficulty. Section 5.1.2 notes 47.18% of CodeChef problems remain unsolved, concentrated in harder difficulty levels.",
       "supported": "moderate"
     },
     {
       "claim": "Solutions generated with test information are robust against unseen private tests, not just tailored to supplied tests.",
       "evidence": "Table 1 shows improvements persist after evaluation with EvalPlus private tests (35x–80x more tests). CodeChef solutions also validated against the platform's private test suites.",
       "supported": "moderate"
     },
     {
       "claim": "Remediation advice becomes repetitive beyond 3–4 iterations with diminishing returns.",
       "evidence": "Section 4.3 reports most problems solved in first 1–2 remediation attempts (MBPP: 11 of 21 on first attempt; HumanEval: 7 of 9 on first attempt). No problems were solved at the 5th attempt despite attempts utilizing all 5 iterations.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination discussion",
       "detail": "MBPP and HumanEval (both 2021) were almost certainly in GPT-4's training data. If the model has memorized solutions, the measured 'baseline' is inflated and the relative improvement from TDD may be distorted. This fundamental confound is never addressed."
     },
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "All comparisons use raw percentage differences with no significance tests, confidence intervals, or error bars. A 3.09% improvement on CodeChef (34 problems out of 1100) could easily be noise, but no statistical test confirms it is meaningful."
     },
     {
       "flag": "Single-run experiments",
       "detail": "Despite using temperature 0 and fixed seed for reproducibility, there is no exploration of sensitivity to seed choice. LLM outputs can still vary across API calls even with fixed parameters, and the results may not be stable."
     },
     {
       "flag": "Compute cost of TDD approach unquantified",
       "detail": "The remediation loop makes up to 10+ LLM API calls per problem (coder + remediation agent alternating for 5 iterations), but the paper never quantifies this cost. The 5.26% improvement from remediation could require 5x the compute of the baseline, making it impractical."
     }
   ],
   "cited_papers": [
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "EvalPlus benchmark used as primary evaluation framework; addresses insufficiency of existing code generation benchmarks."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex; foundational work on LLM code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used as primary evaluation dataset in this study."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Real-world code generation benchmark using GitHub issues; contrasts with function-level benchmarks used here."
     },
     {
       "title": "Reflexion: an autonomous agent with dynamic memory and self-reflection",
       "authors": ["Noah Shinn", "Beck Labash", "Ashwin Gopinath"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Self-improvement through iterative reflection in code agents; related to the remediation loop approach."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Agentic system using test cases for spectrum-based fault localization in code generation."
     },
     {
       "title": "Codet: Code generation with generated tests",
       "authors": ["Bei Chen", "Fengji Zhang", "Anh Nguyen"],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "Uses generated test cases to improve code generation quality; directly related methodology."
     },
     {
       "title": "LLM-based Test-driven Interactive Code Generation: User Study and Empirical Evaluation",
       "authors": ["Sarah Fakhoury", "Aaditya Naik", "Georgios Sakkas"],
       "year": 2024,
       "arxiv_id": "2404.10100",
       "relevance": "Explores LLM-generated tests with user feedback for code generation; closely related TDD approach."
     },
     {
       "title": "MultiPL-E: a scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "relevance": "Polyglot code generation benchmark; raises concerns about accuracy of function-level benchmarks for LLM evaluation."
     },
     {
       "title": "Quality and Trust in LLM-generated Code",
       "authors": ["Claudio Spiess", "David Gros", "Kunal Suresh Pai"],
       "year": 2024,
       "arxiv_id": "2402.02047",
       "relevance": "Addresses trust and correctness issues in LLM-generated code; motivates need for verification mechanisms."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "AlphaCode system for competitive programming; CodeContests benchmark contrasted with CodeChef dataset used here."
     },
     {
       "title": "Interactive code generation via test-driven user-intent formalization",
       "authors": ["Shuvendu K Lahiri", "Aaditya Naik", "Georgios Sakkas"],
       "year": 2022,
       "arxiv_id": "2208.05950",
       "relevance": "Explores test cases for formalizing user intent in code generation; foundational TDD-for-LLM work."
     }
   ]
 }