ai-research-survey

scan.json (26523B)

---

 {
   "paper": {
     "title": "TestGenEval: A Real World Unit Test Generation and Test Completion Benchmark",
     "authors": ["Kush Jain", "Gabriel Synnaeve", "Baptiste Rozière"],
     "year": 2024,
     "venue": "ICLR 2025",
     "arxiv_id": "2410.00752",
     "doi": "10.48550/arXiv.2410.00752"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TestGenEval is a large-scale benchmark of 68,647 tests from 1,210 code-test file pairs across 11 Python repositories for evaluating LLM test generation and completion. The best model (GPT-4o) achieves only 35.2% coverage and 18.8% mutation score on full test suite generation, showing models struggle with real-world test generation. Test completion is easier than full generation, but models add virtually no new coverage when completing the last test in an existing suite. Models frequently make assertion errors and struggle to reason about code execution.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides code and docker images at https://figshare.com/s/51171ae97cd21d233d4f, with detailed instructions on running and extending the benchmark (Section 9, Reproducibility Statement)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (1,210 code-test file pairs) is released via the figshare link, along with docker images for all 11 repositories and model generations."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Docker images are provided for each repository version with all dependencies installed, including coverage and mutation testing tools. This exceeds a requirements.txt."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 states 'We release code to run and extend TESTGENEVAL along with all docker images' and 'detailed instructions on how to run our benchmark, and even extend it.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G reports 95% confidence intervals for all settings (Table 12, Table 17) and performs pairwise statistical comparisons."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G is dedicated to statistical tests, including pairwise comparisons and Elo ratings for model ranking across all settings."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with absolute percentages and differences between models (e.g., GPT-4o at 35.2% coverage vs Llama 405B at 35.0%), along with win rates and Elo scores providing magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 1,210 file pairs was the target size, nor power analysis. The benchmark size is determined by what was extractable from SWEBench rather than by statistical power considerations."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 2-3 are single-run at temperature=0.2. While pass@5 uses temperature=0.8 with 5 samples, no standard deviation or variance across independent runs is reported for the main results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 10 models across 4 size categories (small, medium, large, flagship), serving as baselines for each other. Table 1 compares TestGenEval against prior benchmarks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o, Llama 3.1 (8B/70B/405B), Codestral 22B, DeepSeekCoder V2, and Gemma 2 — all contemporary at time of writing (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates context window size (Section 4.1.3, Figure 8), number of samples (Section 4.1.2, Figure 7), and provides TestGenEvalLite as a representative subset (Appendix E.2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1, pass@5, all pass@1, any pass@1, coverage, coverage@pass, mutation score, mutation score@pass, coverage improvement, and Elo ratings."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes qualitative analysis with manual examination of model outputs (Section 4.2, Appendix F.6-F.8), including detailed examples of correct and incorrect test generation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "TestGenEvalLite (160 pairs) is a separate representative subset from the full TestGenEval (1,210 pairs), and results are reported on both. The benchmark itself uses execution-based evaluation with no tuning on the test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size category (small/medium/large/flagship), by task type (generation vs completion), by completion setting (first/last/extra), and by repository (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive error analysis in Appendix F.5 and F.7 categorizes failures (assertion errors, no-assert errors, timeout errors, value errors) with qualitative examples. Section 4.2 discusses specific failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative findings: models add virtually 0% coverage in last/extra test completion; context window doesn't help much for test generation (Figure 8a); CodeLlama 70B underperforms smaller models. Appendix F.8 shows 'no solve' cases."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (GPT-4o at 35.2% coverage, models struggle with high-coverage suites, assertion errors on complex paths) are all directly supported by Tables 2-3 and the error analysis in Section 4 and Appendix F."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is primarily descriptive (benchmark + evaluation). Causal-style claims are modest and supported by controlled ablations: e.g., context window effects (Section 4.1.3) use the same model with varying context sizes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark covers only Python across 11 repositories, but the title and framing suggest general 'unit test generation' capabilities. The Limitations section (Section 5, Appendix H) notes SWEBench overfitting risk but doesn't strongly bound claims to Python or these specific project types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Appendix H discuss data contamination, SWEBench overfitting, prompt/temperature sensitivity, and file-level context limitations as alternative explanations for observed results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses that coverage and mutation score are proxies for test quality (Appendix H: 'Quantitative metrics such as code coverage and mutation score can approximate the quality of generated tests, however they do not perfectly measure the quality'). They also discuss the oracle problem."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides exact HuggingFace model URLs for all open-source models. GPT-4o is dated to 08/29/2024. Specific model names like 'Meta-Llama-3.1-8B-Instruct' are given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for both test generation (Figure 15) and test completion (Figure 16) is provided in Appendix C."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is specified: 0.2 for pass@1, 0.8 for pass@5 (Tables 2-3). Context window handling is described (maximum possible, truncate starting tokens)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly to generate test code."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 and Figure 2 describe the full pipeline: modifying SWEBench images, extracting test file pairs via heuristic matching, filtering by coverage computation and 60-second timeout."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations' with substantive discussion. Appendix H provides an expanded 'Limitations' section with additional detail."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: overfitting to SWEBench repositories, data contamination (with perplexity measurements), compute cost of mutation score, prompt/temperature sensitivity (Appendix H), file-level context limitation, lack of agentic baselines."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H states specific boundaries: file-level only (not repository-level), Python only, no agentic baselines, 0-shot only, and acknowledges cross-file dependencies are not captured."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All benchmark data, docker images, model generations, and evaluation code are released via figshare. A website with all model generations is also provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the full benchmark construction process: starting from SWEBench docker images, modifying broken images, extracting file pairs via heuristic matching, filtering criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is SWEBench, a standard benchmark derived from public GitHub repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows the full pipeline with three stages. Table 4 shows the distribution across repositories (1,210 pairs from 11 repos). Filtering criteria are explicit (coverage computes, runs in 60 seconds)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or grant information is disclosed. Authors are affiliated with FAIR/Meta AI and CMU but no funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Carnegie Mellon University and FAIR, Meta AI. Meta develops Llama and CodeLlama models which are evaluated in the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are from Meta AI (FAIR), which develops several of the evaluated models (Llama 3.1, CodeLlama). Meta has a financial interest in their models performing well on benchmarks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. The paper uses GPT-4o dated to 08/29/2024 (evaluation date) but not its training cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses data contamination: 'we measure perplexity of 10 randomly selected tests in TESTGENEVAL for Llama 3.1 8B and common frequent, and non recent code from GitHub' finding perplexity of common code lower than TestGenEval tests (1.6 vs 2.0)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 addresses contamination risk directly, arguing the benchmark is unlikely contaminated based on perplexity analysis and universally low model performance. Also notes SWEBench overfitting risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example costs are reported despite evaluating 10 models across thousands of examples at multiple temperatures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. The paper mentions mutation score compute cost as a limitation but does not quantify the total compute used for the evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported at fixed temperatures (0.2 and 0.8) with no seed sensitivity analysis. Main results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "For pass@k metrics: 1 generation for pass@1 at temp=0.2, 5 generations for pass@5 at temp=0.8. Section F.3 reports up to 20 samples for coverage@k and 100 for pass@k analysis."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The paper uses fixed temperatures (0.2, 0.8) and maximum context windows without discussing whether these were tuned or how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Temperature values of 0.2 and 0.8 are used without justification for why these specific values were chosen over alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Appendix G reports pairwise comparisons across 10 models and 4 settings but no multiple comparison correction (Bonferroni, etc.) is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Meta AI authors evaluate Meta's own Llama and CodeLlama models alongside competitors. No acknowledgment of potential bias in implementation or evaluation setup favoring their own models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes (7B to 405B) are compared without discussing compute costs. GPT-4o's compute budget is unknown but likely much larger than smaller open-source models."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 and Table 1 explicitly discuss how TestGenEval's properties (file-level, human-written tests, mutation score) relate to real-world test generation. The paper argues mutation score is more correlated with real fault detection than coverage alone, citing Just et al. (2014)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are evaluated via direct prompting with the same prompt templates."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 addresses temporal leakage through perplexity analysis comparing TestGenEval tests against common GitHub code, finding TestGenEval has higher perplexity (2.0 vs 1.6), suggesting tests are not memorized."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context. The prompts provide imports and code under test, but whether this constitutes unrealistic information leakage compared to real-world usage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 11 SWEBench repositories overlap with model training data at the repository level, or whether file pairs from the same repository are non-independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "A concrete leakage detection method is used: perplexity measurement on 10 randomly selected tests compared against common GitHub code for Llama 3.1 8B (Section 5)."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4o achieves the best coverage (35.2%) and mutation score (18.8%) among all evaluated models on full test suite generation.",
       "evidence": "Table 2 (Section 3.1) shows GPT-4o at 35.2% coverage and 18.8% mutation score, compared to Llama 3.1 405B at 35.0%/16.4%.",
       "supported": "strong"
     },
     {
       "claim": "Models struggle to generate high-coverage test suites for real-world projects, with significantly lower performance than on self-contained benchmarks like TestEval.",
       "evidence": "Table 2 shows max 35.2% coverage vs near 100% on TestEval (Section 1). Weak positive correlation with TestEval (Section 4.1.1, Figure 6b).",
       "supported": "strong"
     },
     {
       "claim": "Test completion is significantly easier than full test generation, but models add virtually no new coverage when completing the last test in an existing suite.",
       "evidence": "Table 3 shows pass@5 up to 74.3% for last test completion, but coverage improvement is near 0% (+Cov column). Table 8 confirms this across all models.",
       "supported": "strong"
     },
     {
       "claim": "Models primarily struggle with reasoning about execution, leading to frequent assertion errors.",
       "evidence": "Appendix F.5 (Figure 31) shows assertion errors are the most common error type for Codestral 22B. Qualitative analysis in Section 4.2 and F.7 provides specific examples.",
       "supported": "strong"
     },
     {
       "claim": "TestGenEvalLite (160 pairs) approximates the full benchmark metrics.",
       "evidence": "Appendix E.2 shows similar trends in TestGenEvalLite vs TestGenEval. Appendix G reports statistical significance for both splits.",
       "supported": "moderate"
     },
     {
       "claim": "Data contamination is unlikely to be a major issue for TestGenEval.",
       "evidence": "Perplexity analysis on 10 randomly selected tests shows higher perplexity than common GitHub code (2.0 vs 1.6) for Llama 3.1 8B (Section 5).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own models",
       "detail": "Meta AI (FAIR) authors evaluate Meta's Llama and CodeLlama models. While GPT-4o outperforms them, the benchmark design choices (Python only, SWEBench-derived, specific prompt format) could inadvertently favor certain model families."
     },
     {
       "flag": "Weak contamination analysis",
       "detail": "Data contamination analysis uses only 10 randomly selected tests with perplexity for a single model (Llama 3.1 8B). This is a very small sample and a single detection method for a benchmark derived from well-known public repositories."
     },
     {
       "flag": "No compute costs reported",
       "detail": "Evaluating 10 models across 1,210 examples with mutation testing is computationally expensive, yet no costs are reported. This limits reproducibility assessment."
     },
     {
       "flag": "Single-run main results",
       "detail": "Main results at temperature=0.2 appear to be single-run without variance estimates. The statistical tests in Appendix G partially address this but main tables lack uncertainty."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the foundational code generation benchmark that TestGenEval compares against."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "TestGenEval is built on SWEBench's infrastructure and docker images, making this a direct methodological dependency."
     },
     {
       "title": "TestEval: Benchmarking large language models for test case generation",
       "authors": ["Wenhan Wang", "Chenyuan Yang"],
       "year": 2024,
       "arxiv_id": "2406.04531",
       "relevance": "Existing test generation benchmark on LeetCode problems that TestGenEval positions against as being too simple/self-contained."
     },
     {
       "title": "Code agents are state of the art software testers",
       "authors": ["Niels Mündler", "Mark Niklas Müller"],
       "year": 2024,
       "arxiv_id": "2406.12952",
       "relevance": "SWT-Bench: adjacent benchmark for test method generation targeted at bug-fixing PRs."
     },
     {
       "title": "R2e: Turning any github repository into a programming agent environment",
       "authors": ["Naman Jain", "Manish Shetty"],
       "year": 2024,
       "relevance": "Repository-level executable environment for evaluating coding agents, uses equivalence test harnesses."
     },
     {
       "title": "CruxEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Rozière"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Executable code benchmark measuring execution reasoning ability, a subset of the test generation task."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Source of Llama 3.1 models (8B/70B/405B) evaluated as primary open-source baselines."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the barrier of closed-source models in code intelligence",
       "authors": ["DeepSeek-AI"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "Code-specialized model evaluated as a medium-sized baseline in the benchmark."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Contemporary code generation benchmark with execution-based evaluation."
     },
     {
       "title": "CAT-LM training language models on aligned code and tests",
       "authors": ["N. Rao", "K. Jain"],
       "year": 2023,
       "relevance": "Software testing model that measures test completion at method level, directly related to TestGenEval's test completion task."
     },
     {
       "title": "TOGA: A neural method for test oracle generation",
       "authors": ["Elizabeth Dinella", "Gabriel Ryan"],
       "year": 2022,
       "relevance": "Neural test oracle generation benchmark focused on assertion completion."
     },
     {
       "title": "RepoBench: Benchmarking repository-level code auto-completion systems",
       "authors": ["Tianyang Liu", "Canwen Xu"],
       "year": 2023,
       "arxiv_id": "2306.03091",
       "relevance": "Repository-level code completion benchmark relevant to understanding context effects on code generation."
     }
   ]
 }