ai-research-survey

scan.json (30367B)

---

 {
   "paper": {
     "title": "ProjectTest: A Project-level LLM Unit Test Generation Benchmark and Impact of Error Fixing Mechanisms",
     "authors": [
       "Yibo Wang",
       "Congying Xia",
       "Wenting Zhao",
       "Jiangshu Du",
       "Chunyu Miao",
       "Zhongfen Deng",
       "Philip S. Yu",
       "Chen Xing"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.06556",
     "doi": "10.48550/arXiv.2502.06556"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ProjectTest introduces a project-level unit test generation benchmark covering 60 projects across Python, Java, and JavaScript, evaluated on 9 frontier LLMs. Java is the hardest language for all models due to strict syntax, with compilation rates as low as 0% for some open-source models. Manual fixing of compilation and cascade errors produces dramatic improvements (e.g., CodeQwen1.5 goes from 0% to 60% correctness on Java), revealing that basic errors mask LLMs' underlying test generation capabilities. LLM self-fixing generally lags behind manual fixing, with open-source models particularly struggling to self-correct.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code and dataset is available at ProjectTest' in the abstract, referencing a public release. The linked resource appears to be a repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code and dataset is available at ProjectTest' in the abstract. The 60 projects and their metadata are described as publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8 NVIDIA A100 GPUs' and names testing frameworks (Pytest, Jacoco, JEST) but provides no requirements.txt, Dockerfile, or detailed dependency/version listing for reproducing the evaluation environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The methodology is described at a high level (Section 3) but lacks specific commands or scripts to replicate the evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, and 4 are reported as point estimates (e.g., '47%', '65%') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'GPT-o1 performs the best in general', 'Java is the most difficult language') based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 report improvements with both the new value and the delta, e.g., '74% (+27%)' and '43% (-16%)', providing baseline context for interpreting the magnitude of changes from vanilla to manual-fixed and self-fixed scenarios."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 20 projects per language (60 total) is not justified. No power analysis or sample size rationale is provided. The paper does not discuss whether 20 projects is sufficient for the comparative claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Temperature is set to 0 for deterministic inference, but no discussion of API non-determinism or variance across runs. All results are single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nine LLMs are compared against each other, including 5 closed-source and 4 open-source models. Three evaluation scenarios (vanilla, manual fixing, self-fixing) serve as comparisons against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes contemporary models: GPT-o1, Claude-3.5-Sonnet (2024-10-22), and Gemini-2.0-Flash-Exp. Open-source models include CodeQwen1.5, DeepSeek-Coder, CodeLlama, and CodeGemma, which are somewhat older but still relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 (Table 11) provides a prompt ablation study removing individual prompt components (PL, CR, ComR, Coverage) and testing their impact on GPT-4-Turbo's performance. Appendix D.2 ablates compilation vs. cascade error fixing."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used: Compilation Rate (ComR), Correctness Rate (CR), Line Coverage (LC), and Branch Coverage (BC). Additionally, unique contribution analysis is performed in Section 5.4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated unit tests' quality is performed. The manual fixing scenario involves humans correcting errors but not evaluating test quality, readability, or meaningfulness."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit dev/test split exists. The prompts were 'carefully designed' (Section 3.3) and the ablation study (Table 11) was conducted on the same 20 Python projects used for main evaluation. There is no separation between prompt development and final evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (Python, Java, JavaScript), by model (9 models), and by evaluation scenario (vanilla, manual-fixed, self-fixed). Tables 2-4 provide comprehensive per-language, per-model breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 and Appendix E provide extensive error analyses covering compilation errors, cascade errors, and post-fix errors per language, with specific examples (Figures 4, 5, 10-12)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: LLM self-fixing often degrades performance compared to manual fixing (Table 4), coverage-related prompts are 'not always beneficial' (Table 11), and open-source models completely fail on Java (0% compilation for CodeQwen1.5, CodeLlama, CodeGemma)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about moderate LLM performance, benchmark difficulty, significant compilation errors in frontier LLMs, and self-fixing lagging behind manual fixing are all supported by Tables 2-4 and the error analyses in Section 5.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims without adequate justification. For example, 'Java is the most difficult language, primarily due to stricter syntax' (Section 5.1) attributes difficulty to a specific cause without controlling for confounds like project complexity or model training data composition."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested setting. The paper refers to 'all tested frontier LLMs' rather than all LLMs, and findings are specific to ProjectTest. The limitations section acknowledges the restriction to three languages and moderate-sized projects."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, performance differences across languages could be explained by training data composition rather than syntax strictness, but this is not considered. No robustness checks are performed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It measures compilation rate, correctness rate, and coverage rate and frames results in those terms without inflating claims to broader constructs like 'code quality' or 'developer productivity'."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Claude-3.5-Sonnet-20241022 is fully specified. Open-source models have exact HuggingFace names. However, GPT-4-Turbo, GPT-3.5-Turbo, and GPT-o1 lack snapshot dates or API version identifiers. Gemini-2.0-Flash-Exp is partially specified. Multiple key models lack reproducible version information."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided for all three languages in Figures 3, 7, 8 (vanilla generation), Figure 6 (self-fixing), and Figure 9 (comment-sign variant). The actual text sent to the models is given, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'The temperature is set to 0 during inference.' Maximum input length is configured per model. Hardware (8 NVIDIA A100 GPUs) is specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation uses direct zero-shot prompting of LLMs without any agent, tool use, or multi-step pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 (Pre-processing) documents the preprocessing steps: syntax checking, making extracted projects self-contained, reorganizing files, adjusting import paths, consolidating multi-line statements, and preserving original code style."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the Conclusion, discussing language coverage restrictions and project size constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations mention specific threats: focusing on only three programming languages (missing C, C#), selecting moderate-sized projects due to LLM input length restrictions, which limits exploration to robustness issues rather than long-context handling."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what is not tested: other programming languages, larger projects, and long-context handling. It bounds the focus to 'issues like the robustness of LLMs in unit test generation (e.g., hallucinations or incorrect assertions) rather than focusing solely on their ability to handle long-context inputs.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and dataset are available at ProjectTest. The raw projects and generated unit tests should be accessible for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data collection procedure in detail: projects collected from GitHub, filtered by size (2-15 files, <1600 LOC), inter-file dependencies, public licenses, and high star/fork counts. Tables 6-8 list every project with stars, forks, license, and links."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The project selection method is described: GitHub repositories filtered by size, dependencies, license type, and community metrics (stars, forks). Some projects were extracted from larger codebases and adjusted to be self-contained. Appendix A provides per-project details."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from collection to evaluation is documented: GitHub filtering → preprocessing (syntax checks, self-containment adjustments, multi-line consolidation) → prompt construction → LLM evaluation → metric computation. Section 3 covers each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed in the paper. There is no acknowledgments section listing grants or sponsors, despite authors being affiliated with a university (UIC), Salesforce Research, and Scale AI."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Illinois Chicago, Salesforce Research, and Scale AI. These are prominently displayed in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making independence impossible to assess. Authors at Salesforce Research and Scale AI (companies with interests in LLM capabilities and evaluation) could have interests in the outcomes, but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Authors at Scale AI (an AI evaluation company) and Salesforce Research have potential interests related to the findings but these are not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 9 evaluated models. This is critical because the benchmark projects are from popular GitHub repositories that are likely in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark projects (many from repos with 2,900-103,000 stars) appeared in any model's training data. This is a significant omission given the high visibility of these repositories."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark projects come from highly popular GitHub repos (e.g., three.js with 103,000 stars, keras preprocessing with 1,024 stars). These were certainly available before training cutoffs of the tested models. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using LLM-generated unit tests."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on code generation tasks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The manual fixing was performed by the authors but demographics are not relevant as this is not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or randomized experimental conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, tokens consumed, or latency is reported for any of the 9 models evaluated. The paper runs hundreds of LLM evaluations without mentioning cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only hardware is mentioned ('8 NVIDIA A100 GPUs' in Section 4.2). No total GPU hours, API spend, or wall-clock time for the experiments is provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is performed. Temperature is set to 0 for deterministic inference, but API non-determinism is not addressed and no multi-run analysis is provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. The use of temperature 0 implies single runs, but this is not confirmed. The paper presents results without stating how many times each experiment was conducted."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The prompts were 'carefully designed' but no information is given about how many prompt variants were tried before selecting the final prompts."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 'Full Prompt' configuration is used as the default but selection is not justified on a separate validation set. The prompt ablation (Table 11) is conducted on the same test data, making it unclear whether the prompt was selected based on test set performance."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite many comparisons across 9 models, 3 languages, 3 scenarios, and 4 metrics."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge potential bias in benchmark construction. The selection of projects and design of prompts could favor certain models, but this is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-o1 (a reasoning model with higher compute) is compared directly to smaller models like CodeGemma-7B without any discussion of compute budget differences. Performance is not contextualized against compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether compilation rate, correctness rate, and coverage rate on 20 moderate-sized projects adequately capture 'project-level unit test generation capability'. No comparison with alternative evaluation approaches."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are evaluated with direct zero-shot prompting in the same pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The benchmark projects are from well-known GitHub repositories (some with 100k+ stars) that certainly existed before the training cutoffs of all tested models. LLMs may have seen these projects and their existing unit tests during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation provides the full project source code as input, but if models memorized these projects during training, they could reproduce or approximate known test suites rather than generating novel tests."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Multiple projects come from the same repositories: 8 Python card game projects from the same repo (2,937 stars), 4 stock-related Python projects from the same book repo (10,700 stars). This creates non-independence between test instances."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, temporal splits, or decontamination of any kind."
       }
     }
   },
   "claims": [
     {
       "claim": "All frontier LLMs exhibit moderate performance on ProjectTest on Python and Java, highlighting its difficulty.",
       "evidence": "Table 2 shows correctness rates of 13-64% for Python and 0-53% for Java across 9 models. Compilation rates range from 50-70% for Python and 0-75% for Java.",
       "supported": "strong"
     },
     {
       "claim": "Java is the most difficult language for LLM unit test generation, primarily due to stricter syntax.",
       "evidence": "Table 2 shows consistently lower scores for Java across all metrics and models. Three open-source models (CodeQwen1.5, CodeLlama, CodeGemma) achieve 0% on all Java metrics.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-o1 performs the best in general, especially in JavaScript.",
       "evidence": "Table 2 shows GPT-o1 achieves 87% CR and 87% LC on JavaScript, far ahead of other models. It leads or is near the top across most language-metric combinations.",
       "supported": "strong"
     },
     {
       "claim": "Manual fixing of compilation and cascade errors reveals significant improvement potential for LLM-generated unit tests.",
       "evidence": "Table 3 shows all models reach 100% compilation rate after fixing. Improvements are dramatic: CodeQwen1.5 goes from 0% to 60% CR on Java (+60%), Gemini-2.0-Flash improves from 14% to 54% LC on Java (+40%).",
       "supported": "strong"
     },
     {
       "claim": "LLM self-fixing capabilities lag behind manual fixing in quality and reliability.",
       "evidence": "Table 4 shows self-fixing performs worse than manual fixing across almost all models and languages. For example, GPT-o1 Python CR drops from 89% (manual) to 67% (self-fix), and CodeLlama Python drops from 31% to 0%.",
       "supported": "strong"
     },
     {
       "claim": "Different LLMs have different language-level expertise.",
       "evidence": "Table 2: Claude-3.5-Sonnet ranks first in Java (53% CR, 75% ComR) while GPT-o1 ranks first in JavaScript (87% CR, 95% ComR). In Python, Claude-3.5-Sonnet leads on CR (64%) while GPT-o1 leads on coverage (56% LC).",
       "supported": "strong"
     },
     {
       "claim": "LLM-generated unit tests have low unique contribution rates, indicating redundant test generation.",
       "evidence": "Table 5 shows unique contributions ranging from 2.70% (CodeGemma) to 11.40% (Claude-3.5-Sonnet) on Python, indicating high redundancy across all models.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Severe contamination risk from popular repositories",
       "detail": "Benchmark projects come from highly popular GitHub repositories (e.g., three.js with 103,000 stars, keras preprocessing with 1,024 stars, svm with 10,800 stars). All tested LLMs were almost certainly trained on these codebases, likely including their existing unit tests. This contamination risk is never discussed, and could mean models are partially recalling memorized tests rather than generating novel ones."
     },
     {
       "flag": "Non-independent test instances from shared repositories",
       "detail": "Multiple projects are extracted from the same repositories: 8 Python card game projects from the same RLCard repo (2,937 stars), 4 stock-related projects from the same book repo (10,700 stars), and multiple JavaScript projects from three.js (103,000 stars). Performance on these projects is likely correlated, inflating the effective sample size of 20 projects per language."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes numerous ranking and comparison claims across 9 models, 3 languages, and 3 scenarios, all based on comparing point estimates from 20 projects without any statistical significance testing. With such small sample sizes, observed differences could easily be due to chance."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or variance measures. Even with temperature 0, API non-determinism exists, and the small sample (20 projects) means results are sensitive to the specific project selection."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the HumanEval benchmark for evaluating LLM code generation, a foundational benchmark in the space."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "relevance": "Class-level code generation benchmark that ProjectTest extends to project level."
     },
     {
       "title": "TestEval: Benchmarking large language models for test case generation",
       "authors": ["Wenhan Wang", "Chenyuan Yang", "Zhijie Wang"],
       "year": 2024,
       "relevance": "Function-level test generation benchmark directly related to evaluating LLM testing capabilities."
     },
     {
       "title": "DevBench: A comprehensive benchmark for software development",
       "authors": ["Bowen Li", "Wenhan Wu", "Ziwei Tang"],
       "year": 2024,
       "arxiv_id": "2403.08604",
       "relevance": "Software development benchmark that includes project-level unit testing but with limited scope, which ProjectTest aims to address."
     },
     {
       "title": "TestGenEval: A real world unit test generation and test completion benchmark",
       "authors": ["Kush Jain", "Gabriel Synnaeve", "Baptiste Rozière"],
       "year": 2024,
       "arxiv_id": "2410.00752",
       "relevance": "Real-world unit test generation benchmark for evaluating LLM capabilities."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2023,
       "relevance": "Empirical evaluation of LLMs for unit test generation using iterative querying, directly related to the survey scope."
     },
     {
       "title": "Using large language models to generate junit tests: An empirical study",
       "authors": ["Mohammed Latif Siddiq"],
       "year": 2024,
       "relevance": "Empirical study of LLM JUnit test generation with zero-shot strategies, directly comparable to ProjectTest's evaluation approach."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the model families evaluated in this benchmark."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM evaluated in the benchmark, relevant to understanding code generation model capabilities."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "relevance": "Open-source code LLM that outperforms some closed-source models on ProjectTest, relevant to code intelligence evaluation."
     },
     {
       "title": "SWT-Bench: Testing and validating real-world bug-fixes with code agents",
       "authors": ["Niels Mündler", "Mark Niklas Mueller", "Jingxuan He", "Martin Vechev"],
       "year": 2024,
       "relevance": "Software testing benchmark using code agents, related to evaluating LLM capabilities in software engineering tasks."
     },
     {
       "title": "R2E: Turning any github repository into a programming agent environment",
       "authors": ["Naman Jain", "Manish Shetty", "Tianjun Zhang"],
       "year": 2024,
       "relevance": "Framework for repository-level evaluation of programming agents, related to project-level code evaluation."
     }
   ]
 }