ai-research-survey

scan.json (29589B)

---

 {
   "paper": {
     "title": "Evaluating Diverse Large Language Models for Automatic and General Bug Reproduction",
     "authors": [
       "Sungmin Kang",
       "Juyeon Yoon",
       "Nargiz Askarbekkyzy",
       "Shin Yoo"
     ],
     "year": 2023,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2311.04532",
     "doi": "10.1109/TSE.2024.3450837"
   },
   "scan_version": 2,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors provide both the tool (https://github.com/coinse/libro) and the replication package for the journal extension (https://github.com/coinse/libro-journal-artifact), mentioned in Sections 1 and 4.3."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Defects4J is a public benchmark. The GHRB dataset is described and the experimental data and analysis scripts are publicly available (Section 1). The replication package includes experimental data."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 specifies: Ubuntu 18.04.6 LTS with 32GB RAM and Intel i7-7700 CPU for test execution; Ubuntu 20.04.6 LTS with 16 Xeon Gold 5222 CPUs and 4 NVIDIA RTX 3090 GPUs (96GB VRAM) for LLM inference. Python 3.9 with javalang library are stated."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated replication package is provided at https://github.com/coinse/libro-journal-artifact with experimental data and analysis scripts. The tool repository is also public."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 reports 50%, 80%, and 95% intervals from 1000-run simulations of generation attempts to performance. Table 4 reports 5th percentile, median, and 95th percentile for the two-example n=10 setting sampled from n=50 results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparisons between LLMs (Figure 6) and settings (Table 4) are based on raw number comparisons without p-values, t-tests, or any formal hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative performance ratios (StarCoder at 70% of Codex on Defects4J, 90% on GHRB), absolute counts (251/750 = 33.5%), and percentage comparisons consistently throughout Section 6."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 750 Defects4J bugs or 31 GHRB bugs are sufficient sample sizes. No power analysis is discussed. The GHRB dataset is particularly small at 31 bugs."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main LLM comparison results (Figure 6, Table 5) are single-run point estimates without variance measures. While Figure 3 and Table 4 show intervals for subsampling simulations, the core comparative results across models lack standard deviations or spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1.2 compares LIBRO against EvoCrash (state-of-the-art crash reproduction) and a Copy&Paste baseline that extracts code snippets from bug reports (Figure 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "EvoCrash was the state-of-the-art crash reproduction technique at the time. The authors note there are no existing general bug reproduction techniques, making EvoCrash the most relevant comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents systematic ablation of prompt components: no example, one example, two examples, within-project examples, constructor info, stack traces. Temperature ablation is in Section 6.4.5."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: number of bugs reproduced, FIB count, ROC-AUC for selection (Figure 4), acc@n and precision@n for ranking (Table 7), wef@n for wasted effort, and GPU memory usage (Figure 7)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated (fail on buggy version, pass on fixed version). No human evaluation of test quality, readability, or developer usefulness is performed, despite the paper's claims about reducing developer effort."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The GHRB dataset (Section 4.1) was specifically collected from PRs after the Codex training data cutoff, serving as a held-out evaluation set to mitigate data leakage concerns."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides per-project breakdown of reproduction performance on Defects4J. Table 8 provides per-project breakdown for GHRB. Figure 6 shows per-LLM breakdown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 presents a detailed failure case (Checkstyle Issue #11365, Listing 5) where LIBRO fails because the test references a non-existent file, illustrating the limitation of not being able to modify execution environments."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: within-project examples hurt performance (Table 4, Section 6.1.1), fine-tuning on natural language hurts code tasks (StarCoderPlus and BloomZ, Section 6.4.1), Closure project shows poor performance, ChatGPT behavior change broke pipeline (Section 6.4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — one-third reproduction rate (251/750 = 33.5%, Table 5), StarCoder at 70% of Codex (Section 6.4.1), 90% on GHRB (Section 6.4.2), performance improving with size (Figure 8a) — are all supported by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'fine-tuning on natural language can hurt performance' are supported by controlled comparisons within the same model family (StarCoder vs StarCoderPlus, Bloom vs BloomZ), where training data/technique is the only difference. Ablation of prompt components (Table 4) also follows single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to Java (all benchmarks are Java), acknowledges Defects4J may be in training data (Section 4.1), creates GHRB to verify generalization, and notes project-specific variation (Table 5). Title uses 'Evaluating' rather than claiming universal capability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 analyzes whether LIBRO's performance is due to code extraction from reports vs genuine synthesis. Section 4.1 and RQ3/RQ4-2 address whether results are due to LLM memorization vs actual capability, verified with GHRB and membership tests."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures bug reproduction (test fails on buggy version, passes on fixed version) and frames results as bug reproduction performance. The measurement matches the claim with no proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 lists exact model identifiers: code-davinci-002, text-davinci-003, gpt-3.5-turbo-0301, gpt-3.5-turbo-0613, Bloom-176B, BloomZ-176B, Incoder-(1,6)B, CodeGen2-(1,3.7,7,16)B, StarCoder-15B, StarCoderBase-15B, StarCoderPlus-15B with sizes and release years."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Listing 1 shows the complete prompt format used, including the Markdown structure, reproduction instruction, and code block initiation. The full prompt text is provided, not just a description."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: temperature 0.7 (default), maximum generated tokens 256, n=10 or n=50 test samples. Temperature is further explored across 7 values in Section 6.4.5."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The LIBRO pipeline is described in detail: Section 3 provides four stages (prompt engineering, LLM querying, postprocessing, selection & ranking) with formal algorithms (Algorithm 1 for test postprocessing, Algorithm 2 for selection and ranking) and worked examples."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents filtering: Defects4J 814 → 750 bugs (58 excluded for poor report-bug mapping, 6 for structural changes). GHRB: 581 PRs → 435 (non-test-introducing removed) → 84 (non-merged or multi-issue removed) → 31 (verified BRT). Each step has explicit criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated 'Limitations' or 'Threats to Validity' section exists. The paper's sections are: Introduction, Motivation, Approach, Evaluation, Research Questions, Results, Discussion, Related Work, Conclusion. Limitations are scattered across RQ discussions but not collected in a dedicated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed throughout: Defects4J likely in training data (Section 4.1), GHRB is small (31 bugs), Checkstyle failures due to external file dependency (Section 7.1), ChatGPT behavior changes breaking the pipeline (Section 6.4.3), and temperature sensitivity (Section 6.4.5). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundary statements about what the results do NOT show. The evaluation is limited to Java and specific benchmarks, but this is implicit rather than explicitly stated as a boundary. The paper doesn't have a 'what this does not show' discussion."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Experimental data and analysis scripts are publicly available via the replication package (https://github.com/coinse/libro-journal-artifact), enabling independent verification of results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes Defects4J v2.0 collection and filtering in detail. GHRB collection from 17 GitHub repositories is described step-by-step, including the criteria for PR selection and verification process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (Defects4J) and publicly available GitHub repositories (GHRB)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents: Defects4J 814 bugs → 58 excluded (poor mapping) → 6 excluded (structural differences) → 750. GHRB: 581 PRs → 435 (test-introducing) → 84 (merged, single-issue) → 31 (verified BRT). Each stage has counts and criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper text. There is no mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed: all authors are from KAIST (Korea Advanced Institute of Science and Technology). They are academic researchers not affiliated with any LLM company they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. The absence of any funding disclosure makes this unanswerable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper references 'the Codex training data cutoff point' (Section 4.1) without stating the actual date. For other LLMs (StarCoder, CodeGen2, etc.), no training cutoff dates are explicitly stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensively discussed. Section 4.1 cites Lee et al. showing Defects4J BRTs are in StarCoder training data. The GHRB dataset was created specifically to mitigate this. StarCoder's dataset membership test was used to verify GHRB tests are not in the Stack dataset."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges Defects4J is likely in most LLM training data (Section 4.1) and creates the GHRB held-out dataset after the Codex training cutoff. They verify using StarCoder's dataset membership test that GHRB reproducing tests are not in the Stack dataset."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated against benchmark programs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on software benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports per-step timing: API querying 5.85s, postprocessing 1.23s, test execution 4.00s, total 444s for 50 tests. Figure 7 plots GPU memory usage per model. Section 1 notes 'more than eight months of GPU time and seven months of CPU time.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the experiments required 'more than eight months of GPU time and seven months of CPU time' (Section 1). Section 4.3 specifies the hardware: 4 NVIDIA RTX 3090 GPUs with 96GB total VRAM."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main LLM comparison results (Figure 6) are single-run results without reporting sensitivity to random seeds. The 1000-run subsampling simulation (Figure 3) shows variance from subsampling but not from different experimental seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly states 'we sample 10 tests (denoted by n=10)' and 'experimenting with sampling 50 tests as well.' The 1000-run simulation count is also stated. Temperature experiments state the number of configurations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While 7 temperature values and multiple prompt configurations are explored as research questions (Table 4, Figure 8b), no formal hyperparameter search budget is reported. The search method and total compute for configuration selection are not described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows all prompt configurations and their results. Figure 8b shows all temperature values tested. The two-example n=50 setting is chosen based on best observed performance, and all alternatives are transparently reported."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 15 LLMs, multiple prompt configurations, and 7 temperature settings without any statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own LIBRO system against baselines (including their own implementation of Copy&Paste) without acknowledging potential author-evaluation bias. No independent evaluation or mitigation strategy is discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 explicitly plots GPU memory usage against reproduction performance for all open-source LLMs, with a Pareto front analysis. Each model is mapped to a specific GPU count, helping practitioners make resource-informed decisions."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Defects4J or GHRB actually measure real-world bug reproduction capability. They define BRT precisely but do not question whether success on these benchmarks translates to practical developer benefit."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All LLMs are evaluated using the same LIBRO pipeline (prompt format, postprocessing, ranking), controlling for scaffold differences. The comparison is strictly between models within the same framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 creates GHRB from PRs after the Codex training data cutoff. They verify using StarCoder's dataset membership test that GHRB tests are not in the Stack training dataset."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 notes 'our specific template format makes it highly unlikely that prompts we generate exist verbatim within the LLM training data' and discusses that bug reports are only connected to BRTs via chains of references, partly mitigating leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges Defects4J is likely in training data (citing Lee et al.), creates GHRB from separate repositories post-cutoff, and verifies using StarCoder's dataset membership test that GHRB test code is not in the Stack dataset."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Two concrete methods are used: (1) StarCoder's dataset membership test (Section 6.4.2) to verify GHRB tests are not in training data, and (2) temporal splits creating GHRB from post-cutoff PRs."
       }
     }
   },
   "claims": [
     {
       "claim": "LIBRO using code-davinci-002 can reproduce 33.5% of bugs (251/750) in Defects4J v2.0.",
       "evidence": "Table 5 and Section 6.1.1 show 251 bugs reproduced out of 750 under the two-example n=50 setting.",
       "supported": "strong"
     },
     {
       "claim": "StarCoder achieves 70% of Codex's reproduction performance on Defects4J and 90% on GHRB.",
       "evidence": "Section 6.4.1 reports StarCoder reproducing 125/173 bugs (n=10, Defects4J). Section 6.4.2 reports StarCoder at 90% of Codex on GHRB (n=50).",
       "supported": "strong"
     },
     {
       "claim": "Bug reproduction performance increases as LLM size increases within the same model family.",
       "evidence": "Figure 8a shows increasing performance with model size for both CodeGen2 (1B→16B) and Incoder (1B→6B) families.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning code LLMs on natural language can hurt bug reproduction performance.",
       "evidence": "Section 6.4.1 shows StarCoderPlus (NL fine-tuned) performed 'substantially worse' than StarCoder, and BloomZ performed worse than Bloom.",
       "supported": "moderate"
     },
     {
       "claim": "Sampling temperature of 0.6 optimizes LIBRO-StarCoder performance.",
       "evidence": "Figure 8b and Section 6.4.5 show systematic evaluation across 7 temperature values (0 to 1.0), with T=0.6 yielding 127 reproduced bugs (best).",
       "supported": "strong"
     },
     {
       "claim": "LIBRO's self-consistency-based selection and ranking strategy generalizes across different LLMs.",
       "evidence": "Table 10 and Figure 9 show the strategy consistently improves precision across all tested LLMs. The ROC-AUC is remarkably consistent (0.76-0.80) across temperatures (Section 6.5.2).",
       "supported": "strong"
     },
     {
       "claim": "Open-source LLMs can generate bug-reproducing tests for bugs not in their training data.",
       "evidence": "Section 6.4.2 shows StarCoder reproducing bugs from GHRB (post-cutoff dataset), and StarCoder's dataset membership test confirms the tests are not in the Stack training data.",
       "supported": "strong"
     },
     {
       "claim": "LIBRO substantially outperforms the crash reproduction baseline EvoCrash.",
       "evidence": "Figure 2 shows LIBRO reproduced 91 more unique bugs (19 being crash bugs) than EvoCrash, and can handle non-crash bugs that EvoCrash cannot.",
       "supported": "strong"
     }
   ],
   "key_findings": "LIBRO, an LLM-based pipeline for generating bug-reproducing tests from natural language bug reports, can reproduce about one-third (251/750) of Defects4J bugs using code-davinci-002. Among 15 evaluated LLMs, StarCoder (15B, open-source) achieves 70% of Codex's performance on Defects4J and 90% on a held-out dataset, demonstrating open-source LLM viability. The self-consistency-based selection and ranking mechanism generalizes across LLMs and temperature settings, with ROC-AUC of 0.76-0.80 for bug selection. Performance scales with LLM size within families, while fine-tuning on natural language hurts code reproduction capability.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All LLM comparisons (15 models, multiple settings) are based on raw number comparisons without any statistical tests. Claims of 'best performance' and 'substantially worse' lack formal hypothesis testing."
     },
     {
       "flag": "Very small held-out dataset",
       "detail": "The GHRB dataset used to verify generalization contains only 31 bugs, which is quite small for drawing conclusions about model capabilities outside training data."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated Limitations or Threats to Validity section, which is unusual for an empirical SE paper in IEEE TSE. Limitations are scattered across research question discussions."
     },
     {
       "flag": "No human evaluation of developer utility",
       "detail": "Despite framing LIBRO as reducing developer effort and measuring metrics like 'wasted effort,' no human evaluation of test quality, readability, or actual developer usefulness was conducted."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational paper on Codex/HumanEval benchmark for LLM code generation, directly relevant to evaluating LLM coding capabilities."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot prompting paradigm used in LIBRO's prompt engineering approach."
     },
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "relevance": "Open-source code LLM that achieved best open-source performance in the LIBRO evaluation, relevant to open-source LLM capability assessment."
     },
     {
       "title": "Large language models are few-shot testers: Exploring llm-based general bug reproduction",
       "authors": ["S. Kang", "J. Yoon", "S. Yoo"],
       "year": 2023,
       "relevance": "Prior conference version of this work (ICSE '23), establishing LLM-based bug reproduction as a research area."
     },
     {
       "title": "CodaMOSA: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["C. Lemieux", "J. P. Inala", "S. K. Lahiri", "S. Sen"],
       "year": 2023,
       "relevance": "Combines LLM generation with search-based testing, directly relevant to LLM-assisted test generation evaluation."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["X. Wang", "J. Wei", "D. Schuurmans"],
       "year": 2023,
       "relevance": "Theoretical basis for LIBRO's selection and ranking algorithm; demonstrates self-consistency as a general LLM property."
     },
     {
       "title": "How is ChatGPT's behavior changing over time?",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "relevance": "Documents LLM behavior drift over time, directly relevant to reproducibility concerns with API-based LLM evaluation."
     },
     {
       "title": "The GitHub recent bugs dataset for evaluating LLM-based debugging applications",
       "authors": ["J. Y. Lee", "S. Kang", "J. Yoon", "S. Yoo"],
       "year": 2023,
       "relevance": "Companion work analyzing data contamination in Defects4J for LLM evaluation, directly relevant to benchmark validity."
     },
     {
       "title": "Defects4j: A database of existing faults to enable controlled testing studies for Java programs",
       "authors": ["R. Just", "D. Jalali", "M. D. Ernst"],
       "year": 2014,
       "relevance": "Primary benchmark used for evaluation; widely used in software engineering research for bug-related tasks."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["J. Wei", "Y. Tay", "R. Bommasani"],
       "year": 2022,
       "relevance": "Referenced for the sudden emergence of bug reproduction capability at certain model sizes (CodeGen2-7B), relevant to understanding LLM scaling behavior."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "relevance": "Evaluates open-source LLMs for SE tasks, relevant to comparative LLM capability assessment."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li", "D. Choi", "J. Chung"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Uses test generation to boost code synthesis, relevant to understanding the relationship between test generation and code quality."
     }
   ]
 }