ai-research-survey

scan.json (32037B)

---

 {
   "paper": {
     "title": "RunBugRun – An Executable Dataset for Automated Program Repair",
     "authors": [
       "Julian Aron Prenner",
       "Romain Robbes"
     ],
     "year": 2023,
     "venue": "arXiv.org",
     "arxiv_id": "2304.01102",
     "doi": "10.48550/arXiv.2304.01102"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RunBugRun provides ~450,000 executable buggy/fixed program pairs across 8 languages with test cases and fine-grained bug labels, derived from programming contest submissions. A CodeT5-based NPR baseline achieved 59.8% plausibility across all languages (best on Python at 69%, worst on JavaScript at 44.5%), while the G&V baseline Cardumen fixed only ~5% of Java bugs under a constrained time budget. Cross-language knowledge transfer significantly boosts performance for low-resource languages (e.g., +34.5pp for PHP, +31.8pp for Go), and performance strongly degrades with the number of token changes required (r = −0.996).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL: https://github.com/giganticode/run_bug_run (Section 8), which includes the dataset and infrastructure to compile, execute, and test programs."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset of ~450,000 buggy/fixed program pairs, test cases, bug labels, and error messages is released through the project website referenced in Section 8."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 specifies exact compiler/interpreter versions: GCC 11 for C/C++, CPython 3.9, CRuby 3.0, OpenJDK 17, PHP 8.1.2, Go 1.18.1, NodeJS 12.22. For the NPR baseline: NVIDIA RTX 3090 with 24GB, HuggingFace transformers library with codet5-small checkpoint."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'Details on how to download, install and run RunBugRun are provided on the project website.' The infrastructure includes sandbox-based execution using bwrap, and the paper provides detailed methodology for reproducing baseline experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '59.8%' plausibility rate, '64.4%' for C) without any confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims performance differences across languages and training regimens (e.g., 'performance is best for Python (69%)... hardest languages were Ruby (47.6%)') without any statistical significance tests. Pearson's r is reported but without p-values for most comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pearson's r values are reported (r = −0.996 for changes vs. NPR performance, r = 0.33 for training size vs. performance). Performance ranges are given with baseline context (e.g., '68.7% for single change fixes to 42.8% for six changes'). Transfer effects are quantified (e.g., '+34.5pp for PHP')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes used. The manual correctness assessment uses 200 randomly selected patches with no power analysis or justification for this number. The 450K dataset size is not justified from a statistical power perspective."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All baseline results appear to be from single experimental runs. No standard deviations, variance across seeds, or any spread measures are reported for either the NPR or G&V baseline evaluations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are provided: Cardumen, a G&V approach (Section 5.2), and a fine-tuned CodeT5-small NPR model. These represent the two main paradigms in APR."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeT5 (2021) was a contemporary pre-trained code model at the time of writing. Cardumen was justified as 'one of the most successful G&V systems on QuixBugs' (Section 4), a similar benchmark. Both were reasonable choices for 2023."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2.3 presents ablation-style experiments with 17 different training regimens: training on all languages, excluding one language, or training on a single language only (Table 5). RQ2.4 varies the number of candidates from 1 to 10 (Figure 7)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary evaluation metric is plausibility rate (whether patches pass all test cases). While the paper provides per-label r-scores and Pearson correlations as analytical tools, the fundamental evaluation metric for the baselines is a single metric: test-pass plausibility rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Plausibility vs correctness') reports manual assessment of 200 randomly selected plausible fixes, finding 3.5% (7/200) to be incorrect. 46.5% were identical to the ground-truth patch."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The dataset has explicit train/validation/test splits (Section 4, 'Data splitting'). The test set contains ~32,500 bugs. Splits ensure no user's submissions to the same problem cross set boundaries, and trigram-based deduplication prevents cross-set near-duplicates."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: per-language performance (Table 5), per-label performance with aggregated categories (Table 4), per-change-count performance, and strengths/weaknesses analysis (RQ2.2). Figure 4 shows language distribution across splits."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2.2 provides a detailed weak-spot analysis for both baselines. For CodeT5: struggles with adding if-branches and continue keywords. For Cardumen: fails entirely on adding function calls or type conversions. Specific failing label categories are identified."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cardumen's poor performance (~5% plausibility) is reported honestly. The paper discusses where models fail (RQ2.2 weak points). Transfer learning limitations are noted: 'perhaps there is a limit to the effectiveness of transfer learning' for complex bugs (Section 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about dataset size (~450K), 8 languages, executable programs, test cases, fine-grained labels, and two baselines are all supported by detailed results in Sections 4-5. The abstract appropriately hedges ('basic evaluation results')."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is about cross-language knowledge transfer (RQ2.3). This is supported by controlled experiments: training with/without specific languages while holding other variables constant (Table 5). The ablation design with 17 models provides adequate causal evidence for transfer effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly bounds the scope: 'The dataset consists of short implementations of solutions to algorithmic problems. There is a significant difference between the program code in the dataset and the code of larger software projects.' The title and abstract accurately describe it as a dataset for APR, not a general code quality tool."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses alternative explanations for Cardumen's poor performance (constrained time budget, low redundancy in small programs). For transfer learning, the paper considers language similarity as a confounding factor ('Java and C we conjecture that this might be due to the languages' similarity to C++')."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes plausibility (passing all tests) from correctness (Section 7, 'Plausibility vs correctness'): 'Passing all test cases is no guarantee for a patch to be fully correct.' They quantify the gap through manual assessment (3.5% false positives) and discuss test suite strength as a mitigating factor."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The NPR baseline uses the specific 'codet5-small' model checkpoint from HuggingFace. Cardumen is run via ASTOR with Flacoco fault localization. All compiler/interpreter versions are specified (GCC 11, CPython 3.9, OpenJDK 17, etc.)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. CodeT5 is fine-tuned as a seq2seq model (buggy→fixed translation) with language identification token prefixes, not prompted. Cardumen is a G&V tool that does not use prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.1 reports: learning rate 1e-4, batch size 16, 2 gradient accumulation steps, 5 epochs, beam search with 6 beams, 5 candidates per bug. For Cardumen: -maxtime 10 -maxgen 100. Memory limit 512MB, execution timeout 3 seconds."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. CodeT5 is a straightforward seq2seq fine-tuned model, and Cardumen is a template-based G&V tool."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 provides extensive documentation: language filtering, buggy/fixed pair extraction using LCS-based diffing with 6-token threshold, syntax error filtering, Python 2→3 conversion, sandbox execution of 130M+ executions, test case filtering (45% pass rate threshold, 12% timeout rate), flakiness checks, trigram deduplication, and rule-based labeling with TreeSitter."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations, Biases and Threats to Validity' is a dedicated section with multiple subsections addressing limited scope, lack of diversity, tokenization, labeling, and plausibility vs. correctness."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed include: short algorithmic programs vs. large projects (limited scope), same-problem solution similarity reducing bug diversity, language-specific tokenizer limitations, rule-based labeling possibly missing important change classes, and 3.5% false-positive plausibility rate from manual assessment."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: programs lack 'complex class hierarchies, UI-related code, large numbers of methods, functions and code files, dependencies on software frameworks and libraries.' The paper also states bugs are limited to 1-6 token changes and are single-file only."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset is released including all source programs, test cases, labels, error messages, and execution results. The underlying Project CodeNet source is also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes data collection in detail: sourced from Project CodeNet (AIZU and AtCoder submissions), with specific criteria for pair selection (same problem, same user, <6 token difference between rejected and accepted submissions)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from Project CodeNet, a well-known public dataset of programming contest submissions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the full pipeline: 13M+ submissions → language filtering → pair extraction → syntax error filtering → Python 2→3 conversion → execution (130M+ runs) → test case filtering → flakiness checks → labeling → deduplication → splitting. Specific counts are given at key stages (e.g., 9,082 example test cases extracted, 331,834 from CodeContests, 600+ generated)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Julian Aron Prenner at inf.unibz.it (University of Bolzano) and Romain Robbes at u-bordeaux.fr (University of Bordeaux). Neither evaluates a product from their own institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, independence of funding cannot be assessed. The absence of a funding statement makes this unanswerable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state CodeT5's pre-training data cutoff date. CodeT5 was pre-trained on code data but no temporal bounds are given for what code was included."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 ('Deduplication') extensively addresses train/test overlap: exact duplicates are removed, cross-set near-duplicates are removed using trigram similarity, and splits ensure the same user's submissions to the same problem don't cross sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Project CodeNet submissions could have appeared in CodeT5's pre-training corpus. Since both Project CodeNet and CodeT5 were released in 2021, there is a non-trivial contamination risk that is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study mines programming contest submissions and runs automated evaluations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available programming contest submissions."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Contest submitters are anonymized in the source dataset."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Inclusion/exclusion criteria for submissions (not people) are thoroughly documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving people."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No specific inference cost or latency per example is reported for the NPR baseline. Cardumen is given a 10-minute time budget per bug but no aggregate cost. The paper mentions 'several weeks' for 130M+ executions but does not quantify per-bug evaluation cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The GPU is stated (single NVIDIA RTX 3090, 24GB) and training duration (5 epochs) but no total GPU hours, wall-clock training time, or total compute budget is reported. Execution cost is described only as 'several weeks.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis. All results appear to be from a single training run of each model configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. The 17 different training configurations appear to each be single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Specific hyperparameters are reported but no search budget or tuning methodology is described. It is unclear how the learning rate (1e-4), batch size (16), etc. were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses specific hyperparameters without explaining how they were selected. No validation-based selection procedure is described."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not perform multiple formal statistical hypothesis tests requiring correction. Analyses are primarily descriptive (plausibility rates, correlations)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implemented the NPR baseline and configured Cardumen with significantly constrained parameters (-maxtime 10 -maxgen 100). While the Cardumen constraints are acknowledged, the paper does not discuss author-evaluation bias or the risk that their implementation choices may systematically favor one baseline over another."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "CodeT5 and Cardumen operate under very different compute regimes (GPU-based neural inference vs. 10-minute search budget per bug). The paper acknowledges Cardumen's constrained time but does not systematically analyze performance as a function of compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Limited scope') explicitly discusses construct validity: short algorithmic programs differ from large software projects, bug diversity is limited by same-problem solutions, and the bug type distribution is compared to real-world distributions (Pan et al. findings). Section 3 provides extensive motivation for why this benchmark design measures what it claims."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. CodeT5 is a direct seq2seq model and Cardumen is a template-based G&V tool."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether CodeT5's pre-training data temporally overlaps with Project CodeNet submissions. Both were publicly available around 2021, creating a plausible temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether any information in the model's input (e.g., problem IDs, language tokens) could leak information about the expected fix."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses non-independence through deduplication: exact duplicates removed, trigram-based cross-set near-duplicate removal, and ensuring same-user/same-problem submissions don't cross splits (Section 4, 'Deduplication' and 'Data splitting')."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Trigram similarity is used as a concrete leakage detection method for cross-set near-duplicates. Exact matching is used for full duplicates. The paper explicitly compares this to prior work which only used exact matching (Section 2)."
       }
     }
   },
   "claims": [
     {
       "claim": "RunBugRun contains ~450,000 executable buggy/fixed program pairs in 8 programming languages with test cases and fine-grained bug labels.",
       "evidence": "Section 5.1 provides detailed statistics: 417,101 training, 32,504 test, 7,144 validation instances across C, C++, Java, Python, Ruby, PHP, JavaScript, Go (Figure 4). Bug labels (136 leaf categories), test cases, and error messages are documented.",
       "supported": "strong"
     },
     {
       "claim": "The CodeT5-based NPR baseline achieves 59.8% plausibility rate across all languages with 5 candidates per bug.",
       "evidence": "Section 5.2.2 reports this figure. Per-language breakdown in Table 5: Python 69%, C 64.4%, Java 62.9%, PHP 55.7%, C++ 54.9%, Go 47.8%, Ruby 47.6%, JavaScript 44.5%.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-language knowledge transfer significantly boosts repair performance for low-resource languages.",
       "evidence": "Table 5 (RQ2.3) shows 17 models with different training regimens. Transfer effects: +34.5pp for PHP, +31.8pp for Go, +28.3pp for JavaScript when training on all languages vs. target language only. Small gains for high-resource languages: +1.2pp for C++, +2.2pp for Python.",
       "supported": "moderate"
     },
     {
       "claim": "There is a strong negative correlation (r = −0.996) between the number of fixing changes and NPR baseline performance.",
       "evidence": "Section 5.2.2 reports Pearson's r = −0.996 for the NPR baseline and r = −0.6823 for the G&V baseline. Performance ranges from 68.7% (1 change) to 42.8% (6 changes) for the NPR model.",
       "supported": "strong"
     },
     {
       "claim": "Cardumen (G&V baseline) fixed only ~5% of Java bugs, performing poorly compared to the NPR baseline.",
       "evidence": "Section 5.2.2 reports 157 of ~3,300 Java bugs fixed. The authors acknowledge the constrained time budget (-maxtime 10 -maxgen 100) and low redundancy in small programs as contributing factors.",
       "supported": "weak"
     },
     {
       "claim": "Only 3.5% of plausible NPR patches were found to be incorrect upon manual assessment.",
       "evidence": "Section 7 reports manual assessment of 200 randomly selected plausible fixes: 7 (3.5%) incorrect, 46.5% identical to ground truth. However, 19 patches were difficult to assess and required execution-based comparison.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or error bars on any results",
       "detail": "All baseline results appear to be from single experimental runs. No standard deviations, confidence intervals, or seed sensitivity analysis is reported for any of the 17+ model configurations trained and evaluated. This makes it impossible to assess result stability."
     },
     {
       "flag": "Unfair G&V baseline comparison",
       "detail": "Cardumen was given a 10-minute time budget and 100 generations per bug, which the paper acknowledges is 'unusually low' compared to typical G&V budgets of hours. This severely constrains Cardumen's performance, making the NPR vs. G&V comparison misleading. The paper notes this but the headline comparison (5% vs 60%) is still presented."
     },
     {
       "flag": "Pre-training contamination not addressed",
       "detail": "CodeT5 was pre-trained on code data, and Project CodeNet was publicly available before CodeT5's training. The paper does not discuss whether CodeT5's pre-training corpus included any programming contest submissions that overlap with the benchmark data."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "Pre-trained code model used as the NPR baseline in the paper, representative of neural approaches to program repair."
     },
     {
       "title": "Defects4J: A Database of Existing Faults to Enable Controlled Testing Studies for Java Programs",
       "authors": ["René Just", "Darioush Jalali", "Michael D. Ernst"],
       "year": 2014,
       "doi": "10.1145/2610384.2628055",
       "relevance": "The most widely used APR benchmark; RunBugRun positions itself as complementary with executable infrastructure inspired by Defects4J."
     },
     {
       "title": "CoCoNuT: Combining Context-Aware Neural Translation Models Using Ensemble for Program Repair",
       "authors": ["Thibaud Lutellier"],
       "year": 2020,
       "doi": "10.1145/3395363.3397369",
       "relevance": "NPR system that mined 3.2M buggy/fixed code pairs for training; relevant for understanding data-hungry neural repair approaches."
     },
     {
       "title": "Neural Program Repair with Execution-Based Backpropagation",
       "authors": ["He Ye", "Matias Martinez", "Martin Monperrus"],
       "year": 2022,
       "doi": "10.1145/3510003.3510222",
       "relevance": "RewardRepair integrates test execution feedback into NPR training, directly relevant to RunBugRun's goal of enabling execution-based NPR."
     },
     {
       "title": "SequenceR: Sequence-to-Sequence Learning for End-to-End Program Repair",
       "authors": ["Zimin Chen"],
       "year": 2019,
       "doi": "10.1109/TSE.2019.2940179",
       "arxiv_id": "1901.01808",
       "relevance": "Pioneering seq2seq approach to NPR that frames repair as neural machine translation, foundational to the NPR paradigm RunBugRun targets."
     },
     {
       "title": "FixEval: Execution-based Evaluation of Program Fixes for Programming Problems",
       "authors": ["Md Mahim Anjum Haque"],
       "year": 2022,
       "arxiv_id": "2206.07796",
       "relevance": "Compared execution-based and match-based evaluation metrics for NPR, finding execution-based metrics are better performance indicators — directly supports RunBugRun's motivation."
     },
     {
       "title": "SelfAPR: Self-supervised Program Repair with Test Execution Diagnostics",
       "authors": ["He Ye"],
       "year": 2022,
       "arxiv_id": "2203.12755",
       "relevance": "Demonstrates that error messages and stack traces from execution improve repair accuracy, motivating RunBugRun's collection of execution data."
     },
     {
       "title": "Competition-Level Code Generation with AlphaCode",
       "authors": ["Yujia Li"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "AlphaCode's CodeContests dataset provided 331,834 additional test cases used in RunBugRun; relevant to AI code generation evaluation."
     },
     {
       "title": "CodeNet: A Large-Scale AI for Code Dataset for Learning a Diversity of Coding Tasks",
       "authors": ["Ruchir Puri"],
       "year": 2021,
       "arxiv_id": "2105.12655",
       "relevance": "The source dataset from which RunBugRun is derived; 13M+ programming contest submissions across 4,000+ problems."
     },
     {
       "title": "Ultra-Large Repair Search Space with Automatically Mined Templates: The Cardumen Mode of Astor",
       "authors": ["Matias Martinez", "Martin Monperrus"],
       "year": 2018,
       "relevance": "G&V baseline used in RunBugRun evaluation; template-based repair approach representative of generate-and-validate paradigm."
     },
     {
       "title": "Unified Pre-training for Program Understanding and Generation",
       "authors": ["Wasi Uddin Ahmad"],
       "year": 2021,
       "arxiv_id": "2103.06333",
       "relevance": "PLBART multi-language code model whose language identification token approach was adopted by RunBugRun's NPR baseline."
     },
     {
       "title": "Empirical review of Java program repair tools: A large-scale experiment on 2,141 bugs and 23,551 repair attempts",
       "authors": ["Thomas Durieux"],
       "year": 2019,
       "relevance": "Found APR tools overfit to Defects4J benchmark, directly motivating RunBugRun's larger and more diverse evaluation set."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a usable benchmark and infrastructure for APR researchers; practitioners building repair tools can directly use this dataset for training and evaluation."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that a simple NPR baseline outperforms G&V by 12x is notable, and cross-language transfer effects are interesting, but the overall message (execution helps, multi-lingual is needed) confirms existing intuitions."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper is about improving automated program repair tools."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward dataset contribution paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Full dataset, infrastructure, and code released on GitHub (https://github.com/giganticode/run_bug_run); researchers can download and run evaluations."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From University of Bolzano and University of Bordeaux; not a major AI lab or associated with a well-known product."
     }
   }
 }