ai-research-survey

scan.json (24240B)

---

 {
   "paper": {
     "title": "BugsPHP: A dataset for Automated Program Repair in PHP",
     "authors": [
       "K.D. Pramod",
       "W.T.N. De Silva",
       "W.U.K. Thabrew",
       "Ridwan Shariffdeen",
       "Sandareka Wickramanayake"
     ],
     "year": 2024,
     "venue": "MSR 2024 (21st International Conference on Mining Software Repositories)",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL in the Conclusion section: https://github.com/bugsphp/bugsPHP.git"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released via the same GitHub repository. The paper states: 'Our dataset can be accessed via GitHub from the following repository: https://github.com/bugsphp/bugsPHP.git'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using PHP version 8.1 for manual validation but does not provide a requirements.txt, Dockerfile, or detailed environment setup section listing library versions for reproducing the dataset curation or the APR model experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While the methodology is described at a conceptual level, there are no specific commands, scripts, or README instructions for replicating the dataset construction or the APR experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The APR evaluation results in Table 2 report only point estimates (e.g., CURE fixes 11 bugs, RewardRepair fixes 43 bugs) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares CURE and RewardRepair performance but does not use any statistical significance test to support the claim that RewardRepair outperforms CURE. The comparison is based solely on raw count differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper presents raw counts of bugs fixed but does not report effect sizes, percentage improvements with baseline context, or any standardized measures of effect."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not justify why 5,000 repositories were chosen, why the top 75 were selected for testing, or whether 513 test bugs is sufficient for meaningful APR evaluation. No power analysis or sample size rationale is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs. The APR experiments appear to be single-run results with no information about result stability across seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two APR models (CURE and RewardRepair) are evaluated as baselines to demonstrate the utility of the dataset. The comparison between these two models serves as a baseline evaluation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CURE (2021) and RewardRepair (2022) are reasonably recent learning-based APR models relative to this 2024 paper, and represent state-of-the-art learning-based approaches for program repair."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is primarily a dataset paper. The system being evaluated (the dataset) is a single artifact without separable components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports multiple metrics: NC (candidate patches generated), NV (failing tests fixed), NP (all tests passed), NE (semantically equivalent patches), and NI (identical patches). This provides a multi-metric view of APR model performance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The test dataset construction involved manual validation of bug-fixing commits. The paper states they 'carry out a manual test case validation' and 'Manual Validation: In this step, we examine the latest bug to validate a repository and see if its fixed version has accompanying test cases.' This is human evaluation of the dataset quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates training (653,606 commits from remaining repositories) and test (513 bugs from top 75 repositories) datasets. The repositories for the test dataset are excluded from the training dataset: 'Excluding the project repositories used for the test data-set, the remaining projects are used to construct the training data-set.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a per-project breakdown of bugs, tests, LOC, and test coverage. Section 4.2 also breaks down bugs by type (462 functional errors, 16 type errors, 15 security vulnerabilities, 13 compatibility issues, 5 usability issues, 2 performance bugs) and by number of file/line changes."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that 'none of the APR models could fix 473 bugs in our test dataset' and discusses that CURE can only generate single-line patches while RewardRepair generates 3-4 line patches, indicating limitations of existing tools."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that both APR models fail on the majority of bugs (473/513 unfixed). CURE generates candidate patches for only 443/513 bugs, and only 11 pass all tests. These are negative results showing the challenge of the dataset."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the dataset consists of 'more than 600,000 bug-fixing commits' (confirmed at 653,606 in Section 4) and '513 manually validated bug-fixing commits equipped with developer-provided test cases' (confirmed in Section 4.1 and Table 1). All claims are supported by the paper's content."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily a dataset paper and does not make causal claims. It presents descriptive statistics and preliminary APR results without claiming any causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract present this as a dataset for 'Automated Program Repair in PHP' broadly, but the test dataset is limited to 15 applications from popular open-source projects. The paper does not explicitly discuss whether these 15 projects are representative of PHP programs in general, or bound its claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed APR results. For example, it does not consider whether the low fix rates could be due to training data quality, PHP-specific language features, or the specific models chosen rather than the inherent difficulty of PHP bug repair."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using CURE and RewardRepair but does not specify exact model versions, checkpoint details, or configuration variants used. It cites the original papers [7] and [17] but does not state which specific trained models or configurations were used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates traditional learning-based APR models (CURE and RewardRepair) that do not use LLM prompting. Prompting is not part of the methodology."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for either CURE or RewardRepair training. The paper states they 'generate 100 candidate patches per bug' but does not report training hyperparameters, beam search settings, or other model configuration details."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. CURE and RewardRepair are traditional learning-based APR models, not agent-based systems."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data preprocessing pipeline is documented in Section 4.1 with explicit filtering criteria: commits containing fix-related keywords, PHP files only, no file addition/deletion/renaming, file changes <= 3, line changes <= 50, commits after Jan 2020. The pipeline stages are shown in Figure 1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The paper moves directly from preliminary results (Section 5) to conclusion (Section 6) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no discussion of potential biases in repository selection, keyword-based commit filtering, or the representativeness of the dataset."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not discuss the limitations of using only top-starred GitHub repositories, the restriction to specific PHP versions, or the exclusion of bugs requiring more than 3 file changes or 50 line changes."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The raw dataset is available via GitHub at https://github.com/bugsphp/bugsPHP.git, allowing independent verification of the bug-fixing commits and test cases."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection procedure in detail: using GitHub REST API to retrieve top 5000 PHP repositories, searching commit messages for bug-fix keywords, applying filtering criteria for file and line changes, and the time period (Jan 2020 to March 2023)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study of public GitHub repositories with no human participants. Recruitment methods are not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented in Section 4.1 and Figure 1, from repository selection through bug collection, filtering, manual validation, and dynamic validation. Each step's criteria are described, and the final counts are provided (5000 repos → top 75 for test → 513 validated bugs; remaining repos → 653,606 training commits)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgement section: 'This work was partially supported by a Singapore Ministry of Education (MoE) Tier3 grant \"Automated Program Repair\", MOE-MOET32021-0001.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Moratuwa (3 authors) and National University of Singapore (1 author). The paper does not evaluate any commercial product, so there are no obvious conflicts."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the Singapore Ministry of Education, a government agency with no financial stake in the outcome of PHP bug dataset research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While there is no obvious conflict, the absence of a declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates traditional learning-based APR models (CURE and RewardRepair) trained on the curated dataset itself, not pre-trained language models evaluated on benchmarks. Contamination in the LLM sense does not apply."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses train/test separation: 'Excluding the project repositories used for the test data-set, the remaining projects are used to construct the training data-set.' The test and training sets come from disjoint sets of repositories."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is not evaluating a pre-trained model on a pre-existing benchmark. The models are trained from scratch on the curated dataset. Pre-training contamination does not apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub repositories."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub repositories."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub repositories."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub repositories."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub repositories."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub repositories."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub repositories."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported for the APR model evaluations. The paper does not mention how long it takes to generate 100 candidate patches per bug or the computational cost of running CURE and RewardRepair."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report GPU hours, hardware used, training time for the APR models, or the time required for dataset curation."
       }
     }
   },
   "claims": [
     {
       "claim": "BugsPHP contains 653,606 bug-fixing commits for training and 513 manually validated bugs for testing from popular open-source PHP applications.",
       "evidence": "Section 4 and Table 1 provide the dataset statistics. The training dataset has 653,606 commits from 4,483 PHP applications. The test dataset has 513 bugs from 15 applications, each with developer-written test cases.",
       "supported": "strong"
     },
     {
       "claim": "RewardRepair outperforms CURE on the BugsPHP test dataset, fixing 43 bugs compared to CURE's 11.",
       "evidence": "Table 2 shows RewardRepair fixes 43 bugs (NP) while CURE fixes 11. RewardRepair generates patches for all 513 bugs while CURE only generates candidates for 443. No statistical significance testing accompanies these claims.",
       "supported": "moderate"
     },
     {
       "claim": "PHP has no existing benchmark bug dataset for automated program repair research.",
       "evidence": "Section 3 (Related Work) surveys existing datasets for Java (Defects4J, Bugs.jar, BEARS, Vul4J), C/C++ (ManyBugs, IntroClass), JavaScript (BugsJS, FixJS), and Python (BugsInPy), but identifies none for PHP.",
       "supported": "strong"
     },
     {
       "claim": "90% of bugs in the test dataset are functional errors.",
       "evidence": "Section 4.2 states: 'We identified 462 bugs (90%) as functional errors, while the remaining bugs consist of 16 type errors, 15 security vulnerabilities, 13 compatibility issues, 5 usability issues and 2 performance bugs.'",
       "supported": "strong"
     },
     {
       "claim": "Existing APR models cannot fix 473 out of 513 bugs in the test dataset.",
       "evidence": "Section 5 states: 'none of the APR models could fix 473 bugs in our test dataset.' RewardRepair fixes 29 unique bugs and CURE fixes 4 unique, with 7 commonly fixed.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "BugsPHP is the first benchmark dataset for automated program repair in PHP, consisting of 653,606 bug-fixing commits for training and 513 manually validated bugs with developer-written test cases for testing. Preliminary evaluation shows that existing learning-based APR models (CURE and RewardRepair) can fix only a small fraction of PHP bugs (11 and 43 out of 513 respectively), highlighting the need for further research on PHP-specific program repair techniques. The dataset is curated from the top 5,000 PHP repositories on GitHub, with training and test sets drawn from disjoint repository sets.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section, which is a significant methodological concern for a dataset paper. There is no discussion of selection bias from using only top-starred GitHub repositories, the impact of keyword-based commit filtering, or whether the 15 test projects are representative of PHP programs in general."
     },
     {
       "flag": "No statistical rigor in APR comparison",
       "detail": "The comparison between CURE and RewardRepair lacks any statistical testing, confidence intervals, or variance reporting. The claim that RewardRepair outperforms CURE is based solely on raw count comparisons from apparently single experimental runs."
     },
     {
       "flag": "Missing hyperparameters and training details",
       "detail": "No hyperparameters, training configurations, or computational details are reported for the APR model experiments, making it impossible to reproduce the preliminary results."
     },
     {
       "flag": "Popularity-based repository selection may bias dataset",
       "detail": "The dataset is constructed from the top 5,000 PHP repositories ranked by GitHub stars. Popular repositories may have different bug patterns, coding standards, and testing practices than typical PHP projects, potentially limiting dataset representativeness. This bias is not acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Defects4J: A database of existing faults to enable controlled testing studies for Java programs",
       "authors": ["René Just", "Darioush Jalali", "Michael D Ernst"],
       "year": 2014,
       "relevance": "Foundational benchmark dataset for Java program repair, directly comparable to BugsPHP's contribution for PHP."
     },
     {
       "title": "Bugs.jar: A large-scale, diverse dataset of real-world java bugs",
       "authors": ["Ripon K Saha", "Yingjun Lyu", "Wing Lam", "Hiroaki Yoshida", "Mukul R Prasad"],
       "year": 2018,
       "relevance": "Large-scale Java bug dataset with 1,158 bugs from 8 projects, methodologically similar to BugsPHP's training dataset curation."
     },
     {
       "title": "BugsInPy: A Database of Existing Bugs in Python Programs to Enable Controlled Testing and Debugging Studies",
       "authors": ["Ratnadira Widyasari", "Sheng Qin Sim"],
       "year": 2020,
       "doi": "10.1145/3368089.3417943",
       "relevance": "Python bug dataset analogous to BugsPHP, enabling comparison of dataset construction methodologies across languages."
     },
     {
       "title": "Bugsjs: a benchmark of javascript bugs",
       "authors": ["Péter Gyimesi", "Béla Vancsics", "Andrea Stocco"],
       "year": 2019,
       "relevance": "JavaScript bug benchmark with 453 verified bugs, comparable in scope and methodology to BugsPHP's test dataset."
     },
     {
       "title": "CURE: Code-aware neural machine translation for automatic program repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "relevance": "One of the two APR models evaluated on BugsPHP; demonstrates LLM-based code repair approach."
     },
     {
       "title": "Neural program repair with execution-based backpropagation",
       "authors": ["He Ye", "Matias Martinez", "Martin Monperrus"],
       "year": 2022,
       "relevance": "RewardRepair model evaluated on BugsPHP; learning-based APR approach using execution feedback."
     },
     {
       "title": "Automated Program Repair",
       "authors": ["Claire Le Goues", "Michael Pradel", "Abhik Roychoudhury"],
       "year": 2019,
       "doi": "10.1145/3318162",
       "relevance": "Survey of automated program repair field providing context for APR dataset research."
     },
     {
       "title": "An empirical study on learning bug-fixing patches in the wild via neural machine translation",
       "authors": ["Michele Tufano", "Cody Watson", "Gabriele Bavota"],
       "year": 2019,
       "relevance": "Foundational work on learning-based program repair from commit history, methodology followed by BugsPHP for commit collection."
     },
     {
       "title": "FixJS: a dataset of bug-fixing JavaScript commits",
       "authors": ["Viktor Csuvik", "László Vidács"],
       "year": 2022,
       "relevance": "JavaScript bug-fixing commit dataset comparable to BugsPHP's training dataset approach."
     },
     {
       "title": "Vul4J: A Dataset of Reproducible Java Vulnerabilities Geared towards the Study of Program Repair Techniques",
       "authors": ["Quang-Cuong Bui", "Riccardo Scandariato", "Nicolás E. Díaz Ferreyra"],
       "year": 2022,
       "doi": "10.1145/3524842.3528482",
       "relevance": "Vulnerability-focused Java dataset for program repair, related to BugsPHP's security vulnerability subset."
     }
   ]
 }