ai-research-survey

scan.json (26966B)

---

 {
   "paper": {
     "title": "GitBugs: Bug Reports for Duplicate Detection, Retrieval Augmented Generation, Triage, and More",
     "authors": ["Avinash Patil"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2504.09651",
     "doi": "10.48550/arXiv.2504.09651"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["observational", "case-study"],
   "key_findings": "GitBugs aggregates over 150,000 bug reports from 9 open-source projects across GitHub, Jira, and Bugzilla. A case study on Cassandra demonstrates dataset utility: ARIMA outperformed Prophet for bug volume forecasting (MAE 10.92 vs 20.10), priority classification achieved 82% accuracy but only 0.35 macro-F1 due to class imbalance, and time-to-fix regression yielded R\u00b2 = -0.09 (worse than a mean predictor). Sentence-BERT duplicate detection achieved Recall@10 of 0.61 across 300 queries.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract provides a GitHub repository URL: https://github.com/av9ash/gitbugs/. The paper states it includes 'exploratory data analysis (EDA) notebooks, model training and, validation scripts.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is described as 'openly licensed' and accessible at the GitHub repository. The abstract states: 'Access the data and code at https://github.com/av9ash/gitbugs/.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or library version specifications are mentioned in the paper. The paper does not describe the computational environment needed to run the provided scripts."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions reproducible artifacts (EDA notebooks, training scripts) at the GitHub repository but provides no step-by-step reproduction instructions in the paper itself. A reader would need to navigate the repository without guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All case study results are point estimates: MAE of 10.92 vs 20.10, accuracy 82%, macro-F1 0.35, R\u00b2 = -0.09, Recall@10 = 0.61. No confidence intervals or error bars are reported for any metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'ARIMA outperformed Prophet' based solely on comparing two MAE numbers (10.92 vs 20.10) without any statistical test. No p-values or significance tests are used anywhere in the case study."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw metrics (MAE, accuracy, F1, R\u00b2) without formal effect size measures or contextualized comparisons against baselines. The R\u00b2 = -0.09 comparison to a mean predictor is implicit but no formal effect sizes are computed."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 300 randomly selected query bugs were used for duplicate detection, why the last 6 months were reserved for testing, or why only the Cassandra project was selected for the case study."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run experiments. No standard deviations, interquartile ranges, or variance across multiple runs are reported for any of the prediction tasks."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "For bug volume forecasting, ARIMA and Prophet are compared against each other and against actual data. The R\u00b2 = -0.09 implicitly compares against a mean-predictor baseline. However, most other tasks (classification, duplicate detection, RAG) use only a single method."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses ARIMA, Prophet, and Sentence-BERT (2019), which are established but not contemporary methods. Table III surveys LLM-based approaches for these same tasks but the case study does not benchmark against any of them."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents a dataset, not a multi-component system. There are no components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used across tasks: MAE for forecasting, accuracy and macro-F1 for classification, MAE/RMSE/R\u00b2 for regression, and Recall@10 for duplicate detection."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. The RAG-generated explanations are shown as a single illustrative example without human quality assessment. Dataset quality is not evaluated by human annotators."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.C states 'reserving the final six months of data for testing' for the Cassandra case study, implementing a temporal train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides per-project statistics. Figure 5 shows per-class confusion matrix for priority classification. Figures 2-3 show per-project resolution time distributions. Table II compares against existing datasets."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses failures: the time-to-fix model's R\u00b2 = -0.09 ('the model underperforms even a naive predictor'), the severity classifier's poor macro-F1 of 0.35 due to class imbalance, and limitations of textual signals."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The time-to-fix prediction R\u00b2 = -0.09 is explicitly reported as a negative result. The severity classifier's 0.35 macro-F1 is honestly presented as poor. The paper states these 'highlight the need for more informative features or alternative modeling approaches.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'comprehensive and up-to-date dataset comprising over 150,000 bug reports from nine actively maintained open-source projects.' Table I confirms 196,387 total reports across 9 projects. Claims about standardized fields, train/test splits, and EDA notebooks are described in the paper."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. Comparisons are descriptive ('ARIMA outperformed Prophet' by metric comparison). Correlations are stated as correlations ('Bug priority correlates with outcomes'). The trend analysis uses 'suggesting' rather than causal language."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the dataset as covering 9 specific projects from 3 specific trackers. Claims are framed as the dataset being a 'resource' and 'benchmark' for specific listed tasks. The scope is described by the included projects and domains (browsers, IDEs, distributed systems, cloud infrastructure)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. The poor R\u00b2 could stem from many factors (feature selection, model choice, outliers, data quality) but only 'more informative features or alternative modeling approaches' is briefly mentioned without substantive discussion."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It reports MAE, accuracy, F1, R\u00b2, and Recall@10 as direct metrics without framing them as proxies for broader constructs. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Sentence-BERT embeddings' without specifying which Sentence-BERT model variant. ARIMA and Prophet library versions are not specified. The language model used in the RAG demonstration is not identified at all."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The RAG demonstration (Section IV.C) shows an input bug report and retrieved examples but does not provide the actual prompt sent to the language model. The LLM used for generation is not even identified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for any model: no ARIMA (p,d,q) parameters, no Prophet configuration, no classifier hyperparameters, no Sentence-BERT parameters, no LLM temperature or sampling settings for the RAG example."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The ML models are standard classifiers and time-series models. The RAG pipeline is described at a high level but is not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section III.B provides only high-level preprocessing: 'filtered reports to exclude non-bug entries such as feature requests or tasks.' How fields were standardized across GitHub, Jira, and Bugzilla trackers is not described. No filtering counts or intermediate pipeline stages are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated Limitations or Threats to Validity section. The Conclusion and Future Work section (Section V) does not discuss limitations. The Insights paragraph briefly mentions 'challenges' but this is not a limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential biases in the dataset (e.g., survivorship bias in open-source projects), the representativeness of 9 projects for software engineering generally, or limitations of the standardization process."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge limitations such as: only English-language reports, only open-source projects, only 3 tracker types, or that the Cassandra case study may not generalize to other projects."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is publicly available at the GitHub repository (https://github.com/av9ash/gitbugs/). The paper describes it as 'openly licensed' with the full bug report data accessible for download."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B describes collection: 'RESTful API calls (for Jira-based systems like Apache projects) and HTML scraping or CSV archive parsing (for Bugzilla-based systems like Firefox and Thunderbird).' Filtering for bug-only entries is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is sourced from public bug tracking systems (GitHub, Jira, Bugzilla) for 9 named open-source projects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The collection method is described at a high level (APIs, scraping, filtering) but the full pipeline is not documented. No counts are given for how many raw entries were collected vs. how many survived filtering. How duplicate mappings were established across trackers is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": false,
         "answer": false,
         "justification": "This appears to be unfunded solo work by an independent researcher (single author with IEEE email and ORCID, no institutional affiliation listed)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The author lists only an IEEE email address and ORCID. No institutional affiliation is disclosed anywhere in the paper, making it impossible to assess potential conflicts with the projects included in the dataset."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "This appears to be unfunded solo work with no disclosed institutional backing."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. The ML models (ARIMA, Prophet, classifiers) are trained from scratch on the dataset. The RAG demonstration is illustrative, not a benchmark evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on a benchmark. The paper trains its own models on the dataset with a temporal split."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on a benchmark. Contamination concerns do not apply to this dataset paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper mines public bug repositories."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public issue trackers."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Project selection criteria (9 well-established open-source projects) are stated but not participant criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving participant assignment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a dataset paper. The case study models are illustrative demonstrations of the dataset's utility, not proposed methods requiring cost analysis."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a dataset paper. The computational cost of the illustrative case study is not the paper's contribution."
       }
     }
   },
   "claims": [
     {
       "claim": "GitBugs comprises over 150,000 bug reports from 9 open-source projects spanning GitHub, Jira, and Bugzilla.",
       "evidence": "Table I shows 196,387 total reports across 9 projects: Cassandra (4,612), Firefox (28,824), Hadoop (2,503), HBase (5,403), Mozilla Core (85,673), VS Code (32,829), Seamonkey (1,076), Spark (20,275), Thunderbird (15,192).",
       "supported": "strong"
     },
     {
       "claim": "ARIMA outperforms Prophet for bug volume forecasting on Cassandra data with MAE of 10.92 vs 20.10.",
       "evidence": "Section IV.C and Figure 4 show the comparison on the last 6 months of Cassandra data. Both MAE values are reported.",
       "supported": "moderate"
     },
     {
       "claim": "Bug priority classification achieves 82% accuracy but only 0.35 macro-F1, indicating severe class imbalance effects.",
       "evidence": "Section IV.C and Figure 5 show the confusion matrix. The Normal class achieves F1=0.90 while minority classes perform poorly.",
       "supported": "moderate"
     },
     {
       "claim": "Time-to-fix regression yields R\u00b2 = -0.09, performing worse than a naive mean predictor.",
       "evidence": "Section IV.C and Figure 6 report MAE of 86.06, RMSE of 158.44, and R\u00b2 of -0.09. The paper explicitly states the negative R\u00b2 indicates underperformance vs. mean prediction.",
       "supported": "moderate"
     },
     {
       "claim": "Sentence-BERT-based duplicate detection achieves Recall@10 of 0.61 across 300 randomly selected query bugs.",
       "evidence": "Section IV.C and Figure 9 describe the experiment: cosine similarity over Sentence-BERT embeddings, top-10 retrieval. Most similarity scores fall below 0.5.",
       "supported": "moderate"
     },
     {
       "claim": "GitBugs supports diverse research tasks including duplicate detection, triaging, resolution prediction, RAG, and temporal analysis.",
       "evidence": "Section IV demonstrates each task type with at least one experiment or example. However, most demonstrations use a single method on a single project subset.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "A dataset paper presenting itself as a benchmark resource has no dedicated limitations section. Key limitations such as English-language bias, open-source-only scope, representativeness of 9 projects, and potential selection effects in the included projects are never discussed."
     },
     {
       "flag": "No statistical rigor in case study",
       "detail": "All case study results are single-run point estimates with no confidence intervals, error bars, significance tests, or variance across runs. Claims like 'ARIMA outperformed Prophet' rest on comparing two numbers without any uncertainty quantification."
     },
     {
       "flag": "Anecdotal RAG demonstration",
       "detail": "The Retrieval-Augmented Generation example (Section IV.C) shows a single hand-picked input and output. No systematic evaluation of the RAG pipeline is conducted \u2014 no metrics, no multiple examples, and the LLM used is not even identified."
     },
     {
       "flag": "Incomplete data standardization documentation",
       "detail": "The paper does not describe how fields were harmonized across three fundamentally different issue trackers (GitHub Issues, Jira, Bugzilla). How categorical fields like Status, Priority, and Resolution were mapped across systems with different taxonomies is not documented."
     }
   ],
   "cited_papers": [
     {
       "title": "Cupid: Leveraging ChatGPT for More Accurate Duplicate Bug Report Detection",
       "authors": ["T. Zhang", "I. C. Irsan", "F. Thung", "D. Lo"],
       "year": 2023,
       "arxiv_id": "2308.10022",
       "relevance": "Evaluates ChatGPT for duplicate bug detection, directly relevant to LLM capability assessment on software engineering tasks."
     },
     {
       "title": "Can we enhance bug report quality using LLMs?: An empirical study of LLM-based bug report generation",
       "authors": ["J. Acharya", "G. Ginde"],
       "year": 2025,
       "relevance": "Empirical evaluation of LLM capabilities for improving bug report quality, relevant to LLM productivity in software engineering."
     },
     {
       "title": "ChatBR: Automated Assessment and Improvement of Bug Report Quality Using ChatGPT",
       "authors": ["L. Bo", "W. Ji", "X. Sun", "T. Zhang", "X. Wu", "Y. Wei"],
       "year": 2024,
       "relevance": "Evaluates ChatGPT for automated bug report quality assessment, relevant to LLM-based software engineering automation."
     },
     {
       "title": "An empirical study on the capability of LLMs in decomposing bug reports",
       "authors": ["Z. Chen", "V. Nava-Camal", "A. Suleiman", "Y. Tang", "D. Hou", "W. Shang"],
       "year": 2025,
       "arxiv_id": "2504.20911",
       "relevance": "Directly evaluates LLM capabilities for bug report decomposition, relevant to understanding LLM limitations in software engineering."
     },
     {
       "title": "Can LLMs demystify bug reports?",
       "authors": ["L. Plein", "T. F. Bissyand\u00e9"],
       "year": 2023,
       "arxiv_id": "2310.06310",
       "relevance": "Evaluates LLM ability to understand and explain bug reports, relevant to LLM comprehension capabilities."
     },
     {
       "title": "Evaluating diverse large language models for automatic and general bug reproduction",
       "authors": ["S. Kang", "J. Yoon", "N. Askarbekkyzy", "S. Yoo"],
       "year": 2024,
       "relevance": "Benchmark evaluation of multiple LLMs for automated bug reproduction, relevant to LLM capability and agentic workflows."
     },
     {
       "title": "Large language models are few-shot testers: Exploring LLM-based general bug reproduction",
       "authors": ["S. Kang", "J. Yoon", "S. Yoo"],
       "year": 2023,
       "relevance": "Evaluates LLMs as few-shot bug reproduction agents, directly relevant to LLM capability evaluation and code generation."
     },
     {
       "title": "Agentic bug reproduction for effective automated program repair at Google",
       "authors": ["R. Cheng", "M. Tufano", "J. Cito", "J. Cambronero", "P. Rondon", "R. Wei", "A. Sun", "S. Chandra"],
       "year": 2025,
       "arxiv_id": "2502.01821",
       "relevance": "Describes an agentic AI system for bug reproduction at Google, directly relevant to agentic workflows and automated program repair."
     },
     {
       "title": "An ensemble method for bug triaging using large language models",
       "authors": ["A. Kumar Dipongkor"],
       "year": 2024,
       "relevance": "Applies LLMs to automated bug triaging, relevant to LLM capability in software engineering automation."
     },
     {
       "title": "Graph Neural Network vs. Large Language Model: A comparative analysis for bug report priority and severity prediction",
       "authors": ["J. Acharya", "G. Ginde"],
       "year": 2024,
       "relevance": "Directly compares GNN vs LLM approaches for bug severity prediction, relevant to LLM benchmark evaluation."
     },
     {
       "title": "Method-level bug severity prediction using source code metrics and LLMs",
       "authors": ["E. Mashhadi", "H. Ahmadvand", "H. Hemmati"],
       "year": 2023,
       "relevance": "Evaluates LLMs for code-level bug severity prediction, relevant to LLM capability assessment."
     },
     {
       "title": "Aegis: An agent-based framework for bug reproduction from issue descriptions",
       "authors": ["X. Wang", "P. Gao", "X. Meng", "C. Peng", "R. Hu", "Y. Lin", "C. Gao"],
       "year": 2025,
       "relevance": "Agent-based framework for automated bug reproduction, directly relevant to agentic AI workflows in software engineering."
     },
     {
       "title": "Let's fix this together: Conversational debugging with GitHub Copilot",
       "authors": ["Y. Bajpai", "B. Chopra", "P. Biyani", "C. Aslan", "D. Coleman", "S. Gulwani", "C. Parnin", "A. Radhakrishna", "G. Soares"],
       "year": 2024,
       "relevance": "Evaluates conversational debugging with GitHub Copilot, relevant to LLM-powered developer productivity and agentic code assistance."
     },
     {
       "title": "Burt: A chatbot for interactive bug reporting",
       "authors": ["Y. Song", "J. Mahmud", "N. De Silva", "Y. Zhou", "O. Chaparro", "K. Moran", "A. Marcus", "D. Poshyvanyk"],
       "year": 2023,
       "relevance": "Chatbot system for interactive bug reporting, relevant to LLM-powered software engineering automation."
     },
     {
       "title": "BugRepro: Enhancing Android Bug Reproduction with Domain-Specific Knowledge Integration",
       "authors": ["H. Yin", "J. Huang", "Y. Li", "Y. Dong", "T. Zhang"],
       "year": 2025,
       "arxiv_id": "2505.14528",
       "relevance": "Uses domain-specific knowledge for automated bug reproduction, relevant to AI-powered software engineering and agentic workflows."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The dataset is immediately downloadable and usable for bug report analysis, duplicate detection, and ML training on realistic software engineering data."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Straightforward dataset paper that confirms known challenges (class imbalance, prediction difficulty) without challenging any conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or security concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository with data, EDA notebooks, and training scripts available for immediate use."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Covers well-known projects (Firefox, VS Code, Cassandra) but the author is an independent researcher, not from a famous lab."
     }
   }
 }