ai-research-survey

calibration.json (16221B)

---

 {
   "calibration": {
     "paper_slug": "agentic-bug-reproduction-2025",
     "calibration_date": "2026-02-28",
     "sonnet_scan_date": "2026-02-28",
     "model": "opus",
     "agreement_rate": 1.0,
     "total_questions": 50,
     "agreements": 50,
     "disagreements": 0,
     "disagreement_details": []
   },
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided anywhere in the paper. BRT Agent and the adapted LIBRO run on Google's internal infrastructure with no public release mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 80-bug evaluation dataset comes from Google's internal issue tracking system (GITS) and is proprietary. No public dataset download link is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or dependency versions are provided. The paper references Google-internal infrastructure (Bazel, GITS) without specifying reproducible environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The entire experimental setup relies on Google-internal tools, datasets, and models not accessible to outside researchers."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates only (e.g., '28% plausible BRT generation rate', '70% precision at K=1'). Despite performing 20-50 runs per bug, no confidence intervals or error bars are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used when comparing BRT Agent (28%) vs LIBRO (10%) or any other comparison in the paper. No p-values, t-tests, or other hypothesis tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw percentage differences are given (28% vs 10%, 30% more bugs fixed) but no formal effect size measures such as Cohen's d or odds ratios are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 80-bug dataset is acknowledged in §7 as 'a relatively small dataset' that 'could limit the generalizability of our results,' but no power analysis or quantitative justification for the sample size is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite running LIBRO 50 times and BRT Agent 20 times per bug, no standard deviation, variance, IQR, or any spread measure is reported in Table 2, Table 3, or anywhere else. Only aggregate pass/fail rates are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "LIBRO is used as the primary baseline comparison throughout the paper (Table 2, Table 3). SWE-Agent+ is discussed as related work but not directly evaluated on the same dataset."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LIBRO (ICSE 2023, TSE 2024 extension) is a contemporary baseline in BRT generation. SWE-Agent+ (NeurIPS 2024) is also cited as state-of-the-art and discussed in section 3."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. Individual components of BRT Agent (fine-tuned code-editing LLM, agentic scaffolding, ReAct loop, code_search action) are not systematically isolated to measure their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 uses Candidate BRTs, Plausible BRTs, and Candidate-to-Plausible rate. RQ3 uses Precision, Recall, F1-score, and MRR. Multiple metrics are used across the evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes manual inspection: 'two authors inspect the BRTs and a third author resolves any disagreement.' This constitutes human evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 states the code-editing LLM's 'training data cutoff predates the reporting of all bugs analyzed in this study.' All bugs are from June 2024 or later, ensuring no training overlap."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides a breakdown of plausible BRT rates by programming language (Java, C++, Go, Python, Kotlin, Dart, TypeScript) for both LIBRO and BRT Agent."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses failure modes: LIBRO's main failure is build errors from which it cannot recover. BRT Agent failures include step exhaustion (21%), irrelevant assertions (16%), invalid patches modifying existing tests (11%), and hallucinated non-existent actions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Dart achieves 0% plausible BRTs for both techniques; LIBRO drops from 33% on Defects4J to 10% on Google data; the agent hallucinates actions not in its action set; 11% of plausible BRT patches are invalid."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are confirmed: 28% vs 10% (Table 2), 30% more bugs with plausible fixes (§6.2: 17/23 vs 13/23), 70% precision at top-1 (§6.3.1, Figure 5). All major abstract claims match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'providing BRTs to Passerine results in 30% more bugs with plausible fixes' — a causal claim. The comparison is with vs without BRT on the same 23 bugs (a selected subset where BRT Agent succeeded), but there is no randomization, no discussion of confounds, and no acknowledgment that the subset selection could bias results."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: 'our study focuses exclusively on Google's internal development environment' (§7) and 'the generalizability of our findings to other industrial settings requires further investigation.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Threats to Validity section (§7) lists methodological limitations but does not discuss alternative explanations for the observed results. For example, it does not consider whether the fine-tuned Gemini model alone (vs the agentic scaffolding) explains the improvement, or whether the 23-bug subset for RQ2 is biased."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'a Gemini model fine-tuned on Google's internal code' and 'a publicly available Gemini' (§5.1.2) without specifying which Gemini version (e.g., 1.5 Pro, 2.0) or any snapshot/API date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language (§4.2.3, §4.1) and mentions 'three synthetic BRT generation examples in the system prompt' but does not provide the actual prompt text. Only a meta task description fragment is quoted. The reader cannot reconstruct the prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.2 reports temperature (0.7 for LIBRO, 0.2 for BRT Agent), top-P (0.95 for both), number of runs (50 for LIBRO, 20 for BRT Agent), and max steps (25 for BRT Agent)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the ReAct loop, Table 1 details all available actions (cat, code_search, edit, bazel test, finish), the dual-LLM architecture (reasoning vs code-editing), the change description mechanism, and termination conditions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 documents the dataset construction: automated extraction and filtering, followed by manual curation ensuring fixes address root causes. The paper references [30] for full pipeline details and explains the 80 vs 78 bug discrepancy in a footnote."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides substantive discussion across internal validity, external validity, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: small 80-bug dataset limiting generalizability, implementation bias from LIBRO adaptation, same Gemini model family for both techniques, LLM randomness, and EPR as an indirect proxy for fix correctness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states results are for Google's internal environment only, findings may not generalize to other industrial settings, and metrics 'may not fully capture all aspects of a BRT, such as its readability, maintainability' (§7)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 80-bug evaluation dataset from Google's internal issue tracker (GITS) is proprietary and not publicly available. Raw data cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 describes data collection: bugs from GITS, reported and fixed by human developers, from diverse Google projects, all recent since June 2024, spanning 7 languages. Dataset constructed via automated extraction, filtering, and manual curation [30]."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is Google's internal issue tracker, not participant recruitment. Bug selection is automated and curated, not recruited."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is described: automated extraction → filtering → manual curation, with criteria (fixes verified to address root cause). A footnote explains the 80 vs 78 bug discrepancy with concurrent work [30]."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding statement is present. A footnote notes Cheng and Cito conducted research at Google, but no explicit funding disclosure exists."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: 6 of 8 authors are from Google. A footnote explicitly states Cheng and Cito conducted this research at Google. The corporate connection is transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Research conducted at Google, evaluating Google's own APR system (Passerine) and fine-tuned Gemini models on Google's proprietary data. Google has a direct financial interest in positive results for its tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations appear in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 states the code-editing LLM's 'training data cutoff predates the reporting of all bugs analyzed in this study,' addressing the training-data boundary."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 explicitly addresses this: 'the code-editing LLM's training data excludes all bugs, code changes, and BRTs used in our empirical evaluation—its training data cutoff predates the reporting of all bugs analyzed in this study, preventing any potential data leakage.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses internal Google bugs reported since June 2024, which postdate the training cutoff of all models used. Contamination risk is directly addressed in §4.2.3."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a software engineering evaluation study using automated tools on an internal bug dataset."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. The study evaluates automated tools on an internal bug dataset."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The manual inspectors (§6.1) are research team members, not recruited participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Bug selection criteria are covered under data collection."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participant randomization applicable. The study evaluates automated tools, not a controlled experiment with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participant blinding applicable. Manual inspection in §6.1 uses two inspectors with a third for disagreements, but this is not a blinded study design."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency figures are reported. Running 80 bugs * 20 runs * up to 25 steps each with Gemini models represents substantial usage, but costs are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or API spend is reported. The computational resources required for 50 LIBRO runs and 20 BRT Agent runs per bug on 80 bugs are not quantified."
       }
     }
   }
 }