ai-research-survey

calibration.json (17356B)

---

 {
   "paper_slug": "agentic-adoption-github-2026",
   "calibration_model": "opus",
   "scan_model": "sonnet",
   "timestamp": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "conflicts_of_interest",
       "question": "funder_independent_of_outcome",
       "sonnet": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed; applicability cannot be assessed. The authors appear to be academic researchers without commercial ties to the evaluated products."
       },
       "opus": {
         "applies": true,
         "answer": false,
         "justification": "The authors are researchers at CNRS, Inria, UFMG, and Institut Polytechnique de Paris — institutions that routinely fund research through government grants. The absence of a funding disclosure does not mean the work is unfunded; it means we cannot verify funder independence. Since the paper should have disclosed funding and did not, applies=true and answer=false is appropriate."
       },
       "direction": "applies_boundary",
       "explanation": "Sonnet treats the absence of funding disclosure as evidence the paper is unfunded (applies=false). Opus reasons that academic researchers at major public research institutions almost certainly receive institutional or grant funding, and the absence of disclosure is itself a deficiency — the question is applicable but unanswerable, hence applies=true, answer=false."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Footnote 22 states 'we will share all the datasets and analyses that we made to facilitate this' in future tense. No working URL or archive is provided at time of publication."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper promises future release (footnote 22) but provides no download link, DOI, or archive. The dataset of 129,134 GitHub projects and derived metrics is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or dependency specification is provided. The pipeline is described conceptually but without environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Section 4.2 describes the pipeline at a high level but does not constitute a reproducible recipe with commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 explicitly computes a 99% CI with 1% margin of error for commit-level sampling using Cochran's formula. Section 10.1 uses 95%/96% CI with Bonferroni correction for the qualitative commit sample."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Chi-square goodness-of-fit tests are applied in Sections 6.2 and 6.3 to assess distribution uniformity across deciles, with all p-values < 0.01."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported alongside Chi-square tests in Sections 6.2 and 6.3: 'The effect sizes are small for lines of code, contributors, commits, and issues, and medium for pull requests and age.' Contextual effect sizes for commit comparisons (median 34 vs 10 added lines) are also provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Sample sizes are justified using Cochran's formula (Section 4.2.5) with explicit parameters: N=120,000, p=0.5, 99% CI, 1% margin yields n=14,575, increased to 16,000. Section 10.1 similarly justifies n=790 using Cochran's formula with finite population correction."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Distributions are reported with quartiles (Q1, median, Q3) in Figure 11. Ridgeline plots and sparklines convey spread throughout. Interquartile ranges are visible for commit size comparisons across human, bot, and AI-assisted categories."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human-authored commits serve as natural baselines in RQ5 (Section 9) for commit size comparison. Zeng et al.'s human-authored commit type baseline is used in RQ6 (Section 10.3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Zeng et al. [43] is from 2025. Human and bot commit comparisons are drawn from the same 2025 time-period dataset."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining study with no system components to ablate. The study evaluates different detection heuristic categories but these are complementary measurement strategies, not ablatable components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: file-level adoption rate, commit-level adoption rate, adoption by project characteristic deciles, commit size (lines added, deleted, files changed), commit type distribution, temporal evolution, and co-adoption counts."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 10.2: Two authors manually classified 277 commits into seven categories by inspecting commit messages and diffs, with 90% initial agreement (250/277). The remaining 27 were resolved through discussion."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of real-world repositories. There is no machine learning model being evaluated and no train/test split concept applies."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns: by LOC, age, contributors, commits, issues, PRs (Tables 3-4), by organization (Table 5), by topic (Figures 4-5), by programming language (Figure 6), and by tool (Figure 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 extensively discusses where heuristics fail: CONVENTIONS.md excluded due to false positive risk, AGENTS.md attribution ambiguity, developers disabling traces, squash merge imprecision, and systematic under- and over-estimation reasons."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative/unexpected findings reported: 41.2% of file-level adopters have zero AI-assisted commits; very low correlation (r=0.1 at best) between file and commit metrics; AI/LLM topic projects having lower commit ratios despite higher file-level adoption."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims — 15.85%-22.60% adoption, rapid adoption, breadth across project types/languages/organizations, larger AI commits — are supported by results in Sections 5-10 with appropriate hedging."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is explicitly observational and avoids causal claims. It uses language like 'we find that,' 'we observe,' and explicitly states 'How this affects productivity is an open question, which we do not try to answer' (Section 9). Hedged speculations (e.g., 'likely raised the popularity') are not presented as findings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are explicitly bounded to public GitHub repositories with >=10 stars, >=5,000 LOC, >=100 commits. Section 11.1 acknowledges results 'may not translate well to more general use, and to industry in particular.' Estimates are labeled as conservative vs. high bounds."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 systematically discusses alternatives: high PR metric may be 'self-fulfilling'; low commit ratios for AI/LLM topics likely reflect deliberate trace-hiding rather than lower use; extensive enumeration of over- and under-estimation reasons including developer behavior, agent configuration, and workflow differences."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study that does not run LLM inference. The authors study coding agents used by other developers in repositories; they do not invoke any LLM themselves."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The study does not use LLM prompting. The analysis pipeline mines repository artifacts using GitHub APIs and git commands."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM hyperparameters are involved. The study uses GitHub REST and GraphQL APIs, git log, and heuristic pattern matching. Statistical parameters (confidence levels, margins of error) are reported separately."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in the analysis methodology. This is a mining study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents the pipeline in detail: 130,621 initial projects filtered to 129,134, with 900 pipeline failures and 500 dotfile exclusions explained with counts. Commit extraction is documented in 6 numbered steps (Section 4.2.5). Filtering criteria are explicit."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 'Discussion' contains dedicated subsections: 11.1 'Limitations', 11.2 'Reasons for over-estimating coding agent use', 11.3 'Reasons for under-estimating coding agent use', and 11.4 'Putting it all together', spanning multiple pages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: Peril of Partial Observability (file markers without commit markers), squash merge attribution bias, AGENTS.md attribution uncertainty, AI/LLM topic projects deliberately hiding traces, GitHub Linguist file-extension limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope restrictions: GitHub public repositories with >=10 stars, >=5,000 LOC, >=100 commits, active in last 3 months. The paper states results 'may not translate well to more general use, and to industry in particular' (Section 11.1). Conservative and high estimates are explicitly labeled."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available at time of publication. Footnote 22 promises future sharing of datasets and analyses but provides no current download link, DOI, or archive."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in Section 4.2: the sampling tool by Dabic et al. [10] is used with explicit criteria, accessed on 29/08/2025. GitHub REST and GraphQL APIs are described. Analysis run on October 31st, 2025."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study with no human participants. The 'sample' is GitHub repositories satisfying objective inclusion criteria, not recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in Section 4.2 with explicit step counts: 130,621 initial -> 129,134 analyzed (900 failures, 500 dotfiles). Commit extraction documented in 6 numbered steps in Section 4.2.5. Each filtering stage has stated criteria and counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. No funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed on the title page: Univ. Bordeaux/CNRS, Univ. Rennes/Inria/CNRS/IRISA, UFMG, and Institut Polytechnique de Paris. None are affiliated with coding agent companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The authors are researchers at CNRS, Inria, UFMG, and Institut Polytechnique de Paris — institutions that routinely fund research through government grants. The absence of a funding disclosure does not mean the work is unfunded; it means we cannot verify funder independence. Since the paper should have disclosed funding and did not, the criterion applies but is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study that does not evaluate a pre-trained model's capability on any benchmark. No LLM is tested for knowledge or capability."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a mining study, not a benchmark evaluation of an LLM."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not use a benchmark to evaluate LLM knowledge or capabilities."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study. Mining public GitHub repositories is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Mining public GitHub repository data does not require IRB approval. No human subjects are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Repository selection criteria are stated but these are not participant inclusion/exclusion criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The random sampling of repositories uses statistical principles (Cochran's formula) documented in Section 4.2.5, but this is not human randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The two-author manual classification in Section 10.2 is a qualitative coding exercise, not a human subjects experiment."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions storage requirements ('more than half a terabyte in total' in Section 4.2.4) but does not report time, API cost, or per-unit analysis cost for the mining pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Storage requirements (>0.5 TB) are mentioned but total computational budget, hardware specifications, and wall-clock time are not stated."
       }
     }
   }
 }