ai-research-survey

scan.json (28009B)

---

 {
   "paper": {
     "title": "Agentic Much? Adoption of Coding Agents on GitHub",
     "authors": [
       "Romain Robbes",
       "Théo Matricon",
       "Thomas Degueule",
       "Andre Hora",
       "Stefano Zacchiroli"
     ],
     "year": 2026,
     "venue": "arXiv preprint (submitted to ACM journal)",
     "arxiv_id": "2601.18341"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states in footnote 22 'we will share all the datasets and analyses that we made to facilitate this' (future tense). No working URL or archive is provided at time of publication."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper promises to share datasets (footnote 22) but provides no download link or archive. The dataset of 129,134 GitHub projects and derived metrics is not yet released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or dependency specification is provided. The paper describes the pipeline conceptually but does not list library versions or environment details needed to reproduce."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the analysis are provided. The methodology section describes the pipeline at a high level but is not a reproducible recipe."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly computes a 99% confidence interval with 1% margin of error for the commit-level sampling (Section 5.2): 'we select a large sample of non-adopters at the file level for this analysis, in order to have a 99% confidence interval and a small 1% margin of error.' Confidence intervals are also used in the qualitative commit sample (95%/96% CI with Bonferroni correction, Section 10.1)."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Chi-square goodness-of-fit tests with p-values are applied to assess whether adoption distributions are uniform across deciles in Sections 6.2 and 6.3. The paper reports 'p-value < 0.01' for all metrics."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported alongside Chi-square tests in Sections 6.2 and 6.3: 'The effect sizes are small for lines of code, contributors, commits, and issues, and medium for pull requests and age.' Effect sizes are also given contextually for commit size comparisons (e.g., median AI-assisted lines added is 34, triple the human median of 10)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Sample sizes are justified using Cochran's formula (Section 4.2.5) for the commit-level sample (n=16,000) and again using Cochran's formula for the qualitative commit sample (n=790, Section 10.1), with explicit finite population corrections and confidence level specifications."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Distributions of commit sizes are reported with quartiles (Q1, median, Q3) in Figure 11 and Figure 12 shows the relative differences across size categories. Distribution plots (boxplots/ridgeline) and Sparklines convey spread throughout the analysis."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares AI-assisted commits against human-authored commits and bot commits as natural baselines in RQ5 (Section 9). For commit type classification in RQ6, results are compared against Zeng et al.'s human-authored baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are contemporary: the comparison study by Zeng et al. [43] is from 2025, and the human and bot commit comparisons are drawn from the same 2025 time period dataset."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining study with no system components to ablate. The paper does evaluate different detection heuristics (file-based, commit-based, gitignore-based) as complementary measurement strategies, not as ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used throughout: file-level adoption rate, commit-level adoption rate, adoption ratio by project characteristic deciles, commit size (lines added, lines deleted, files changed), commit type distribution, and temporal evolution."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual human classification is performed in Section 10.2 for commits that do not follow the Conventional Commits specification: 'two authors were tasked to manually and individually classify the remaining 277 commits into one of the seven categories by inspecting their commit message and diff, with an initial agreement on 250 of the 277 commits (90%).' Inter-rater reliability is reported."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of real-world repositories, not a machine learning evaluation. There is no train/test split concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-category breakdowns are extensive: adoption is broken down by project metrics (LOC, age, contributors, commits, issues, PRs) in Tables 3 and 4; by organization in Table 5; by topic in Figures 4 and 5; by programming language in Figure 6; and by tool in Figure 8."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where heuristics fail (e.g., CONVENTIONS.md excluded due to false positive risk, AGENTS.md ambiguity), under-detection of certain agents (e.g., Codex not signing commits, developers disabling traces), and Section 11 extensively discusses reasons for over- and under-estimation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 11.2 explicitly reports reasons why the study may over-estimate adoption. The finding that 41.2% of file-level adopters have zero AI-assisted commits is reported and discussed. The low correlation (r=0.1 at best) between file-level and commit-level metrics is presented as a key negative/unexpected finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims of 15.85%-22.60% adoption, rapid adoption, breadth across project types, and larger AI commits are all supported by results in Sections 5-10. The paper appropriately hedges its estimates as conservative and high bounds."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is explicitly observational and makes no causal claims. It consistently uses language like 'adoption is associated with,' 'we find that,' and explicitly states it does not try to answer productivity or causal questions (Section 9, 'How this affects productivity is an open question, which we do not try to answer')."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Generalizations are appropriately bounded. The paper explicitly restricts claims to GitHub public repositories with at least 10 stars, 5,000 LOC, and 100 commits; acknowledges results may not translate to industry (Section 11.1); and qualifies that metrics are undercounts. Section 11.4 clearly delineates what is known vs. hypothesized."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 extensively discusses alternative explanations. For example: the high PR metric may be 'self-fulfilling' since agents contribute PRs (Section 6.2); the low commit ratios for AI/LLM topics might reflect developer awareness and deliberate hiding of traces rather than lower use (Section 11.4); and reasons for both over- and under-estimation are systematically enumerated."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study, not an LLM evaluation study. The authors study coding agents used by other developers in repositories — they do not run LLM inference themselves. No model version specification is needed."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The authors do not use LLM prompting in their analysis pipeline. The paper mines repository artifacts. No prompting is involved in the study methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM hyperparameters are involved. The study uses GitHub APIs, git log, and heuristic matching. Statistical parameters (confidence levels, margin of error) are reported in Sections 4.2.5 and 10.1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding in its analysis. It is a mining study using GitHub APIs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data preprocessing pipeline is described in detail in Section 4.2: starting with 130,621 projects, filtering to 129,134 after pipeline failures and dotfile exclusions, with explicit counts and reasons at each step. Filtering criteria are stated (not forks, ≥5,000 LOC, ≥100 commits, active in last 3 months)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 'Discussion' contains extensive dedicated subsections on limitations: 11.1 'Limitations', 11.2 'Reasons for over-estimating coding agent use', and 11.3 'Reasons for under-estimating coding agent use', spanning several pages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: the Peril of Partial Observability (projects with file markers but no commit markers), the specific bias introduced by squash merges, the impossibility of attributing AGENTS.md to Codex with certainty, and the specific finding that AI/LLM topic projects likely hide their traces deliberately. Not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Scope boundaries are explicitly stated throughout: results apply only to open-source GitHub repositories meeting specific minimum criteria (10 stars, 5,000 LOC, 100 commits, active); the paper explicitly states results 'may not translate well to more general use, and to industry in particular' (Section 11.1); estimates are explicitly labeled as conservative vs. high bounds."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available at time of publication. The paper promises to share datasets (footnote 22) but provides no download link, DOI, or archive."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail in Section 4.2: the sampling tool by Dabic et al. [10] is used, with explicit criteria, accessed on 29/08/2025. The GitHub REST and GraphQL APIs are described, and the analysis was run on October 31st, 2025."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study with no human participants. The 'sample' is GitHub repositories satisfying objective criteria, not recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented in Section 4.2 with explicit step numbering and counts: from 130,621 initial projects to 129,134 analyzed, with 900 pipeline failures and 500 dotfile exclusions explained. Commit extraction steps are numbered 1-6 in Section 4.2.5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper text. No funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed on the title page: Univ. Bordeaux, Univ. Rennes/Inria, UFMG (Brazil), and Institut Polytechnique de Paris. None are affiliated with the coding agent companies whose tools are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The authors are researchers at CNRS, Inria, UFMG, and Institut Polytechnique de Paris — institutions that routinely fund research through government grants. The absence of a funding disclosure does not mean the work is unfunded; it means we cannot verify funder independence. Since the paper should have disclosed funding and did not, the criterion applies but is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Absence of disclosure is not evidence of absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study that does not evaluate a pre-trained model's capability on any benchmark. No LLM is evaluated for knowledge or capability; the study observes adoption traces left by coding agents."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a mining study, not a benchmark evaluation of an LLM."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not use a benchmark to evaluate LLM knowledge or capabilities."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study. Mining public GitHub repositories is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Mining public GitHub repository data does not require IRB approval. No human subjects are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The repository selection criteria are clearly stated but these are not participant inclusion/exclusion criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The random sampling of repositories uses stratified statistical principles (Cochran's formula), which is documented in Section 4.2.5."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The two-author manual classification in Section 10.2 did not use blinding, but this is a qualitative coding exercise, not a human subjects experiment."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not report the cost of the API calls to GitHub APIs or the computational cost of the pipeline. Section 4.2.4 mentions that 'the whole analysis still requires more than half a terabyte in total' of storage but does not report time or API cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. Storage requirements (>0.5 TB) are mentioned but not time or hardware specifications."
       }
     }
   },
   "claims": [
     {
       "claim": "Between 15.85% and 22.60% of a large sample of GitHub projects (129,134 projects) show traces of coding agent adoption as of October 2025.",
       "evidence": "Section 5 (RQ1) and Table 1 present the full estimation: 7.89% file-level detection + extrapolated 8.64% commit-level detection in non-file adopters = 15.85% conservative estimate; 22.60% high estimate. Statistics computed with 99% CI and 1% margin of error.",
       "supported": "strong"
     },
     {
       "claim": "AI-assisted commits are substantially larger than human-authored commits: the median added lines for AI-assisted commits is 34, triple the human median of 10.",
       "evidence": "Section 9 (RQ5) and Figure 11 show distribution of commit sizes by author type (human n=3.1M, AI n=171K, bot n=285K). The median comparison (34 vs 10 vs 3) is clearly stated with quartiles.",
       "supported": "strong"
     },
     {
       "claim": "Coding agent adoption is broad across project types, organizations, and programming languages, with very few categories having near-zero adoption.",
       "evidence": "Sections 7 and RQ3 analyze adoption by topic (Figure 4), organization (Table 5), and programming language (Figure 6). Most topics and languages show adoption within ±10% of each other.",
       "supported": "strong"
     },
     {
       "claim": "Feature implementation (feat) and bug fixes (fix) account for two-thirds (65.6%) of AI-assisted commits, compared to ~44% for human-authored commits.",
       "evidence": "Table 7 (Section 10.3) shows feat=35.7% and fix=29.9% in 790 sampled Claude Code commits. Comparison to Zeng et al.'s human-authored baseline (feat=17%, fix=27%) is made directly.",
       "supported": "strong"
     },
     {
       "claim": "Younger projects (≤1 year old) show dramatically higher file-level coding agent adoption (21.17%) compared to the oldest projects (>11 years, 4.69%).",
       "evidence": "Table 3 (Section 6.2) shows adoption by age decile. Chi-square test p<0.01 with medium effect size for age.",
       "supported": "strong"
     },
     {
       "claim": "Claude Code and GitHub Copilot together account for more than half of all detected coding agent adoption, with Claude Code being the single most popular tool (4,896 projects).",
       "evidence": "Section 8.2 and Figure 8 present per-tool adoption counts. The paper states 'Just Claude and Copilot are responsible for more than half of the adoption.'",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "observational"
   ],
   "key_findings": "This large-scale mining study of 129,134 GitHub projects finds that between 15.85% and 22.60% of projects show traces of coding agent adoption as of October 2025 — a remarkably rapid adoption for a technology category that only launched in 2024. Adoption is broad across project types, sizes, organizations, and programming languages, though with a strong bias toward younger projects. AI-assisted commits are substantially larger than human-authored ones (median 34 lines added vs. 10) and skew heavily toward feature implementation and bug fixes rather than maintenance. The study likely undercounts true adoption because agents can be configured to hide their traces.",
   "red_flags": [
     {
       "flag": "Heuristic-based detection with potential false positives/negatives",
       "detail": "The paper's detection relies on file-based, commit-based, branch-based, and label-based heuristics. By the authors' own admission, these may produce false positives (e.g., projects that once had agent files but abandoned agents) and false negatives (e.g., agents configured not to leave traces). The true adoption rate is bounded by the conservative and high estimates, but the true value is unknown."
     },
     {
       "flag": "Data not yet released",
       "detail": "The paper promises to share datasets and analysis code (footnote 22) but provides no actual download link or archive. The study cannot currently be independently replicated or verified."
     },
     {
       "flag": "RQ6 qualitative analysis limited to Claude Code commits only",
       "detail": "The commit type analysis in Section 10 is based solely on a sample of 790 Claude Code commits, chosen because Claude Code is the most popular agent. This sampling choice may not be representative of commit types for other agents (e.g., Copilot, Codex, Cursor), which may be used for different tasks."
     },
     {
       "flag": "No funding disclosure",
       "detail": "No funding source or acknowledgments section is present in the paper. While author affiliations suggest no commercial conflicts, the absence of disclosure means this cannot be verified."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": [
         "Joel Becker",
         "Nate Rush",
         "Elizabeth Barnes",
         "David Rein"
       ],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT measuring coding agent (Cursor) impact on developer productivity — directly relevant to the survey scope on agentic AI evaluation."
     },
     {
       "title": "Sharp Tools: How Developers Wield Agentic AI in Real Software Engineering Tasks",
       "authors": [
         "Aayush Kumar",
         "Yasharth Bajpai",
         "Sumit Gulwani",
         "Gustavo Soares",
         "Emerson Murphy-Hill"
       ],
       "year": 2025,
       "relevance": "Observational study of developers using agentic AI (Cursor) for real software engineering tasks, studying barriers and successes."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": [
         "Carlos E. Jimenez",
         "John Yang",
         "Alexander Wettig",
         "Shunyu Yao",
         "Kexin Pei",
         "Ofir Press",
         "Karthik R. Narasimhan"
       ],
       "year": 2024,
       "relevance": "Foundational benchmark for evaluating coding agent capability on real GitHub issues — widely used in the agentic AI evaluation literature."
     },
     {
       "title": "Speed at the Cost of Quality? The Impact of LLM Agent Assistance on Software Development",
       "authors": [
         "Hao He",
         "Courtney Miller",
         "Shyam Agarwal",
         "Christian Kästner",
         "Bogdan Vasilescu"
       ],
       "year": 2025,
       "arxiv_id": "2511.04427",
       "relevance": "Empirical study of the impact of LLM agent (Cursor) adoption on software quality using difference-in-differences design."
     },
     {
       "title": "Understanding Software Engineering Agents: A Study of Thought-Action-Result Trajectories",
       "authors": [
         "Islem Bouzenia",
         "Michael Pradel"
       ],
       "year": 2025,
       "arxiv_id": "2506.18824",
       "relevance": "Studies the traces of coding agent interactions on SWE-bench, analyzing diversity of agent trajectories and failure modes."
     },
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "authors": [
         "Sida Peng",
         "Eirini Kalliamvakou",
         "Peter Cihon",
         "Mert Demirer"
       ],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Controlled experiment finding 55% productivity increase from Copilot — a key reference for evaluating coding assistant impact claims."
     },
     {
       "title": "AIDev: Studying AI Coding Agents on GitHub",
       "authors": [
         "Hao Li",
         "Haoxiang Zhang",
         "Ahmed E. Hassan"
       ],
       "year": 2025,
       "relevance": "Concurrent MSR study of coding agent pull requests on GitHub — directly comparable methodology to the paper being scanned."
     },
     {
       "title": "Context Engineering for AI Agents in Open-Source Software",
       "authors": [
         "Seyedmoein Mohsenimofidi",
         "Matthias Galster",
         "Christoph Treude",
         "Sebastian Baltes"
       ],
       "year": 2025,
       "arxiv_id": "2510.21413",
       "relevance": "Qualitative study of coding agent guidance file contents in open-source repositories — directly extends the analysis in this paper."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": [
         "Chunqiu Steven Xia",
         "Yinlin Deng",
         "Soren Dunn",
         "Lingming Zhang"
       ],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Presents an agentic approach for SWE-bench, evaluating cost and performance tradeoffs of coding agents."
     },
     {
       "title": "Promises, Perils, and (Timely) Heuristics for Mining Coding Agent Activity",
       "authors": [
         "Romain Robbes",
         "Théo Matricon",
         "Thomas Degueule",
         "Andre Hora",
         "Stefano Zacchiroli"
       ],
       "year": 2025,
       "relevance": "Companion paper by the same authors describing the heuristics used for detecting coding agent activity in repositories — foundational for the current study's methodology."
     },
     {
       "title": "Measuring AI ability to complete long tasks",
       "authors": [
         "Thomas Kwa",
         "Ben West",
         "Joel Becker"
       ],
       "year": 2025,
       "arxiv_id": "2503.14499",
       "relevance": "Evaluates the ability of AI systems to complete progressively longer tasks, with findings about task completion time doubling every 7 months — cited as evidence for increasing agent capability."
     },
     {
       "title": "A Large-Scale Survey on the Usability of AI Programming Assistants: Successes and Challenges",
       "authors": [
         "Jenny T Liang",
         "Chenyang Yang",
         "Brad A Myers"
       ],
       "year": 2024,
       "relevance": "Survey of 410 developers on AI coding assistant use cases and barriers — relevant to understanding developer behavior with coding tools."
     },
     {
       "title": "Unveiling ChatGPT's Usage in Open Source Projects: A Mining-based Study",
       "authors": [
         "Rosalia Tufano",
         "Antonio Mastropaolo",
         "Federica Pepe",
         "Ozren Dabic",
         "Massimiliano Di Penta",
         "Gabriele Bavota"
       ],
       "year": 2024,
       "arxiv_id": "2402.16480",
       "relevance": "MSR study of ChatGPT usage traces in GitHub — predecessor methodology to the current study of coding agent adoption."
     }
   ]
 }