ai-research-survey

scan-v5.json (27266B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Are Coding Agents Generating Over-Mocked Tests? An Empirical Study",
     "authors": [
       "Andre Hora",
       "Romain Robbes"
     ],
     "year": 2026,
     "venue": "MSR '26",
     "arxiv_id": "2602.00409",
     "doi": "10.1145/3793302.3793362"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All quantitative claims in the abstract (60%, 23% vs 13%, 68%, 36% vs 26%, 95% mock type concentration) are directly backed by contingency tables and statistical tests in Section 3.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes observational association claims ('more likely to') rather than causal claims; the study design is a mining study without intervention and the authors consistently use correlational language throughout.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states 'our findings cannot be directly generalized to repositories written in other languages or using other agents,' bounding scope to Python, JavaScript, and TypeScript in 2025.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss key alternative explanations for why agents mock more — e.g., selection effects (agent-adopting repos may have more complex code requiring more mocking) or developer-preference confounds; only the 'easier to generate automatically' hypothesis is briefly proposed without evaluation.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between what is measured (presence of mock identifiers in test commit diffs, validated at 94% precision) and what is claimed (mocking frequency tendencies of coding agents), and does not conflate commit counts with test quality.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' provides a dedicated limitations discussion covering detection precision, agent commit attribution, and generalization.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are quantified: 94% precision for mock detection (manually inspected 100 commits across 10 repositories), 100% precision for agent commit detection (500 manually inspected commits), and handling of Co-Authored-By variant casing.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope to three languages, three specific coding agents, commits from 2025, and repositories meeting stated criteria (≥100 commits, ≥5,000 non-blank LOC, not forks, recently active).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments disclose funding from CNPq grants (408817/2024-0 and 403304/2025-3), CAPES, FAPEMIG, INES.IA, and the French State/IdEx université de Bordeaux.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Hora at UFMG (Brazil) and Robbes at Univ. Bordeaux, CNRS, Bordeaux INP, LaBRI (France).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from government and academic agencies (CNPq, CAPES, French State) with no affiliation to the coding agent companies (Anthropic, GitHub, Cursor) whose products are studied.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests (patents, equity, consulting) is included anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are explicitly defined: 'coding agents' (Section 2.1.1 — autonomous tools that invoke external tools, execute code, and author commits), 'test doubles/mocks' (Section 2.6 — Meszaros taxonomy: dummy, stub, spy, mock, fake), and 'agent commits', 'test commits', 'mock commits' are operationally defined.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Contributions are explicitly stated: '(1) the first empirical study to analyze agent-generated tests in real-world software systems; and (2) multiple actionable implications for practitioners and researchers.'",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 engages substantively with prior work on coding agents (Becker et al., Kumar et al., Bouzenia & Pradel), LLM-generated test quality (Alshahwan et al., Ouédraogo et al.), and mocking practices (Spadini et al., Qin 2025), positioning this study as the first to examine mocking in agent-generated code at scale in the wild.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.7 explicitly states 'Our scripts and dataset are publicly available at: https://doi.org/10.5281/zenodo.17427638.'",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The dataset (commits and repository metadata) is publicly available on Zenodo at the stated DOI.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions using PyDriller and GitEvo but provides no requirements.txt, Dockerfile, or specific version numbers for any dependency.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper provides a Zenodo link for scripts but includes no step-by-step reproduction instructions; reproducing the pipeline would require inferring the full workflow from the methodology description.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported for any headline percentages (23%, 36%, etc.); only Chi-squared statistics, p-values, and Cliff's delta are provided.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Chi-squared tests of independence are applied for commit-level analyses in RQ1 and RQ2; paired Wilcoxon tests (with normality confirmed via Shapiro-Wilk and D'Agostino) are used for repository-level comparisons.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Cliff's delta effect sizes are reported for both repository-level comparisons: negligible for lower agentic activity repositories and small (0.252) for higher agentic activity.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No power analysis or principled sample size justification is provided; the sample of 2,168 repositories emerges from SEART selection criteria rather than any prospective sizing calculation.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Table 10 reports medians for repository-level mock commit ratios but provides no standard deviations, interquartile ranges, or other spread measures for any result.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Non-agent commits serve as the direct baseline throughout all three RQs, with explicit agent vs. non-agent proportions in every contingency table.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Non-agent commits are drawn from the same repositories and same time period (2025) as agent commits, making comparisons directly contemporary.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": false,
           "answer": false,
           "justification": "This is an observational mining study, not a system design paper; ablation analysis is not applicable.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses commit-level ratios, repository-level proportions, Chi-squared statistics with standardized residuals, Wilcoxon p-values, Cliff's delta, and mock type distribution across all five test double categories.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Authors manually inspected 500 agent commits to validate classifier precision (100%) and 100 randomly selected mock commits across 10 repositories to validate mock detection precision (94%).",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is an observational mining study, not a prediction task; held-out test sets are not applicable.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by programming language (Python vs JS/TS) in Tables 5, 8, and 10, and by individual coding agent (Claude, Copilot, Cursor) in Tables 5 and 8.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The browser-use example where agents added mocks despite explicit configuration to the contrary is a descriptive observation, not a systematic discussion of failure modes or when the methodology breaks down.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports null results: no notable language difference in mock rates (Python 37% vs JS/TS 35%), and negligible Cliff's delta for lower-agentic-activity repositories despite a statistically significant Wilcoxon result.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": false,
           "answer": false,
           "justification": "This is a mining study that does not run LLM inference; the specific versions of Claude Code, Copilot, and Cursor active during studied commits cannot be determined from commit metadata and are not reported.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "No LLMs are invoked in the authors' analysis pipeline; the study mines existing commit data rather than querying models.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": false,
           "answer": false,
           "justification": "No models are run by the authors; hyperparameters are not applicable to a repository mining study.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "The authors do not deploy agentic scaffolding; they analyze traces left by existing coding agents in real repositories.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Preprocessing is thoroughly documented: SEART selection criteria (Section 2.2), agent file detection patterns (Table 1), commit author/co-author matching logic (Section 2.4), test file patterns (Table 2), mock identifier detection rules (Section 2.6.1), and mock commit classification (Section 2.6.2) are all fully specified.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The dataset is publicly available at Zenodo (doi.org/10.5281/zenodo.17427638) as explicitly stated in Section 2.7.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection is thoroughly described: SEART tool and selection criteria, filtering from 114,098 to 2,168 repositories, cloning for agent file detection, and commit metadata parsing for all three classification steps.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; this is a repository mining study using automated collection from GitHub via the SEART tool.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline is documented across Sections 2.2–2.7: SEART selection → language/agent filter → agent commit detection → test commit detection → mock commit detection → RQ analysis, including the tools used (PyDriller, GitEvo).",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "This is not a benchmark evaluation of model capabilities; no models are evaluated on test sets.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable; this is a mining study, not a model capability evaluation.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable; no benchmarks are used for model evaluation.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; inclusion/exclusion criteria apply to repositories and are fully documented in Section 2.2.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": false,
           "answer": false,
           "justification": "No LLM inference is performed by the authors; this is a repository mining study.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not report the computational cost of cloning and analyzing 2,168 repositories and 1.2 million commits, which is non-trivial.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "23% of commits made by coding agents add or modify test files, compared with 13% for non-agents",
       "evidence": "Table 4: 11,035/48,563 agent commits are test commits vs 158,326/1,206,315 non-agent commits; Chi-squared = 3,683.06, p < 0.001, standardized residual = 55.35",
       "supported": "strong"
     },
     {
       "claim": "60% of repositories with agent activity also contain agent test activity",
       "evidence": "Table 4 and Section 3.1: 729 out of 1,219 repositories with agent commits also contain agent test commits",
       "supported": "strong"
     },
     {
       "claim": "36% of test commits made by coding agents add mocks, compared with 26% for non-agents",
       "evidence": "Table 7: 3,934/11,035 agent test commits are mock commits vs 40,966/158,326 non-agent test commits; Chi-squared = 505.5, p < 0.001",
       "supported": "strong"
     },
     {
       "claim": "In repositories with higher agentic activity (≥50 agent commits), agents have a significantly higher mock ratio (36%) than non-agents (28%) with small effect size",
       "evidence": "Table 10b: Wilcoxon p < 0.001, Cliff's delta = 0.252 across 179 repositories; lower-agentic-activity repos show negligible effect despite statistical significance",
       "supported": "moderate"
     },
     {
       "claim": "Coding agents predominantly use the 'mock' type (95%) while non-agents use a wider variety — fake (57%), spy (51%), mock (91%)",
       "evidence": "Figure 5: Distribution of mock types across 496 repositories with agent mock activity; agents are concentrated on the generic mock type while non-agents show broader distribution",
       "supported": "strong"
     },
     {
       "claim": "Repositories created in 2025 show a higher share of agent test commits (17%) and mock commits (19%) compared to the full dataset (7% and 9%)",
       "evidence": "Tables 6 and 9: For 2025-created repos, 4,526/26,654 test commits are agent commits (17%) and 1,529/7,855 mock commits are agent commits (19%)",
       "supported": "strong"
     },
     {
       "claim": "Mock-related instructions in agent configuration files are far less common than test instructions, suggesting a guidance gap",
       "evidence": "Table 12: GitHub Code Search finds 13k CLAUDE.md files with 'mock' vs 102k with 'test' out of 112k total; causal link between guidance and behavior not established",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "observational",
     "case-study"
   ],
   "key_findings": "Coding agents are significantly more likely to modify test files (23% vs 13% of commits) and add mocks to those tests (36% vs 26%) than non-agent contributors, with both differences statistically significant (p < 0.001) and the mock difference confirmed in a paired within-repository analysis (Cliff's delta = 0.252, small). Agents show markedly less diversity in test double types, relying almost exclusively on the generic 'mock' type (95%) compared to non-agents who also commonly use 'fake' (57%) and 'spy' (51%). The proportion of agent-generated tests and mocks is growing rapidly, accounting for 17–19% of recently created repositories' test/mock commits vs 7–9% overall. The paper finds that mock guidance in agent configuration files (e.g., CLAUDE.md) is uncommon, and agents occasionally add mocks even in repositories that explicitly prohibit it, suggesting configuration-based guidance has limited enforcement.",
   "red_flags": [
     {
       "flag": "Title implies quality judgment not demonstrated",
       "detail": "The paper establishes that agents mock more frequently but cannot demonstrate this constitutes 'over-mocking' — no assessment of mock appropriateness, test effectiveness, bug-detection rates, or maintenance cost is included; the normative claim in the title exceeds the observational evidence."
     },
     {
       "flag": "Selection confound not fully addressed",
       "detail": "Repositories adopting coding agents may systematically differ in type (newer projects, higher complexity, specific domains) creating selection effects that independently explain higher mocking rates; the paired within-repository analysis partially mitigates this but developer-preference confounds remain (agent-adopting developers may already favor mocking)."
     },
     {
       "flag": "Agent versions not tracked",
       "detail": "Specific versions of Claude Code, Copilot, and Cursor active during the studied commits are not identified; since model updates change agent behavior rapidly, findings may not reflect current or future agent behavior."
     },
     {
       "flag": "No confidence intervals on main estimates",
       "detail": "All headline percentages (23%, 36%, 95%, etc.) are reported as point estimates without confidence intervals, making precision of the key comparative claims unassessable."
     },
     {
       "flag": "Unknown recall of mock detection method",
       "detail": "The identifier-based mock detection is validated only for precision (94%) but not recall; unknown false-negative rate could systematically bias the agent vs. non-agent comparison if agents use different naming conventions than the patterns searched."
     }
   ],
   "cited_papers": [
     {
       "title": "Promises, Perils, and (Timely) Heuristics for Mining Coding Agent Activity",
       "relevance": "Foundational companion paper by same authors establishing the methodology for detecting agent commits via co-authorship metadata in real repositories — directly enables this study"
     },
     {
       "title": "Agentic Much? Adoption of Coding Agents on GitHub",
       "relevance": "Under-submission companion paper measuring overall adoption rates of coding agents on GitHub, providing broader context for this study's scope and agent selection rationale"
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "relevance": "Becker et al. controlled experiment finding 19% task completion time increase despite 20% perceived productivity gain — key context for evaluating coding agent real-world effectiveness"
     },
     {
       "title": "To Mock or Not to Mock: Divergence in Mocking Practices Between LLM and Developers",
       "relevance": "Direct predecessor: Qin 2025 compared GPT-4o mock decisions vs developers in a single system, finding LLMs generate more mocks; this paper scales that finding to real-world agent commits across thousands of repositories"
     },
     {
       "title": "Mock objects for testing Java systems: Why and how developers use them, and how they evolve",
       "relevance": "Spadini et al. foundational empirical study of human mocking practices in Java; establishes baseline understanding for comparison with agent behavior"
     },
     {
       "title": "Use of test doubles in Android testing: An in-depth investigation",
       "relevance": "Fazzini et al. study whose identifier-based mock detection methodology is directly adapted by this paper for detecting test doubles in commits"
     },
     {
       "title": "Understanding Software Engineering Agents: A Study of Thought-Action-Result Trajectories",
       "relevance": "Bouzenia & Pradel study of agent interaction logs from SWE-bench; related characterization of coding agent action patterns in software engineering tasks"
     },
     {
       "title": "The Rise of AI Teammates in Software Engineering: How Autonomous Coding Agents Are Reshaping Software Engineering",
       "relevance": "Li et al. survey providing context on the broader adoption and capabilities of coding agents used to motivate the scope of this study"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable for anyone using Claude Code, Copilot, or Cursor — the recommendation to add mock guidance to CLAUDE.md configuration files is immediately applicable and the finding applies to millions of developers."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Finding that agents mock at 36% vs 26% and concentrate almost exclusively on the generic 'mock' type (95% vs 91%/57%/51% for non-agents) is a concrete, counterintuitive result about agent behavior that challenges assumptions of quality parity."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Tests with excessive mocking may mask integration bugs and allow code to drift from mock contracts, with software reliability implications, but no direct safety or security concerns are raised."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild controversy around AI-generated code quality; Kent Beck's LinkedIn quote adds human interest color but the paper is primarily technical without major conflict angles."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Scripts and dataset are available on Zenodo, but reproducing requires cloning thousands of GitHub repositories and running analysis scripts — not a quick demo, though practitioners can immediately apply configuration file guidance."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Directly studies Claude Code, GitHub Copilot, and Cursor with data from Microsoft/VS Code, home-assistant/core, and Apache repositories — high brand recognition among software engineering practitioners."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }