ai-research-survey

scan.json (20165B)

---

 {
   "paper": {
     "title": "Configuring Agentic AI Coding Tools: An Exploratory Study",
     "authors": ["Matthias Galster", "Seyedmoein Mohsenimofidi", "Jai Lal Lulla", "Muhammad Auwal Abubakar", "Christoph Treude", "Sebastian Baltes"],
     "year": 2026,
     "venue": "Anonymous Conference (under review)",
     "arxiv_id": "2602.14690"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references supplementary material with scripts and data at a Zenodo archive (ref [5], doi:10.5281/zenodo.18625980): 'Our data collection and analysis scripts and the analyzed data are available online [5].'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo supplementary material includes the analyzed data: 'Our data collection and analysis scripts and the analyzed data are available online [5].'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of environment specifications, requirements.txt, or dependency details in the paper. The lingua-language-detector and GPT-5.2 are mentioned but no environment setup is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While scripts and data are released, the paper does not describe step-by-step reproduction instructions. The supplementary material may contain them but the paper itself does not provide specific commands or a README for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports IQR (interquartile ranges) for all metrics, e.g., 'Age: 6.7 (4.3–9.4)'. This serves as uncertainty quantification for the descriptive statistics."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 uses Mann-Whitney U tests with Benjamini-Hochberg FDR correction across 20 comparisons, reporting significance levels (*p<.05, **p<.01, ***p<.001)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports Cliff's delta effect sizes for each comparison, e.g., 'δ=-0.08', 'δ=0.21'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the initial 37,249 repositories were sufficient or why this sample size is adequate. The sample is convenience-based from SEART GitHub search with no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports IQR for all metrics. Section 6.2 reports min, max, and median for Skills per repository (min=1.0, max=28.0, median=2)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares across tools (Claude, Copilot, Codex, Cursor, Gemini) and against the full sample baseline in Table 3. Related work in Section 2 positions findings against prior studies."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is across contemporary tools (all active in 2025-2026). Related work references are from 2025-2026."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining/observational study with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: repository age, contributor count, commit count, size in KB (Table 3), plus adoption rates, co-occurrence correlations, and temporal trends."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not relevant to this mining study. The paper measures adoption patterns, not output quality."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study, not a predictive modeling study. No train/test split is applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-tool breakdowns (Figures 2, 3, Tables 2, 3), per-mechanism breakdowns, per-language breakdowns, and per-artifact type breakdowns are provided throughout."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses limitations of its heuristic detection approach in Section 6.4, and notes gaps like no evidence of persistent memory usage for Subagents (Section 6.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Skills are shallowly adopted (83.3% with no additional resources), Subagent memory is unused, and advanced mechanisms see very low adoption. These are negative/null findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The three abstract claims (Context Files dominate, Skills/Subagents shallowly adopted, distinct configuration cultures) are all supported by data in Sections 5-6 with figures and tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims like 'A reason for this could be that Claude Code is the most popular agentic AI coding tool' (Section 6.1) without evidence beyond correlation. However, these are hedged with 'could be' language, making them speculative rather than strong causal claims. The Discussion section offers explanations for adoption patterns without causal evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 (External validity) explicitly states: 'Our study covers only open-source repositories on GitHub; practices in proprietary or enterprise settings may differ.' The paper also notes it is a 'point-in-time snapshot (February 2026) of a rapidly evolving landscape.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) considers alternative explanations for shallow adoption: novelty of mechanisms, effort required, lack of empirical evidence on effectiveness, and developers gravitating to lowest-friction options."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "GPT-5.2 is specified for the repository classification step (Section 3): 'uses the GPT-5.2 model to determine...'. Default parameter values were used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'In our supplementary material, we share the final prompt and previous iterations of the prompt' (Section 3). The prompt content itself is in the supplementary material [5], not the paper body, but it is released."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states 'we kept OpenAI's default values for all parameters,' which is sufficient to reproduce the configuration."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in the methodology. GPT-5.2 is used as a simple classifier, not in an agentic loop."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 provides a detailed pipeline with counts at each stage: 37,249 → 36,184 (language filter) → 32,564 (engineered filter) → 2,926 (config detection). Filtering criteria are described at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 'Threats to Validity' provides a dedicated subsection organized by construct, internal, and external validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 discusses specific threats: heuristics detect presence not active use, GPT-5.2 classification used a single labeling run without inter-model agreement, 2,204 'unsure' cases excluded, cannot isolate agentic vs conversational usage for some tools."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 explicitly states: only OSS on GitHub, cannot claim representativeness for closed-source, no examination of application domain variation, point-in-time snapshot. Section 7 also notes 'early empirical signals rather than settled findings.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The supplementary material (Zenodo, ref [5]) includes the analyzed data and scripts."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and Figure 1 describe the data collection in detail: SEART GitHub search tool, repository selection criteria (non-fork, ≥2 contributors, license, created before Jan 2024, commits since June 2025), with specific dates."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is GitHub repositories selected via SEART search tool with documented criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the complete pipeline with counts at each filtering stage: 37,249 → excluded 24 archived → excluded 60 without README and 994 non-English → 32,564 engineered → 2,926 with config files. Each exclusion count is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Bamberg, Heidelberg University, Singapore Management University. None are affiliated with the tools being studied."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure is not the same as confirming independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study. GPT-5.2 is used only as a classifier for README files, not evaluated on a benchmark. No model capability is being measured."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of a pre-trained model's capability. GPT-5.2 is a utility tool in the pipeline, not the subject of evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The study mines repositories for configuration artifacts."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a repository mining study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-5.2 to classify 36,184 repositories but does not report the API cost or token consumption of this classification step."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of computational budget, hardware used, or time required for the data collection and analysis pipeline."
       }
     }
   },
   "claims": [
     {
       "claim": "Context Files dominate the configuration landscape and are often the sole mechanism in a repository, with AGENTS.md emerging as an interoperable standard.",
       "evidence": "2,634 of 2,926 repositories use Context Files (Section 5.2). AGENTS.md receives 368 incoming references (Section 6.1). CLAUDE.md→AGENTS.md is the most common reference pair (311 cases).",
       "supported": "strong"
     },
     {
       "claim": "Advanced mechanisms such as Skills and Subagents are only shallowly adopted, with Skills predominantly relying on static instructions rather than executable workflows.",
       "evidence": "83.3% of Skills include no additional resources (Section 6.2). Median Skills per repository is 2 (min=1, max=28). No repositories use Subagent persistent memory (Section 6.3).",
       "supported": "strong"
     },
     {
       "claim": "Distinct configuration cultures are forming around different tools, with Claude Code users employing the broadest range of mechanisms.",
       "evidence": "Figure 3 shows Claude appears across all mechanism types. Cursor emphasizes Rules (59.8%). Table 3 shows statistically significant differences in repository characteristics across tools.",
       "supported": "strong"
     },
     {
       "claim": "Repositories using Cursor are significantly younger than those using other tools.",
       "evidence": "Table 3: Cursor median age 5.5 years vs 6.7 overall, p<.001, Cliff's δ=-0.19.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["observational"],
   "key_findings": "This study systematically identifies eight configuration mechanisms across five agentic AI coding tools (Claude Code, Copilot, Codex, Cursor, Gemini) and analyzes their adoption in 2,926 GitHub repositories. Context Files (especially CLAUDE.md and AGENTS.md) dominate, appearing in 90% of repositories with configuration artifacts, while advanced mechanisms like Skills and Subagents remain shallowly adopted. Skills primarily serve as static documentation (83.3% have no executable resources), and distinct tool-specific configuration cultures are emerging, with Claude Code users showing the broadest mechanism diversity.",
   "red_flags": [
     {
       "flag": "LLM-based classification without validation",
       "detail": "GPT-5.2 was used to classify 36,184 repositories as 'engineered' or not, with only spot-checks for validation. No inter-annotator agreement, no comparison with human labels, and no alternative model was used. 2,204 'unsure' cases were excluded without analysis."
     },
     {
       "flag": "Presence as proxy for adoption",
       "detail": "The study detects configuration file presence but acknowledges it cannot confirm whether the corresponding tool is actively used. This is noted in threats to validity but could significantly affect adoption counts."
     }
   ],
   "cited_papers": [
     {
       "title": "Agent READMEs: An Empirical Study of Context Files for Agentic Coding",
       "authors": ["Worawalan Chatlatanagulchai", "Hao Li", "Yutaro Kashiwa"],
       "year": 2025,
       "arxiv_id": "2511.12884",
       "relevance": "Directly studies context files for agentic coding tools, finding they are actively maintained and focus on functional development instructions."
     },
     {
       "title": "Speed at the Cost of Quality: How Cursor AI Increases Short-Term Velocity and Long-Term Complexity in Open-Source Projects",
       "authors": ["Hao He", "Courtney Miller", "Shyam Agarwal"],
       "year": 2026,
       "relevance": "Empirical study of Cursor AI's impact on code quality in open-source projects."
     },
     {
       "title": "Beyond the Prompt: An Empirical Study of Cursor Rules",
       "authors": ["Shaokang Jiang", "Daye Nam"],
       "year": 2026,
       "relevance": "Empirical study of Cursor rule-based configuration mechanisms for AI coding tools."
     },
     {
       "title": "On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents",
       "authors": ["Jai Lal Lulla", "Seyedmoein Mohsenimofidi", "Matthias Galster"],
       "year": 2026,
       "arxiv_id": "2601.20404",
       "relevance": "Controlled study measuring the impact of AGENTS.md on agent runtime and token consumption."
     },
     {
       "title": "Context Engineering for AI Agents in Open-Source Software",
       "authors": ["Seyedmoein Mohsenimofidi", "Matthias Galster", "Christoph Treude"],
       "year": 2026,
       "relevance": "Studies context engineering practices in open-source projects for AI agent configuration."
     },
     {
       "title": "Decoding the Configuration of AI Coding Agents: Insights from Claude Code Projects",
       "authors": ["Hélio Victor F Santos", "Vitor Costa"],
       "year": 2025,
       "arxiv_id": "2511.09268",
       "relevance": "Analyzes configuration artifacts in Claude Code projects specifically."
     },
     {
       "title": "Swe-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez"],
       "year": 2024,
       "relevance": "Foundational work on agent-computer interfaces for automated software engineering."
     },
     {
       "title": "Prompts as Software Engineering Artifacts: A Research Agenda and Preliminary Findings",
       "authors": ["Hugo Villamizar", "Jannik Fischbach"],
       "year": 2025,
       "relevance": "Argues prompts should be treated as software engineering artifacts, relevant to configuration-as-code perspective."
     },
     {
       "title": "Using AI-based coding assistants in practice: State of affairs, perceptions, and ways forward",
       "authors": ["Agnia Sergeyuk", "Yaroslav Golubev"],
       "year": 2025,
       "doi": "10.1016/j.infsof.2024.107610",
       "relevance": "Survey of AI coding assistant usage in practice, providing context for adoption patterns."
     },
     {
       "title": "Meta Context Engineering via Agentic Skill Evolution",
       "authors": ["Haoran Ye", "Xuning He"],
       "year": 2026,
       "arxiv_id": "2601.21557",
       "relevance": "Framework for co-evolving agentic skills and context artifacts via evolutionary search."
     }
   ]
 }