ai-research-survey

scan.json (27294B)

---

 {
   "paper": {
     "title": "An Empirical Study of Agent Developer Practices in AI Agent Frameworks",
     "authors": [
       "Yanlin Wang",
       "Xinyi Xu",
       "Jiachi Chen",
       "Tingting Bi",
       "Wenchao Gu",
       "Zibin Zheng"
     ],
     "year": 2025,
     "venue": "ACM (preprint, December 2025)",
     "arxiv_id": "2512.01939",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, Zenodo archive, or other code release link is provided anywhere in the paper. The paper performs automated analysis using GPT-4o and GPT-4 but does not share scripts or code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The study collects 1,575 GitHub repositories and 20,620 developer discussions but does not provide a download link or public dataset. The raw discussion corpus and cleaned datasets are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or dependency list is provided. The paper only mentions using GPT-4o and GPT-4 without specifying any library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, reproduction script, or step-by-step guide is provided. The methodology is described in natural language in Section 3 but not in a form that would allow a researcher to replicate the data collection and analysis pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (percentages like '25.6%', '14%', '25%', '23%'). No confidence intervals, error bars, or uncertainty estimates are provided for any finding."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims across frameworks (e.g., AutoGen vs. LangChain vs. others on learning cost, development efficiency) but performs no statistical significance tests. All comparisons are based on frequency counts alone."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports proportions (e.g., '28.40% of discussions relate to development efficiency') but these describe category frequencies, not effect sizes for comparisons between frameworks. No Cohen's d, relative risk, or comparable measures are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample sizes (1,575 repos, 8,710 discussions, 11,910 discussions) are described by the filtering criteria but are never justified relative to statistical power or the claims being made. There is no power analysis or discussion of whether the sample size is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported for any result. All findings are presented as single aggregate proportions with no measure of uncertainty across frameworks or discussion categories."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare against prior empirical studies of agent frameworks or developer surveys. No baseline comparison study is included despite claiming to be 'the first' empirical study."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines are included in this observational/mining study, so contemporariness of baselines cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining/qualitative study, not a system with components to ablate. No experimental interventions are performed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates frameworks across five dimensions (learning cost, development efficiency, functional abstraction, performance optimization, maintainability) and uses multiple indicators for popularity (stars, forks, used-by counts, contributor counts, repo counts)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two domain experts independently reviewed all functional role annotations (Section 4.1.2) and a two-researcher open coding procedure was used in Section 6.1, both with Cohen's kappa reported (κ=0.82 and κ=0.81 respectively)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study mining GitHub discussions; there is no predictive model and no train/test split. The concept of a held-out test set does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per framework for all five evaluation dimensions (Sections 6.2.1–6.2.5) and by challenge category (Logic, Tool, Performance, Version) across SDLC stages (Section 5.2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire RQ2 section (Section 5) is dedicated to analyzing failure cases and challenges developers face, with detailed examples such as infinite loops (Figure 7), memory management failures (Figure 8), and version incompatibilities (Figure 9)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly reports negative findings including that performance optimization is 'a common shortcoming across all frameworks' (Finding 12) and that frameworks widely acclaimed (AutoGen, LangChain) face 'the highest maintenance complexity' (Finding 13)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (10 frameworks identified, taxonomy of four challenge categories, five-dimensional evaluation framework) are all substantiated in the results sections. Specific percentages in the abstract (25.6% task termination, 14% tool issues) match findings reported in Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-sounding claims such as 'LangChain and CrewAI lower the technical threshold for beginners' and 'AutoGen and LangChain excel at rapid prototyping' derived from observational discussion data. No controlled comparison is made; these are associations from developer complaints, not causal evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The study covers only GitHub open-source projects and this limitation is acknowledged in Section 7.3, but the abstract and findings present conclusions about 'agent developers' generally ('more than 80% of developers report difficulties') without consistently scoping claims to the GitHub open-source context."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 (Threats to Validity) discusses alternative explanations including LLM-generated bias in classification, topic tags not reflecting actual usage, and GitHub sample bias excluding closed-source environments. These are specific and relevant to the study's findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4o' and 'GPT-4' without specifying API version strings or snapshot dates. Section 5.1.2 mentions 'GPT-4' and Section 4.1.2 mentions 'GPT-4o' but neither provides a model version identifier, which matters significantly as model behavior varies across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides the actual system and user prompts used for discussion summarization verbatim, including the full prompt text. This allows replication of the LLM-based summarization step."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 states 'we standardize the generation process by setting the temperature to 0.1 and using top-1 sampling.' Temperature is explicitly reported for the LLM calls used in the study."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding; it uses LLMs as classification tools in a pipeline but not as autonomous agents. No agentic scaffolding applies here."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data collection pipeline with counts at each stage: 10,265 raw discussions → 8,710 after removing duplicates and invalid content; and the filtering criteria are stated (keyword inclusion, non-empty README, stars>10, forks>5, recency). Criteria are explicit."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 'Threats to Validity' provides a dedicated limitations discussion covering internal validity (LLM bias, topic tag accuracy) and external validity (GitHub-only sample, enterprise environments)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific: LLM-generated bias from using GPT-4o for summarization, topic tags possibly not reflecting actual framework usage, and the exclusion of closed-source enterprise projects. These are study-specific, not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 explicitly states 'Our sample covers only GitHub open-source projects, excluding closed-source projects, which may limit the generalizability of our conclusions. In enterprise or closed-source environments, framework preferences... may differ.' This is a specific scope boundary."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw GitHub discussion corpus (8,710 and 11,910 discussions) is not made available for independent verification. No supplementary data files, dataset DOIs, or download links are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection in detail: GitHub API search with compound keyword strategy, filtering criteria, time period (2022–July 2025), and metadata collected (stars, forks, used-by, contributors, topics, discussions)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study with no human participants. GitHub projects and discussions are the data source, not recruited participants. The sampling criteria are fully described in Section 3."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the full pipeline from raw collection to analysis for both phases (RQ1/RQ2 and RQ3), with explicit counts at each step (10,265 raw → duplicates removed → invalid filtered → 8,710 cleaned). The pipeline steps and their outputs are all documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section discloses funding from the State Key Laboratory of Blockchain and Data Security at Zhejiang University, the National Natural Science Foundation of China (grants 62332004, 62276279), and the Guangdong Basic and Applied Basic Research Foundation (2024B1515020032)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are disclosed in the paper header: Sun Yat-sen University (4 authors), Zhejiang University (1), University of Melbourne (1), Technical University of Munich (1). No authors are affiliated with the frameworks being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (NSFC, Guangdong research foundation, State Key Lab at Zhejiang University) are government and academic bodies with no financial stake in any of the ten frameworks evaluated. The funder is independent of the study outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement, patent disclosure, or declaration of financial interests anywhere in the paper. Absence of such a statement means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining/observational study of GitHub repositories and discussions. The paper uses GPT-4o/GPT-4 as classifiers/summarizers of developer text, not to evaluate pre-trained model capability on a benchmark. Contamination criteria do not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained model capabilities is performed. The LLMs used are tools for text classification and summarization, not the subject of evaluation. Contamination criteria do not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No capability benchmark is used. The study evaluates developer discussions about agent frameworks, not pre-trained model performance on code or reasoning tasks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study with no human participants. Mining GitHub discussions does not constitute a human subjects study; no pre-registration applies."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study analyzes publicly available GitHub data; IRB/ethics approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study examines GitHub repositories and discussion threads, not recruited participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Repository selection criteria are described in Section 3 but these are data filtering criteria, not participant eligibility criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental assignment. This is an observational study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no longitudinal tracking of individuals. Attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o and GPT-4 to summarize and classify 20,620 discussion threads but reports no API costs, token counts, or cost per example. Given the scale (tens of thousands of LLM calls), this omission is notable."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, API spend, wall-clock time, or hardware specifications are provided for the LLM-based analysis pipeline. The compute budget for running GPT-4/GPT-4o on the full dataset is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "96% of top-starred GitHub agent projects adopt two or more different agent frameworks, indicating that single-framework solutions can no longer meet the complex demands of real-world agent applications.",
       "evidence": "Analysis of top 25% starred projects (approximately 394 of 1,575 repos) for co-occurrence of framework tags and dependency file checks (Section 4.2.2, Finding 2, Figure 3).",
       "supported": "moderate"
     },
     {
       "claim": "More than one-third of developer-reported agent development failures stem from internal logic control deficiencies; task termination and message cooling issues account for 21.63% and 9.86% of failures respectively.",
       "evidence": "Analysis of 8,710 LLM-generated issue summaries from GitHub discussions, mapped to SDLC stages (Section 5.2, Finding 5). Exact figures cited from the taxonomy analysis.",
       "supported": "moderate"
     },
     {
       "claim": "Version compatibility conflicts directly cause over 25% of technical obstacles, occurring most frequently during deployment and maintenance stages.",
       "evidence": "Frequency analysis of 8,710 developer discussion summaries classified into challenge categories (Section 5.2, Finding 8). Examples include LangChain/Pydantic conflicts and AutoGen v0.2-to-v0.4 breaking changes.",
       "supported": "moderate"
     },
     {
       "claim": "AutoGen and LangChain excel at rapid prototyping, with over 78% of developers in discussions citing these frameworks for rapid prototype verification.",
       "evidence": "Analysis of developer discussions in Section 6.2.2 (Finding 10). The 78% figure is derived from frequency analysis of the 11,910 framework-specific discussions, but no statistical test or confidence interval is provided.",
       "supported": "weak"
     },
     {
       "claim": "Performance optimization is a common shortcoming across all ten evaluated frameworks.",
       "evidence": "Qualitative analysis of performance-related discussions per framework in Section 6.2.4 (Finding 12), with per-framework breakdown of specific performance issues listed in Figure 13.",
       "supported": "moderate"
     },
     {
       "claim": "Community popularity (GitHub stars) is not a reliable predictor of real-world framework adoption (e.g., MetaGPT has 48.7K stars but appears in only 2 repos; LangGraph has fewer stars but appears in 26 repos).",
       "evidence": "Table 2 and Figure 4 comparing star counts, fork counts, and repository adoption counts for all ten frameworks (Section 4.2.3, Finding 3).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "observational",
     "qualitative"
   ],
   "key_findings": "This is a large-scale mining and qualitative analysis of 1,575 GitHub agent projects and 20,620 developer discussions covering ten major LLM-based agent frameworks. The study finds that 96% of highly-starred projects combine multiple frameworks, that over one-third of failures stem from logic control deficiencies (especially task termination), and that version conflicts cause over 25% of technical obstacles. A comparative evaluation across five dimensions reveals that AutoGen and LangChain lead in prototyping and functional abstraction but face the highest maintenance complexity, while performance optimization is a universal weakness across all frameworks.",
   "red_flags": [
     {
       "flag": "No statistical testing for comparative claims",
       "detail": "The paper makes comparative claims across ten frameworks (e.g., 'AutoGen and LangChain excel', 'CrewAI lowers the technical threshold') based solely on frequency counts of developer discussions, with no statistical tests. All differences between frameworks could be attributable to sampling variation or the unequal number of discussions per framework (LangChain has 6,006 vs. Swarm's 391)."
     },
     {
       "flag": "Unequal sample sizes confound framework comparisons",
       "detail": "Framework discussion counts range from 224 (CrewAI) to 6,006 (LangChain). Comparing proportions of discussion topics across frameworks with such different sample sizes without adjustment (e.g., normalization or confidence intervals) risks misleading conclusions about which frameworks have worse or better properties."
     },
     {
       "flag": "LLM used as classifier without validation of accuracy",
       "detail": "GPT-4o is used to semantically classify 8,710+ developer discussions and generate summaries. While the paper validates inter-rater agreement for the open coding step (κ=0.82), there is no systematic validation of the LLM classification accuracy or comparison against human-only coding for the automated summarization phase."
     },
     {
       "flag": "Causal language without causal design",
       "detail": "Findings use causal language ('LangChain and CrewAI lower the technical threshold', 'AutoGen and LangChain excel at rapid prototyping') from observational discussion data. These are associations between discussion content and framework choice, not evidence that the frameworks cause these outcomes."
     },
     {
       "flag": "Code and data not released",
       "detail": "The study claims to be a systematic empirical investigation but releases neither the collection scripts, the cleaned discussion corpus, nor the LLM-generated summaries. This makes independent verification of the key findings impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent llm systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang", "Lakshya A Agrawal"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Provides a failure taxonomy for multi-agent LLM systems with 14 failure modes, directly relevant to agentic AI methodology evaluation."
     },
     {
       "title": "Autogen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang", "Yiran Wu", "Beibin Li"],
       "year": 2024,
       "relevance": "Introduces the AutoGen multi-agent framework, one of the ten frameworks studied; directly relevant to agentic workflow evaluation."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "Presents MetaGPT framework for automated software project generation using multi-agent SOPs, relevant to agentic programming evaluation."
     },
     {
       "title": "Survey on evaluation of llm-based agents",
       "authors": ["Asaf Yehudai", "Lilach Eden", "Alan Li", "Guy Uziel", "Yilun Zhao"],
       "year": 2025,
       "arxiv_id": "2503.16416",
       "relevance": "Surveys evaluation strategies and gaps for LLM-based agents, highly relevant to methodology quality in agentic AI research."
     },
     {
       "title": "Agentless: Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Proposes a simpler approach to LLM-based software engineering without complex agent scaffolding; relevant to evaluating agentic frameworks."
     },
     {
       "title": "AI agents that matter",
       "authors": ["Sayash Kapoor", "Benedikt Stroebl", "Zachary S Siegel", "Nitya Nadgir", "Arvind Narayanan"],
       "year": 2024,
       "arxiv_id": "2407.01502",
       "relevance": "Critically examines evaluation practices for AI agents and identifies methodological flaws; directly relevant to methodology quality survey."
     },
     {
       "title": "Agent-SafetyBench: Evaluating the Safety of LLM Agents",
       "authors": ["Zhexin Zhang", "Shiyao Cui", "Yida Lu", "Jingzhuo Zhou", "Junxiao Yang"],
       "year": 2025,
       "arxiv_id": "2412.14470",
       "relevance": "Benchmarks safety of LLM agents across multiple environments; relevant to evaluating agentic AI methodology and safety evaluation practices."
     },
     {
       "title": "An Empirical Study of Testing Practices in Open Source AI Agent Frameworks and Agentic Applications",
       "authors": ["Mohammed Mehedi Hasan", "Hao Li", "Emad Fallahzadeh", "Gopi Krishnan Rajbahadur", "Bram Adams", "Ahmed E. Hassan"],
       "year": 2025,
       "arxiv_id": "2509.19185",
       "relevance": "Complementary empirical study of testing practices in the same agent framework ecosystem; directly relevant to the survey scope."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran"],
       "year": 2023,
       "relevance": "Introduces the ReAct framework for LLM agent reasoning and acting; foundational to the agentic AI landscape surveyed."
     },
     {
       "title": "Agents in software engineering: Survey, landscape, and vision",
       "authors": ["Yanlin Wang", "Wanjun Zhong", "Yanxian Huang", "Ensheng Shi"],
       "year": 2025,
       "relevance": "Survey of LLM agent applications in software engineering from the same research group; relevant for citation chasing in the agentic programming domain."
     },
     {
       "title": "LLM-Based Multi-Agent Systems for Software Engineering: Literature Review, Vision, and the Road Ahead",
       "authors": ["Junda He", "Christoph Treude", "David Lo"],
       "year": 2025,
       "relevance": "Literature review on multi-agent LLM systems for software engineering; directly relevant to agentic AI programming methodology survey."
     },
     {
       "title": "The prompt report: a systematic survey of prompt engineering techniques",
       "authors": ["Sander Schulhoff", "Michael Ilie", "Nishant Balepur"],
       "year": 2024,
       "arxiv_id": "2406.06608",
       "relevance": "Systematic survey of prompt engineering, relevant to evaluating how agent frameworks handle prompting and LLM interaction."
     }
   ]
 }