ai-research-survey

scan.json (27089B)

---

 {
   "paper": {
     "title": "AI Agentic Programming: A Survey of Techniques, Challenges, and Opportunities",
     "authors": [
       "Huanting Wang",
       "Jingzhi Gong",
       "Huawei Zhang",
       "Jie Xu",
       "Zheng Wang"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.11126"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided. The survey could have released analysis scripts, extraction tools, or dataset metadata but did not. No mention of code availability anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The corpus of 152 reviewed papers and the extracted taxonomy data are not released. No links to supplementary data files or public dataset are provided."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experiments or software environment; no runtime environment is needed or applicable."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions for reproducing the search, screening, or selection process are provided beyond a general description of the methodology. A researcher could not replicate the corpus without full query strings, database access dates, and screening decisions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no primary statistical analysis; confidence intervals are not applicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No hypothesis tests are conducted; this is a literature survey with qualitative taxonomy and descriptive counts."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effect sizes are applicable; the paper does not conduct meta-analytic or comparative statistical analyses."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No participant sample; the paper reviews a corpus of papers using inclusion/exclusion criteria, which is a standard SLR approach, not a statistical sample requiring power analysis."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No repeated measurements or experiments are run; variance reporting is not applicable to this survey."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare itself against prior surveys on the topic (e.g., the 2024 survey on LLM-based agents for SE by Liu et al., or Hou et al. 2024 on LLMs for SE). No structured comparison of coverage, methodology, or conclusions with related surveys is provided."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experimental baselines are used; this criterion is not applicable to a survey paper with no experiments."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or model is evaluated; ablation studies are not applicable to a survey paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "The survey presents a taxonomy and qualitative analysis without using multiple evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of the survey's outputs (taxonomy, findings) is not conducted; this is a theoretical/taxonomical survey paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No prediction or classification model is trained; held-out test sets are not applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides per-category breakdowns in several ways: Table 5 compares representative systems across behavioral dimensions, Table 8 breaks down benchmarks by source, language, task, and difficulty, and Figure 6 shows the temporal distribution of papers by year."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Challenges) discusses failure modes of existing agentic systems, including limitations of benchmarks (Section 5.1), communication protocol deficiencies (5.2), domain-specific weaknesses (5.3), and safety failures (5.4). Figure 9 shows where all top-10 LLMs still fail on software optimization tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and the benchmark analysis in Section 6.3 explicitly note negative findings: all models perform poorly on software optimization (GSO benchmark), benchmarks are biased toward Python, and existing frameworks lack multi-turn interactive evaluation support."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to introduce a taxonomy, review techniques, and discuss challenges. The paper delivers all these: taxonomy in Section 4, techniques in Sections 2 and 4, challenges in Section 5, and opportunities in Section 6. Descriptive claims are appropriately qualified."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The survey makes no causal empirical claims; it describes, categorizes, and synthesizes existing work without claiming causal relationships from its own evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We focus primarily on LLM-driven agentic systems for software development' but the title 'AI Agentic Programming: A Survey of Techniques, Challenges, and Opportunities' is quite broad. No search date cutoff is stated, no explicit boundaries on what was excluded (non-English papers, specific databases, adjacent domains), and no statement of what the survey's results do NOT show. The scoping is a topic statement rather than explicit scope boundaries per the schema's requirement."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/taxonomy paper with no empirical results of its own; alternative explanations for observed patterns are not applicable. The paper presents the state of the field rather than testing hypotheses."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The survey itself does not use any LLM; it reviews papers that do. No LLM is deployed by the survey authors."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The survey does not use any prompting; it is a literature review conducted by human researchers."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models or algorithms are run by the survey authors; hyperparameter reporting is not applicable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used by the survey authors; this is a human-conducted literature review."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 describes the filtering pipeline stages with counts (7,700 → 395 → 141 → 152), but does not fully state the actual filtering CRITERIA applied at each stage. The inclusion/exclusion criteria are listed (Section 3.2), but the criteria for moving from 141 full-text reviewed to 152 final (the increase via citation chaining) are not clearly specified. The screening criteria are general and lack operationalized decision rules a second researcher could apply independently."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The paper has a 'Challenges' section (Section 5) and a 'Conclusion' (Section 7), but these address challenges in the field, not limitations of the survey itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity section exists. The paper does not discuss potential biases in its own paper selection process, publication bias in the literature, or limitations of the inclusion/exclusion criteria. Generic disclaimers about the field moving fast (Introduction) do not substitute for specific methodological threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper states it focuses on LLM-driven agentic systems for software development, it does not explicitly state what it does NOT cover (e.g., which adjacent areas were excluded and why, whether non-English papers were excluded, what date range was searched, or what databases were excluded). The search cutoff date is not mentioned."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The list of 152 reviewed papers is not released as a dataset. There is no appendix or supplementary material listing the full corpus. The bibliography covers only cited papers, not the complete reviewed corpus."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the search strategy in detail: databases searched (Google Scholar, ACM DL, IEEE Xplore, SpringerLink, arXiv), the Boolean search string combining agent terms, programming terms, and AI/LLM terms, and the three-stage selection process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a literature review. The criterion about recruitment methods applies to human subjects research, which this paper is not."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline counts (7,700 → 395 → 141 → 152) are given in Section 3.3 and Figure 5, but the criteria applied at each stage are not operationalized. The jump from 141 to 152 (via citation chaining adding 11 papers) lacks explanation of how these 11 were selected or why the original search missed them. Disagreement resolution criteria for the two-researcher screening are not specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper. There is no mention of grants, institutional support, or funding agencies anywhere in the text."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors list their affiliation as University of Leeds (UK) on the title page, with individual email addresses. Author affiliations are clearly disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed. The paper appears to be academic survey work without declared external funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. There is no declaration that authors hold (or do not hold) patents, equity, or financial interests related to the tools and systems reviewed (including GitHub Copilot, Claude, etc.)."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; the authors do not train or evaluate any pre-trained models on benchmarks. Contamination questions are not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; no models are evaluated. The contamination question is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; no models are evaluated on benchmarks by the authors. Contamination is not applicable to the survey itself, though the paper discusses contamination as a challenge in the reviewed literature (Section 6.3)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a literature review of published papers."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; IRB approval is not applicable to a systematic literature review of published papers."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; the inclusion/exclusion criteria in Section 3.2 refer to papers, not human subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is not an experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; the authors do not run any LLM inference. The paper reviews cost considerations of others' systems (Table 6 shows token pricing) but does not report the cost of the survey's own methods."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; no compute budget is required. No models are trained or run by the survey authors."
       }
     }
   },
   "claims": [
     {
       "claim": "The field of AI agentic programming is characterized by no standard taxonomy, benchmark suite, or evaluation methodology.",
       "evidence": "Introduction section: 'There is no standard taxonomy, benchmark suite, or evaluation methodology.' The paper motivates this gap as the reason for the survey.",
       "supported": "moderate"
     },
     {
       "claim": "Existing benchmarks such as HumanEval and SWE-Bench are inadequate for capturing the full complexity of real-world software engineering workflows.",
       "evidence": "Section 5.1 and Table 7: SWE-Bench is restricted to Python, most tasks are function/module-level, with no multi-turn feedback, third-party library usage, or build pipeline management. Figure 9 shows all top-10 LLMs still score poorly on software optimization (GSO benchmark).",
       "supported": "strong"
     },
     {
       "claim": "Current benchmarks are heavily biased toward Python, limiting generalizability of results to other programming languages.",
       "evidence": "Section 6.3: 'they are heavily biased toward a small set of programming languages - especially Python' with citations [59, 222]. Table 8 confirms Python dominates the listed benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "AI agentic programming represents a distinct paradigm from program synthesis, code completion, and DevOps automation, characterized by multi-step planning, tool integration, and iterative refinement.",
       "evidence": "Section 2.5 systematically compares agentic programming to each related paradigm with specific capability distinctions. The taxonomy in Figure 7 and Table 5 operationalizes these distinctions.",
       "supported": "moderate"
     },
     {
       "claim": "Most current mainstream AI coding agents use persistent memory mechanisms, while simple assistants like GitHub Copilot rely on transient context.",
       "evidence": "Table 4 compares context management of eight representative agents. GitHub Copilot uses 'Sliding window over active buffer' with no persistent memory, while SWE-agent, Devika, OpenDevin use vector DB or structured stores.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "meta-analysis"
   ],
   "key_findings": "This systematic literature review of 152 papers introduces a taxonomy of AI agentic programming systems along behavioral (reactive vs. proactive, single-turn vs. multi-turn, tool-augmented vs. standalone, static vs. adaptive) and architectural (interactive code assistants, autonomous task-oriented agents, planning-centric agents, multi-agent systems) dimensions. The paper identifies key challenges including inadequate benchmarks biased toward Python and single-turn tasks, communication protocol limitations for multi-agent systems, domain-specific model weaknesses, and safety/trust concerns. It also synthesizes opportunities in compiler-agent integration, scalable memory architectures, domain specialization, and safety alignment. The paper covers tools ranging from GitHub Copilot and Cursor to SWE-agent, ChatDev, and AutoCodeRover, providing comparative tables of LLM capabilities and pricing.",
   "red_flags": [
     {
       "flag": "No limitations section on the survey itself",
       "detail": "The paper has a 'Challenges' section (Section 5) covering limitations of the field, but includes no discussion of limitations of the survey's own methodology — no acknowledgment of potential publication bias, search string limitations, database coverage gaps, or the possibility that the 152-paper corpus is incomplete or systematically skewed."
     },
     {
       "flag": "Survey does not assess methodological quality of reviewed papers",
       "detail": "The survey collects and categorizes papers by architecture and behavior type, but does not evaluate the quality of evidence in those papers. This is the 'laundering' pattern: a survey that aggregates papers without quality assessment inherits and amplifies the methodological weaknesses of its sources. There is no quality appraisal instrument or scoring reported."
     },
     {
       "flag": "No funding disclosure",
       "detail": "Five authors at a UK university produced a substantial systematic review with no funding acknowledgment. This is unusual and raises questions about whether institutional or industry support exists but was not declared."
     },
     {
       "flag": "Screening criteria underspecified for replication",
       "detail": "Section 3.2 states two-researcher screening was used, but decision rules for borderline cases, disagreement resolution procedures (beyond 'through discussion'), and the criteria for citation chaining additions (11 papers added, bringing 141 to 152) are not specified. Another team could not replicate the corpus selection."
     },
     {
       "flag": "Search date range not stated",
       "detail": "The paper does not disclose when the database searches were conducted, making it impossible to assess whether recent papers (e.g., from mid-2025) were included or excluded. The arXiv paper itself is dated September 2025 and cites tools accessed 'Accessed: 2025-09-15', suggesting the searches were recent, but no explicit search cutoff date is given."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": [
         "Carlos E Jimenez",
         "John Yang",
         "Alexander Wettig",
         "Shunyu Yao",
         "Kexin Pei",
         "Ofir Press",
         "Karthik Narasimhan"
       ],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Core benchmark for evaluating agentic code-repair agents on real GitHub issues; heavily cited as the dominant agentic SE benchmark."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": [
         "John Yang",
         "Carlos E Jimenez",
         "Alexander Wettig",
         "Kilian Lieret",
         "Shunyu Yao",
         "Karthik Narasimhan",
         "Ofir Press"
       ],
       "year": 2024,
       "relevance": "Key agentic system for automated software engineering that uses a multi-role agent scaffold to solve GitHub issues; exemplifies the multi-agent collaborative category."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": [
         "Chen Qian",
         "Wei Liu",
         "Hongzhang Liu",
         "Nuo Chen",
         "Yufan Dang",
         "Jiahao Li",
         "Cheng Yang",
         "Weize Chen",
         "Yusheng Su",
         "Xin Cong",
         "Juyuan Xu",
         "Dahai Li",
         "Zhiyuan Liu",
         "Maosong Sun"
       ],
       "year": 2024,
       "arxiv_id": "2307.07924",
       "relevance": "Pioneering multi-agent software development simulation with role-playing agents (CEO, CTO, programmers) for end-to-end software development."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": [
         "Xinyi Hou",
         "Yanjie Zhao",
         "Yue Liu",
         "Zhou Yang",
         "Kailong Wang",
         "Li Li",
         "Xiapu Luo",
         "David Lo",
         "John Grundy",
         "Haoyu Wang"
       ],
       "year": 2024,
       "relevance": "Broad systematic review of LLMs in software engineering; directly related prior survey that the current paper extends toward the agentic paradigm."
     },
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "authors": [
         "Junwei Liu",
         "Kaixin Wang",
         "Yixuan Chen",
         "Xin Peng",
         "Zhenpeng Chen",
         "Lingming Zhang",
         "Yiling Lou"
       ],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Close prior survey covering LLM-based agents specifically for SE; the current paper extends and updates this coverage."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu",
         "Nan Du",
         "Izhak Shafran",
         "Karthik Narasimhan",
         "Yuan Cao"
       ],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational prompting paradigm for agentic systems that interleaves reasoning and action; widely adopted in coding agents."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": [
         "Qingyun Wu",
         "Gagan Bansal",
         "Jieyu Zhang",
         "Yiran Wu",
         "Beibin Li",
         "Erkang Zhu",
         "Li Jiang",
         "Xiaoyun Zhang",
         "Shaokun Zhang",
         "Jiale Liu"
       ],
       "year": 2024,
       "relevance": "Leading multi-agent framework for LLM-based applications; key infrastructure for agentic programming systems."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": [
         "Guanzhi Wang",
         "Yuqi Xie",
         "Yunfan Jiang",
         "Ajay Mandlekar",
         "Chaowei Xiao",
         "Yuke Zhu",
         "Linxi Fan",
         "Anima Anandkumar"
       ],
       "year": 2023,
       "arxiv_id": "2305.16291",
       "relevance": "Influential example of a planning-centric agent that generates and accumulates reusable skills; cited as a model for long-horizon agentic planning."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": [
         "Xingyao Wang",
         "Boxuan Li",
         "Yufan Song",
         "Frank F Xu",
         "Xiangru Tang",
         "Mingchen Zhuge",
         "Jiayi Pan",
         "Yueqi Song",
         "Bowen Li",
         "Jaskirat Singh"
       ],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Open-source agentic platform for software development; directly relevant to the survey's coverage of autonomous task-oriented agents."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": [
         "Naman Jain",
         "King Han",
         "Alex Gu",
         "Wen-Ding Li",
         "Fanjia Yan",
         "Tianjun Zhang",
         "Sida Wang",
         "Armando Solar-Lezama",
         "Koushik Sen",
         "Ion Stoica"
       ],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free benchmark for LLM code evaluation; addresses the training data leakage problem that affects static benchmarks like HumanEval."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": [
         "Yuntong Zhang",
         "Haifeng Ruan",
         "Zhiyu Fan",
         "Abhik Roychoudhury"
       ],
       "year": 2024,
       "relevance": "Multi-agent system for autonomous navigation, editing, and validation of real-world repositories; key example of collaborative agentic SE."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": [
         "Noah Shinn",
         "Federico Cassano",
         "Ashwin Gopinath",
         "Karthik Narasimhan",
         "Shunyu Yao"
       ],
       "year": 2023,
       "relevance": "Foundational technique for self-improving agents using verbal reinforcement; directly enables the feedback-loop patterns central to agentic programming."
     }
   ]
 }