ai-research-survey

scan.json (21417B)

---

 {
   "paper": {
     "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
     "authors": [
       "Junwei Liu",
       "Kaixin Wang",
       "Yixuan Chen",
       "Xin Peng",
       "Zhenpeng Chen",
       "Lingming Zhang",
       "Yiling Lou"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.02977"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey collects 124 papers on LLM-based agents for software engineering, categorized from both SE task perspectives (requirements engineering, code generation, testing, debugging, IT operations, end-to-end development/maintenance) and agent architecture perspectives (planning, memory, perception, action, multi-agent systems). The survey finds that 59.7% of SE agents use multi-agent systems, with waterfall being the most popular process model for end-to-end development. Key challenges identified include reliance on test feedback quality, cascading errors from incorrect model feedback, and degraded long-context reasoning. The authors note that approximately 75% of collected papers are peer-reviewed publications.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository at https://github.com/FudanSELab/Agent4SE-Paper-List for the paper list."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper list repository serves as the released data artifact. The collected 124 papers and their categorizations are available via the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a survey, analysis scripts or tools used for paper collection/analysis are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for replicating the survey methodology or paper selection process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments or produce statistical results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no statistical comparisons requiring significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experimental results requiring effect size reporting."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; no experimental sample sizes to justify."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experimental runs requiring variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 discusses related surveys on LLMs for SE and general-domain LLM agents, positioning this survey against prior work including surveys by Hou et al. [2], Fan et al. [3], and He et al. [40]."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared surveys are recent (2023-2024), representing the current state of survey literature in this domain."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey paper — there is no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a survey paper without experimental evaluation requiring metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey paper's claims, which are about coverage and categorization of literature."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no experimental evaluation requiring held-out sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed breakdowns by SE task category (Figure 4), publication venue distribution (Figure 3b), and temporal trends (Figure 3a). Tables throughout the paper break down agents by task type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases of LLM-based agents in each SE task category (Sections 4.2.3, 4.5.4) including coordination failures, cascading errors, and degraded long-context reasoning."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings including that dual planners did not outperform single planners in Flows [102] (Section 4.2.1), and that autonomous localization did not lead to better performance (Section 4.8.8)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to collect 124 papers and categorize them from SE and agent perspectives, which is substantiated throughout the paper. The claim about 'remarkable effectiveness' is supported by cited results throughout Section 4."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that does not make causal claims. It summarizes and categorizes existing work."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Large Language Model-Based Agents for Software Engineering: A Survey' broadly, but the paper collection is bounded to July 2024 with DBLP keyword search. The scope section (3.1) defines boundaries but the title and abstract do not reflect these temporal and source limitations."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "As a survey/taxonomy paper presenting no empirical results of its own, alternative explanations are not applicable."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical/survey paper with no measurements of its own."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use any LLM models — it is a survey paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting — it is a survey paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used — this is a survey paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the paper collection methodology in detail: keyword searching on DBLP (57 searches, 10,362 hits → 67 after manual filtering), snowballing (adding 41 papers), and author feedback collection (adding 16 more). Table 1 provides inclusion/exclusion criteria. Table 2 shows per-keyword hit counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 'Threats to Validity' provides a dedicated discussion of threats including manual inspection bias, publication status of collected papers, and validation gaps for specific strategies."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 discusses specific threats: manual paper inspection may lead to inadvertent exclusion, several strategies are supported primarily by preprints (multi-agent RE, knowledge-enhanced bug detection, iterative coverage improvements, visual input), with specific counts of published vs. unpublished papers for each claim."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explicitly defines scope boundaries: focuses on LLM-based agents that 'iteratively interact with the environment' (not standalone LLMs), SE tasks along the software life cycle, and positions as 'comprehensive survey rather than a systematic literature review.' The paper also states it does not conduct extra experimental analysis."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper list is available at the GitHub repository https://github.com/FudanSELab/Agent4SE-Paper-List, allowing verification of the collected papers."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the collection process in detail: DBLP keyword searching on July 1, 2024, snowballing between July 1-10, 2024, and author feedback collection with 321 authors contacted and 36 responses received."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is a standard literature database (DBLP) with documented search queries."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows the full pipeline: 10,362 hits → 67 after manual inspection → 108 after snowballing → 124 after author feedback. Each stage is described in Sections 3.2.1-3.2.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Fudan University (Liu, Wang, Chen, Peng), Nanyang Technological University (Z. Chen), and University of Illinois Urbana-Champaign (Zhang, Lou)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a pre-trained model on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a pre-trained model on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a pre-trained model on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no method of its own requiring cost reporting."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes a structured protocol: keyword searching on DBLP with documented search strings, inclusion/exclusion criteria (Table 1), snowballing, and author feedback collection. The process follows established SE survey practices citing [41], [42], [43], [44], [45]. While not PRISMA specifically, it is a documented systematic protocol."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its 124 source papers. All papers meeting inclusion criteria are treated equally regardless of study rigor. Section 7.2 acknowledges that ~25% are preprints, but no quality scoring or risk-of-bias assessment is applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not discuss publication bias. There is no consideration of whether the collected papers skew toward positive results, no funnel plot analysis, and no acknowledgment that papers showing agents don't work may be underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "124 papers on LLM-based agents for SE were collected through keyword searching, snowballing, and author feedback.",
       "evidence": "Section 3 and Table 2 document the full collection pipeline: 67 from keyword searching, 41 from snowballing, 16 from author feedback.",
       "supported": "strong"
     },
     {
       "claim": "59.7% of existing agents for SE are multi-agent systems.",
       "evidence": "Section 5.2 states this statistic based on the collected papers.",
       "supported": "strong"
     },
     {
       "claim": "Approximately 75% of references are peer-reviewed publications from reputable journals and conferences.",
       "evidence": "Section 3.3 states this, with Figure 3b showing venue distribution.",
       "supported": "moderate"
     },
     {
       "claim": "Agentless, using a simplistic workflow, can outperform more complex fully autonomous agents on SWE-bench.",
       "evidence": "Section 4.8.8 and Figure 17 show Agentless achieves competitive resolve rates on SWE-bench Lite, confirmed by OpenAI [269].",
       "supported": "strong"
     },
     {
       "claim": "Pure autonomous localization methods do not necessarily lead to better performance in end-to-end software maintenance.",
       "evidence": "Section 4.8.8 notes SWE-agent, which allows fully autonomous localization, performed worst among all agents on SWE-bench Lite (Figure 17).",
       "supported": "moderate"
     },
     {
       "claim": "The Scrum model achieves the best and most stable performance among tested software process models for code generation.",
       "evidence": "Section 4.7.1 cites experiments from FlowGen [237] on function-level code generation benchmarks.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of source papers",
       "detail": "The survey treats all 124 papers equally regardless of methodological quality. ~25% are preprints that have not undergone peer review. Without quality assessment, the survey may launder weak results from low-quality studies alongside rigorous work."
     },
     {
       "flag": "No publication bias discussion",
       "detail": "The survey does not consider whether negative results about LLM-based agents for SE are underrepresented in the literature. Given the hype around LLM agents, there is likely significant publication bias toward positive findings."
     },
     {
       "flag": "Self-citation concentration",
       "detail": "Several of the authors (from Fudan University and UIUC) have published papers included in the survey. While their affiliations are disclosed, potential bias in how their own work is characterized is not explicitly acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2024,
       "relevance": "Comprehensive SLR on LLMs in SE, key comparison point for this survey's scope."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Primary benchmark for end-to-end software maintenance agents, central to Section 4.8."
     },
     {
       "title": "Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng"],
       "year": 2025,
       "relevance": "Agentless approach showing simpler workflows can outperform complex agents on SWE-bench."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu"],
       "year": 2024,
       "relevance": "Key multi-agent system for end-to-end software development using waterfall model with chat-based collaboration."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge"],
       "year": 2024,
       "relevance": "Influential multi-agent framework with structured message passing and shared memory for software development."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez"],
       "year": 2024,
       "relevance": "Pioneering agent-computer interface for autonomous software issue resolution."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano"],
       "year": 2023,
       "arxiv_id": "2310.11591",
       "relevance": "Foundational work on self-reflection and verbal feedback for iterative code generation."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal"],
       "year": 2024,
       "relevance": "Major multi-agent framework enabling conversational collaboration between LLM agents."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Zhiheng Xi", "Wenxiang Chen"],
       "year": 2025,
       "relevance": "General survey on LLM-based agents providing the agent framework taxonomy used in this survey."
     },
     {
       "title": "Fuzz4All: Universal fuzzing with large language models",
       "authors": ["Chunqiu Steven Xia", "Matteo Paltenghi"],
       "year": 2024,
       "relevance": "Universal LLM-based fuzzer for system-level testing across multiple programming languages."
     },
     {
       "title": "RepairAgent: An autonomous, LLM-based agent for program repair",
       "authors": ["Islem Bouzenia", "Premkumar T. Devanbu"],
       "year": 2025,
       "relevance": "Highly autonomous agent for program repair with state-machine-based middleware."
     },
     {
       "title": "MASAI: Modular architecture for software-engineering AI agents",
       "authors": ["Nalin Wadhwa", "Atharv Sonwane"],
       "year": 2024,
       "relevance": "Modular multi-agent architecture for end-to-end software maintenance with task decomposition."
     }
   ]
 }