ai-research-survey

scan-v5.json (20225B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
     "authors": [
       "Junwei Liu",
       "Kaixin Wang",
       "Yixuan Chen",
       "Xin Peng",
       "Zhenpeng Chen",
       "Lingming Zhang",
       "Yiling Lou"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.02977",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims (comprehensive survey, 124 papers, two-perspective categorization, open challenges discussion) are substantiated by the paper's content with documented methodology and dedicated sections.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Comparative claims about agents vs. standalone LLMs are supported by citing specific empirical results from reviewed papers (e.g., pass@1 results from [325],[94] for code generation), which is appropriate attribution for a survey.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.1 makes broad claims that 'LLM-based agents offer stronger performance and wider applicability in real-world software engineering' without acknowledging publication bias in the reviewed corpus or that the sample is non-random.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.2 only discusses threats to coverage (missed papers) but not threats to interpretation — publication bias, cherry-picked benchmarks, or confounds in why agents appear to outperform standalone LLMs are not considered.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly notes metric limitations: 'software accuracy is merely assessed based on manually scored executability or code similarity, which may not fully capture the quality of the generated software.'",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 'Threats to Validity' is a dedicated multi-paragraph section with five distinct named threats, well beyond a single sentence in a conclusion.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific: e.g., 'Multi-Agent Strategy in Requirements Engineering Lacks Sufficient Validation' with exact counts (3 of 4 papers use multi-agent, 2 of those are unreviewed preprints); 'Iterative Coverage Improvements in Unit Testing' supported by 1 of 3 published works.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explicitly defines scope (LLM-based agents with iterative environment interaction, SE lifecycle tasks), explicitly excludes standalone LLM workflows, and states 'conducting extra experimental analysis is beyond the scope of this survey.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure statement appears anywhere in the paper. The acknowledgements section only thanks authors who provided feedback on the draft.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed: Fudan University (China), Nanyang Technological University (Singapore), and University of Illinois Urbana-Champaign (USA).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, making this criterion not applicable.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests declaration anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides explicit operational definitions for both 'SE tasks' (along the software lifecycle) and 'LLM-based agents' (must iteratively perceive feedback from and act upon a dynamic environment), explicitly distinguishing agents from standalone LLMs.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are listed: (1) comprehensive survey of 124 papers on LLM agents for SE, (2) analysis from SE and agent perspectives, (3) discussion of research opportunities and future directions.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 explicitly compares to five related surveys ([2],[3],[24],[34],[40]), specifying how this work differs: wider SE task coverage (end-to-end tasks), agent architecture taxonomy (memory/planning/action), and empirical survey vs. vision paper.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "DBLP was searched on July 1, 2024 with the complete keyword set provided verbatim, yielding 10,362 hits. Table 2 shows per-keyword hit counts. The iterative keyword refinement procedure is documented in detail.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 provides 2 explicit inclusion criteria and 7 explicit exclusion criteria covering agent type, evaluation presence, LLM integration degree, paper length, grey literature exclusion, and deduplication.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "Authors explicitly state 'we position this paper as a comprehensive survey rather than a systematic literature review' and no PRISMA or equivalent structured protocol is followed or mentioned.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The complete final keyword query is provided verbatim in the text: ('agent' OR 'llm' OR 'language model') AND ('api' OR 'bug' OR 'code' OR ... 'vulnerab'), with 19 OR-combined SE terms.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "DBLP is explicitly named as the primary database with justification (covers 7M+ publications, 6,500+ conferences, 1,850+ journals, includes arXiv). Google Scholar is used for forward snowballing.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 shows counts at each stage: 10,362 hits → 67 after manual inspection → 108 after snowballing → 124 after author feedback. Three screening stages are described with two independent reviewers and a third arbiter for disagreements.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "DBLP choice is justified by citing prior SE surveys showing other databases are typically DBLP subsets. The temporal scope (up to September 2024) is justified by the rapidly evolving field requiring timely coverage.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": true,
           "justification": "Conflicting findings are acknowledged in multiple places: simpler Agentless outperforms complex agents on SWE-bench Lite; pure autonomous localization (SWE-agent) performs worst despite full autonomy; multi-planner approaches don't always surpass single-planner (Flows experiment).",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No systematic quality rubric or risk-of-bias assessment is applied to the 124 reviewed papers. The only quality dimension noted is peer-review status (published vs. preprint), and this is flagged only for specific strategies in Section 7.2 rather than applied systematically.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias is not discussed. The paper notes 75% peer-reviewed sources but never acknowledges that published papers skew toward positive results — a significant omission given the conclusion that agents broadly outperform standalone LLMs.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "Performance numbers from individual papers are tabulated (e.g., Figure 17 for SWE-bench Lite resolve rates) but no formal quantitative synthesis, meta-analysis, or effect size aggregation is performed across studies.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 recommendations are grounded in specific evidence: the call for better benchmarks cites documented SWE-bench issues [260],[269]; the priority for efficiency metrics is backed by finding only 46.7% of surveyed papers quantify computational costs.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "59.7% of existing LLM-based agents for SE are multi-agent systems",
       "evidence": "Internal count of the 124 collected papers",
       "supported": "strong"
     },
     {
       "claim": "LLM-based agents consistently achieve higher pass@1 than foundation LLMs across code generation benchmarks of various complexity",
       "evidence": "Cited from papers [325] and [94] reporting results on HumanEval, LiveCodeBench, and CodeAgentBench",
       "supported": "moderate"
     },
     {
       "claim": "Simpler traditional fault localization approaches (Agentless) surpass many complex agentic approaches on SWE-bench Lite",
       "evidence": "Figure 17 comparing resolve rates from original papers; explicitly discussed in Section 4.8.8",
       "supported": "strong"
     },
     {
       "claim": "Only 46.7% of surveyed papers explicitly quantify efficiency through time, token consumption, monetary cost, or feedback loop metrics",
       "evidence": "Internal analysis of reporting practices across the 124 reviewed papers",
       "supported": "strong"
     },
     {
       "claim": "Agents employing dynamic patch verification and patch ranking generally achieve higher resolve rates on SWE-bench Lite",
       "evidence": "Observation from Figure 17: all top-5 agents (Agentless, MASAI, SpecRover, DEIBase, CodeR) use dynamic checking or ranking",
       "supported": "moderate"
     },
     {
       "claim": "Multi-agent collaboration is the predominant strategy in requirements engineering but lacks peer-reviewed validation",
       "evidence": "3 of 4 RE papers use multi-agent architectures, but only 2 of those 3 are published in peer-reviewed venues",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "qualitative"
   ],
   "key_findings": "This survey of 124 papers maps LLM-based agents for software engineering across both SE task types (requirements, code generation, static checking, testing, debugging, IT ops, end-to-end development and maintenance) and agent components (planning, memory, perception, action, multi-agent collaboration). Key structural finding: 59.7% of agents are multi-agent systems, with vertical collaboration architecture most common. A notable empirical counterpoint emerges: simpler, less-agentic approaches like Agentless outperform complex autonomous agents on SWE-bench Lite, suggesting agent complexity does not guarantee performance gains. The survey identifies critical methodological gaps in the field: lack of fine-grained metrics beyond final success rates, limited benchmarks that reflect real-world SE complexity, and insufficient exploration of human-agent collaboration beyond predefined interaction points.",
   "red_flags": [
     {
       "flag": "Publication bias not discussed",
       "detail": "The survey concludes agents broadly outperform standalone LLMs (Section 7.1) but never acknowledges that the reviewed literature is a non-random sample skewed toward positive results by publication bias."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "All 124 papers are treated as equally credible regardless of evaluation rigor; no methodological quality rubric or risk-of-bias tool is applied to the reviewed papers."
     },
     {
       "flag": "Broad superiority claims from biased sample",
       "detail": "Section 7.1 makes sweeping claims ('LLM-based agents offer stronger performance and wider applicability in real-world software engineering') that overstate what a survey of published papers can support."
     },
     {
       "flag": "25% of sources are unreviewed preprints",
       "detail": "Approximately 31 arXiv preprints without peer review are included; several support key findings (e.g., in requirements engineering and bug detection sections) and may not be replicable."
     },
     {
       "flag": "No funding disclosure",
       "detail": "No funding sources are disclosed despite the paper representing a large multi-institution collaborative effort."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Primary benchmark for end-to-end software maintenance; central to the survey's performance comparison of agents"
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "relevance": "Major exemplar of multi-agent end-to-end software development using waterfall model with role specialization"
     },
     {
       "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework",
       "relevance": "Demonstrates structured communication via documents and shared message pool in multi-agent SE systems"
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "relevance": "Key paper in end-to-end maintenance; notable finding that full autonomy leads to worst performance among compared agents"
     },
     {
       "title": "Demystifying LLM-based Software Engineering Agents (Agentless)",
       "relevance": "Demonstrates that non-agentic simpler approaches can outperform complex agent systems on SWE-bench Lite"
     },
     {
       "title": "The Rise and Potential of Large Language Model Based Agents: A Survey",
       "relevance": "Foundational general-purpose agent survey providing the four-component framework (planning, memory, perception, action) used throughout this paper"
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "relevance": "Foundational paper on iterative self-refinement and memory mechanisms in agents"
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "relevance": "Widely-used multi-agent framework enabling diverse SE agent architectures reviewed in this survey"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Comprehensive taxonomy of agent tools across all major SE tasks with concrete performance comparisons is directly actionable for SE practitioners and researchers."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that simple non-agentic approaches (Agentless) outperform complex autonomous agents challenges the prevailing assumption that more sophisticated agents are better."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Security-related agents (PentestGPT, vulnerability detection) are covered but the survey does not foreground safety risks from autonomous agents."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild debate about complex vs. simple agents; the simplicity-beats-complexity observation creates some implicit controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository at https://github.com/FudanSELab/Agent4SE-Paper-List is provided; many of the 124 reviewed tools have public implementations."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Fudan University and UIUC are respected but not the top-tier industry labs (Google/Meta/OpenAI) that drive viral attention for AI papers."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "40295298",
         "title": "Agent Hospital that simulates the entire process of treating illness",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40295298",
         "created_at": "2024-05-08T07:31:20Z"
       },
       {
         "hn_id": "39870179",
         "title": "SportsNGEN: Sustained Generation of Multi-Player Sports Gameplay",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39870179",
         "created_at": "2024-03-29T23:27:12Z"
       },
       {
         "hn_id": "39680785",
         "title": "Abstracting Denotational Interpreters",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=39680785",
         "created_at": "2024-03-12T15:36:26Z"
       },
       {
         "hn_id": "40556931",
         "title": "Agent Hospital: A Simulacrum of Hospital with Evolvable Medical Agents",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40556931",
         "created_at": "2024-06-02T20:17:22Z"
       },
       {
         "hn_id": "39680189",
         "title": "VideoMamba: State Space Model for Efficient Video Understanding",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39680189",
         "created_at": "2024-03-12T14:48:56Z"
       },
       {
         "hn_id": "28684178",
         "title": "Inconsistency in Conference Peer Review: Revisiting the 2014 NeurIPS Experiment",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=28684178",
         "created_at": "2021-09-28T15:47:40Z"
       },
       {
         "hn_id": "28615448",
         "title": "Inconsistency in Conference Peer Review: Revisiting the 2014 NeurIPS Experiment",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=28615448",
         "created_at": "2021-09-22T12:15:06Z"
       },
       {
         "hn_id": "38924816",
         "title": "Complex systems approach to natural language",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38924816",
         "created_at": "2024-01-09T11:19:20Z"
       }
     ],
     "top_points": 5,
     "total_points": 19,
     "total_comments": 1
   }
 }