ai-research-survey

scan.json (20631B)

---

 {
   "paper": {
     "title": "A Comprehensive Survey on Benchmarks and Solutions in Software Engineering of LLM-Empowered Agentic System",
     "authors": ["Jiale Guo", "Suizhi Huang", "Mei Li", "Dong Huang", "Xingsheng Chen", "Regina Zhang", "Zhijiang Guo", "Han Yu", "Siu-Ming Yiu", "Pietro Lio", "Kwok-Yan Lam"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.09721"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey reviews 150+ papers on LLM-powered software engineering, providing a taxonomy across Solutions (prompt-based, fine-tuning, agent-based) and Benchmarks (code generation, translation, repair, others). It maps 50+ benchmarks to their corresponding solution strategies and identifies research gaps including multi-agent collaboration, self-evolving systems, and formal verification integration. The analysis traces the evolution from simple prompt engineering to sophisticated agentic systems with planning, reasoning, memory, and tool augmentation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We maintain a GitHub repository that continuously updates the reviewed and related papers at https://github.com/lisaGuojl/LLM-Agent-SE-Survey' in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No structured dataset of the surveyed papers (e.g., extracted metadata, classification labels, analysis data) is released. The GitHub repo appears to be a paper list, not analysis data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a survey, analysis scripts and their dependencies could have been documented but were not."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the survey's paper selection, classification, or analysis are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments or report quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no statistical comparisons of its own."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental results."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no experimental sample size to justify."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I compares this survey against 5 prior surveys (Zhang et al., Jiang et al., Wang et al., Dong et al., Sapkota et al.) across features like taxonomy, pipeline, prompt, agent, benchmarks, solutions, and scope."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared surveys are from 2023-2025, which are contemporary and relevant."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no system evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for a survey's taxonomy claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The taxonomy provides per-category breakdowns across Solutions (prompt-based, fine-tune-based, agent-based subcategories) and Benchmarks (code generation, translation, repair, others). Figure 4 shows distribution across subcategories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses limitations and failure modes of reviewed approaches, e.g., 'current LLMs struggle to use iterative feedback when presented as I/O examples' (Sec. IV-A), performance degradation on R language (Sec. IV-A), and SWE-bench solutions failing in production (Sec. VII-B/VIII-B)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings from reviewed work, e.g., 'incorporating semantic information from execution traces provided limited SFT benefit' (Sec. IV-B), LLM performance degrades on R language, and most LLM-correct solutions would fail code review in production."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: the paper does review 150+ papers, provides a taxonomy along Solutions and Benchmarks dimensions, connects 50+ benchmarks to solutions, and identifies future directions. These are structural claims about the survey's coverage, which the paper delivers."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey and does not make its own causal claims. It reports causal claims from reviewed papers."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims comprehensive coverage of 'Software Engineering of LLM-Empowered Agentic System' but the scope is primarily code generation, translation, and repair. Requirements engineering, deployment, DevOps, and other SE activities receive minimal coverage. The selection criteria mention top venues but actual coverage includes substantial arXiv preprints."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "As a survey/taxonomy paper with no empirical results of its own, alternative explanations are not applicable."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical/survey paper with no measurements of its own."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use any models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states selection criteria (papers from 2023-2025 from top venues, novel benchmarks/methods, significant empirical evaluation) but does not provide counts at each filtering stage, the exact search queries used, or the specific process that led from an initial set to the final 150+ papers. No PRISMA-style flow or filtering pipeline with counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The paper has 'Challenges and Future Directions' sections (VII and VIII, which appear duplicated) but these discuss challenges of the field, not limitations of the survey itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity of the survey methodology are discussed. No acknowledgment of potential selection bias in paper collection, classification subjectivity, or coverage gaps."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper states it covers papers from 2023-2025 and focuses on code generation, translation, and repair, it does not explicitly state what is NOT covered or what the results do NOT show. The title claims comprehensive coverage without bounding it."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No list of all reviewed papers with their classifications, extraction data, or analysis spreadsheets is made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section I describes the paper selection process: sources (top conferences, journals, preprints), selection criteria (4 criteria listed), and time period (2023-2025)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is published papers from known venues."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists selection criteria but does not document the pipeline from initial search to final inclusion — no counts of papers found, screened, excluded at each stage, or who performed the screening."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: NTU, HKUST, HKU, Cambridge, SJTU."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no model evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no model evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no method or system to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, no protocol registration, no reproducible search queries. The paper lists general selection criteria and venue sources but does not follow a structured review protocol."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its source papers. All papers are treated equally regardless of their rigor, sample sizes, or evaluation quality. No quality scoring or risk-of-bias assessment."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether the papers it reviews skew toward positive results or whether negative results are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "This is the first comprehensive survey connecting benchmarks and solutions in LLM-empowered software engineering.",
       "evidence": "Table I compares against 5 prior surveys, showing none cover both Benchmarks and Solutions columns with full SE scope. The paper reviews 150+ papers across both dimensions.",
       "supported": "moderate"
     },
     {
       "claim": "The field has evolved from prompt-based methods to sophisticated agent-based systems incorporating planning, reasoning, memory, and tool augmentation.",
       "evidence": "Sections IV-A through IV-C document this progression with specific papers at each stage, from instructional prompting through fine-tuning to multi-agent architectures.",
       "supported": "strong"
     },
     {
       "claim": "Existing surveys lack coverage of benchmarks, making it difficult to evaluate and compare different approaches systematically.",
       "evidence": "Table I shows that 4 of 5 compared surveys have '×' in the Benchmarks column. Only Zhang et al. (2023) covers benchmarks but is limited to program repair.",
       "supported": "strong"
     },
     {
       "claim": "A majority of algorithmically successful solutions on benchmarks like SWE-Bench would fail in production due to security flaws, performance regressions, or coding standard violations.",
       "evidence": "Stated in Section VII-B/VIII-B as a finding from 'recent analyses' but no specific citation or data is provided for this claim.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of sources",
       "detail": "The survey reviews 150+ papers but does not assess their methodological quality. Papers with rigorous evaluation and papers with weak evidence are treated equally, potentially laundering weak results."
     },
     {
       "flag": "No structured review protocol",
       "detail": "No PRISMA flow, no registered protocol, no reproducible search queries. The paper selection process is described only in general terms, making the review difficult to reproduce or verify for completeness."
     },
     {
       "flag": "Duplicated sections",
       "detail": "Sections VII and VIII are both titled 'Challenges and Future Directions' with substantially overlapping content (scalability, evaluation, deployment themes appear twice with slight rewording), suggesting incomplete editing."
     },
     {
       "flag": "Overclaimed scope",
       "detail": "The title claims 'comprehensive' coverage of 'Software Engineering' but the taxonomy focuses primarily on code generation, translation, and repair. Other major SE activities (requirements, design, deployment, maintenance at scale) receive cursory treatment in the Applications section."
     },
     {
       "flag": "Unsupported production-readiness claims",
       "detail": "Section VII-B/VIII-B claims 'a majority of algorithmically successful solutions on benchmarks like SWE-Bench would fail in production' without citing a specific source or data for this assertion."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. R. Narasimhan"],
       "year": 2024,
       "relevance": "Foundational benchmark for evaluating LLM agents on real-world GitHub issue resolution."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["J. Yang", "C. E. Jimenez", "A. Wettig", "K. Lieret", "S. Yao", "K. Narasimhan", "O. Press"],
       "year": 2024,
       "relevance": "Introduced the Agent-Computer Interface concept for software engineering agents."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Y. Zhang", "H. Ruan", "Z. Fan", "A. Roychoudhury"],
       "year": 2024,
       "relevance": "Autonomous agent for program repair using iterative code search and patch generation."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark, foundational for code generation evaluation."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["T. Y. Zhuo"],
       "year": 2025,
       "relevance": "Benchmark evaluating LLMs on complex code generation with diverse function calls."
     },
     {
       "title": "Demystifying llm-based software engineering agents",
       "authors": ["C. S. Xia", "Y. Deng", "S. Dunn", "L. Zhang"],
       "year": 2025,
       "relevance": "Agentless approach to software engineering showing simple plans can be sufficient."
     },
     {
       "title": "MAGIS: LLM-based multi-agent framework for github issue resolution",
       "authors": ["W. Tao", "Y. Zhou", "Y. Wang", "W. Zhang", "H. Zhang", "Y. Cheng"],
       "year": 2024,
       "relevance": "Multi-agent framework with Manager, Developer, QA Engineer roles for issue resolution."
     },
     {
       "title": "Training software engineering agents and verifiers with SWE-gym",
       "authors": ["J. Pan", "X. Wang", "G. Neubig"],
       "year": 2025,
       "relevance": "Environment for training and evaluating tool-using SE agents via SFT."
     },
     {
       "title": "SWE-smith: Scaling data for software engineering agents",
       "authors": ["J. Yang", "K. Lieret", "C. E. Jimenez"],
       "year": 2025,
       "relevance": "Extends SWE-bench for training SE agents at scale."
     },
     {
       "title": "Kimi K2: Open agentic intelligence",
       "authors": ["K. Team"],
       "year": 2025,
       "arxiv_id": "2507.20534",
       "relevance": "Open model optimized for agentic tool use in code generation and debugging."
     },
     {
       "title": "PatchAgent: A practical program repair agent mimicking human expertise",
       "authors": ["Z. Yu", "Z. Guo", "Y. Wu"],
       "year": 2025,
       "relevance": "Program repair agent with integrated security and functional verification."
     },
     {
       "title": "MASAI: Modular architecture for software-engineering AI agents",
       "authors": ["N. Wadhwa", "A. Sonwane", "D. Arora"],
       "year": 2024,
       "relevance": "Modular agent architecture decomposing issue resolution into specialized sub-agents."
     }
   ]
 }