ai-research-survey

scan.json (24471B)

---

 {
   "paper": {
     "title": "From LLM Reasoning to Autonomous AI Agents: A Comprehensive Review",
     "authors": [
       "Mohamed Amine Ferrag",
       "Norbert Tihanyi",
       "Merouane Debbah"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.19678",
     "doi": "10.48550/arXiv.2504.19678"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, analysis scripts, or supplementary materials are mentioned or linked in the paper. A survey can release its search corpus, comparison data, or analysis scripts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The benchmark comparison tables (Tables II-IV) and framework tables (Tables V, VII-XI) are in the paper text only, not available as structured downloadable data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tooling specifications are provided. The paper is a literature survey with no computational experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions for reproducing the survey's paper selection, taxonomy construction, or analysis process are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not conduct experiments; no original statistical results to report uncertainty for."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experiments or statistical comparisons."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experiments. Numbers reported are from reviewed papers, not the authors' own analyses."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no experimental sample sizes involved."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides a structured comparison of the authors' survey against 13 prior related surveys, marking coverage of benchmarks, frameworks, applications, protocols, and challenges for each."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared surveys are from 2024-2025 (Wang et al. 2024, Jin et al. 2024, Singh et al. 2025, Yehudai et al. 2025, Yan et al. 2025, etc.), which are contemporary with this 2025 paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments requiring evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments or test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive categorization: Figure 2 classifies benchmarks into 8 categories; Table IV breaks down benchmarks by multimodal capability, task diversity, reasoning level, and agentic AI support; frameworks and applications are organized by domain (healthcare, finance, SE, etc.)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V.B ('Why Do Multi-Agent LLM Systems Fail?') explicitly discusses 14 failure modes of multi-agent systems grouped into three categories. Section V also discusses reasoning limitations and protocol vulnerabilities."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings from reviewed benchmarks: LLMs score below 10% on HLE (Section III.D), GPT-4 achieves only 15% on GAIA (Section III.V), BBEH average accuracy is 9.8% (Section III.T), and Claude 3.5 Sonnet achieves only 26.2% pass rate on SWE-Lancer (Section III.O)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to present a benchmark comparison (Tables II-IV), a taxonomy of ~60 benchmarks (Figure 2), a review of AI agent frameworks (Table V), applications (Tables VII-XI), and agent protocols (Section IV.C, Table XII). All are substantiated in the paper body."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a descriptive survey that makes no causal claims of its own. Claims about system performance are reported from reviewed papers."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract explicitly bounds scope: 'benchmarks developed between 2019 and 2025' and 'AI-agent frameworks introduced between 2023 and 2025.' The title's claim of 'comprehensive review' is somewhat overbroad, but the specific scope is stated."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey/taxonomy with no original empirical results to explain."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no measurements of its own."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use any models directly."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the reviewed papers were selected. There is no documentation of search queries, databases searched, screening criteria, or the filtering pipeline. The paper jumps directly to presenting benchmarks and frameworks without explaining how they were identified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section V ('Challenges and Open Problems') discusses challenges in the field of AI agents, not limitations of the survey itself. Section VI (Conclusion) does not include limitations of the survey's methodology. No dedicated section discusses the survey's own shortcomings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to the validity of the survey are discussed. There is no consideration of selection bias in paper inclusion, potential gaps in coverage, or limitations of the ad-hoc collection methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states temporal boundaries: 'benchmarks developed between 2019 and 2025' and 'AI-agent frameworks introduced between 2023 and 2025.' The paper structure (Figure 1) delineates what is covered."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No supplementary data, paper lists, or analysis artifacts are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the ~60 benchmarks, frameworks, or applications were identified. No search databases, queries, or collection methodology are documented."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "No description of how papers were found or selected for inclusion. The survey appears to be based on ad-hoc collection rather than a systematic search."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the pipeline from initial paper discovery to final inclusion. No filtering stages, exclusion criteria, or counts at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present. Authors are affiliated with Technology Innovation Institute (UAE) and Khalifa University, but no funding source is stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Guelma University (Algeria), Technology Innovation Institute (UAE), Eötvös Loránd University (Hungary), and Khalifa University (UAE)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors' affiliations with TII and Khalifa University are noted, but without funding disclosure, funder independence is unverifiable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no method of its own whose cost could be reported."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, protocol registration, or systematic search strategy is described. The paper does not document search queries, databases used, or reproducible selection criteria. Papers appear to be collected ad-hoc."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its source papers. Table IV classifies benchmarks by features (multimodal, reasoning, agentic) but does not evaluate rigor, validity, or quality. All papers are treated equally regardless of their methodological strength."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether its sources are biased toward positive results, whether negative or null benchmark results are underrepresented, or whether the landscape of published frameworks is biased toward novel approaches."
       }
     }
   },
   "claims": [
     {
       "claim": "The landscape of LLM and AI agent evaluation remains fragmented and lacks a unified taxonomy or comprehensive survey.",
       "evidence": "Table I compares 13 prior surveys showing each covers only a subset of themes (benchmarks, frameworks, applications, protocols, challenges). Section II.E argues none integrate the full spectrum.",
       "supported": "moderate"
     },
     {
       "claim": "This survey is the first to systematically combine benchmarks, framework design, application domains, communication protocols, and a forward-looking discussion in a single unified treatment.",
       "evidence": "Table I uses a checkmark matrix to show prior surveys' partial coverage vs. the authors' full coverage across all five themes.",
       "supported": "weak"
     },
     {
       "claim": "State-of-the-art LLMs perform below 10% accuracy on Humanity's Last Exam (HLE).",
       "evidence": "Section III.D reports this from Phan et al. [60], citing DeepSeek R1, OpenAI models, Gemini Thinking, and Anthropic Sonnet 3.5.",
       "supported": "strong"
     },
     {
       "claim": "Even advanced models like Claude 3.5 Sonnet achieve only 26.2% pass rate on SWE-Lancer independent tasks.",
       "evidence": "Section III.O reports this from the OpenAI SWE-Lancer paper [71].",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 with plugins achieves only 15% on GAIA while humans achieve 92%.",
       "evidence": "Section III.V reports this from Mialon et al. [78].",
       "supported": "strong"
     },
     {
       "claim": "Multi-agent LLM systems continue to underperform compared to single-agent counterparts due to 14 distinct failure modes.",
       "evidence": "Section V.B cites Pan et al. [222] who studied 5 open-source frameworks across 150 tasks with expert human annotators.",
       "supported": "moderate"
     },
     {
       "claim": "The MCP protocol faces critical security vulnerabilities due to its decentralized design, including lack of standardized authentication.",
       "evidence": "Section V.F discusses vulnerabilities cited from Hou et al. [216], including uneven defenses, no standardized authentication, and deficient logging/debugging.",
       "supported": "moderate"
     }
   ],
   "key_findings": "This survey taxonomizes approximately 60 LLM and AI agent benchmarks from 2019-2025 across eight categories, reviews major agent frameworks (LangChain, LlamaIndex, CrewAI, Swarm, Agents SDK), and catalogs applications across healthcare, finance, software engineering, mathematics, chemistry, and multimedia domains. The paper also surveys three agent communication protocols (ACP, MCP, A2A) and identifies key challenges including multi-agent failure modes, reasoning limitations, and protocol security vulnerabilities. A notable gap is that the survey does not employ a systematic review methodology or assess the quality of its source papers.",
   "red_flags": [
     {
       "flag": "No systematic search methodology",
       "detail": "The survey does not describe how the ~60 benchmarks, frameworks, or applications were identified. No search databases, queries, inclusion/exclusion criteria, or PRISMA-style flow diagram are provided. Papers appear to be collected ad-hoc, making the claimed 'comprehensive' coverage unverifiable."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "The survey treats all reviewed papers equally regardless of methodological quality. Benchmarks, frameworks, and applications are described descriptively without evaluating their rigor, validity, or limitations. This launders the signal-to-noise ratio of its sources — weak papers are given equal standing with strong ones."
     },
     {
       "flag": "Duplicate paragraph",
       "detail": "Section II.A contains Jin et al. [48] described twice in nearly identical paragraphs, suggesting incomplete editing. The text 'In a complementary study, Jin et al. [48] investigate...' repeats the preceding paragraph verbatim."
     },
     {
       "flag": "Self-citation prominence",
       "detail": "Authors prominently feature their own benchmarks — CyberMetric [75] by Tihanyi et al., DIA [74] by Tihanyi et al., and CASTLE [79] by Dubniczky/Tihanyi/Ferrag — in the benchmark tables and discussion. While self-citation is not inherently problematic, three self-authored benchmarks are included among ~60 total (~5%) without explicit disclosure of authorship overlap."
     },
     {
       "flag": "Comprehensiveness claim without evidence",
       "detail": "The paper claims to be a 'comprehensive review' and 'the first to systematically combine' five themes, but provides no evidence of systematic coverage. Without a defined search protocol, it is impossible to assess what was missed or whether inclusion was biased toward certain subfields or author networks."
     },
     {
       "flag": "No survey limitations section",
       "detail": "Section V discusses challenges in the AI agent field, but the paper never discusses limitations of the survey itself — potential selection bias, coverage gaps, recency bias, or geographic/language biases in the reviewed literature."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real World Freelance Software Engineering?",
       "authors": ["S. Miserendino", "M. Wang", "T. Patwardhan", "J. Heidecke"],
       "year": 2025,
       "arxiv_id": "2502.12115",
       "relevance": "Benchmark evaluating LLM agents on real-world freelance software engineering tasks with monetary payouts."
     },
     {
       "title": "Agent-as-a-Judge: Evaluate Agents with Agents",
       "authors": ["M. Zhuge", "C. Zhao", "D. Ashley"],
       "year": 2024,
       "arxiv_id": "2410.10934",
       "relevance": "Novel evaluation methodology using agentic systems to evaluate other agents, achieving 90% alignment with human judgments on code generation tasks."
     },
     {
       "title": "MultiAgentBench: Evaluating the Collaboration and Competition of LLM Agents",
       "authors": ["K. Zhu", "H. Du", "Z. Hong"],
       "year": 2025,
       "arxiv_id": "2503.01935",
       "relevance": "Benchmark for multi-agent LLM coordination across six domains including collaborative coding and research writing."
     },
     {
       "title": "R2E-Gym: Procedural Environments and Hybrid Verifiers for Scaling Open-Weights SWE Agents",
       "authors": ["N. Jain", "J. Singh", "M. Shetty"],
       "year": 2025,
       "arxiv_id": "2504.07164",
       "relevance": "Training environment for SWE agents achieving 51% pass rate on SWE-Bench Verified with synthetic data curation."
     },
     {
       "title": "Training Software Engineering Agents and Verifiers with SWE-Gym",
       "authors": ["J. Pan", "X. Wang", "G. Neubig"],
       "year": 2024,
       "arxiv_id": "2412.21139",
       "relevance": "First dedicated training environment for real-world SWE agents with 2,438 Python task instances."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["G. Mialon", "C. Fourrier", "T. Wolf", "Y. LeCun", "T. Scialom"],
       "year": 2023,
       "relevance": "Benchmark exposing large gap between human (92%) and SOTA model (15%) performance on general AI assistant tasks."
     },
     {
       "title": "Why Do Multiagent Systems Fail?",
       "authors": ["M. Z. Pan", "M. Cemri", "L. A. Agrawal"],
       "year": 2025,
       "relevance": "Critical analysis of 14 failure modes in multi-agent LLM systems across 150 tasks and 5 frameworks."
     },
     {
       "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["J. S. Chan", "N. Chowdhury", "O. Jaffe"],
       "year": 2025,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark evaluating ML engineering agents on Kaggle-style competitions."
     },
     {
       "title": "MLGym: A New Framework and Benchmark for Advancing AI Research Agents",
       "authors": ["D. Nathani", "L. Madaan", "N. Roberts"],
       "year": 2025,
       "arxiv_id": "2502.14499",
       "relevance": "Framework and benchmark for ML research agents that automate ML research workflows."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["N. Shinn", "F. Cassano", "A. Gopinath", "K. Narasimhan", "S. Yao"],
       "year": 2023,
       "relevance": "Foundational work on language agent self-improvement through verbal reinforcement learning."
     },
     {
       "title": "CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection",
       "authors": ["R. A. Dubniczky", "K. Z. Horvát", "T. Bisztray", "M. A. Ferrag", "N. Tihanyi"],
       "year": 2025,
       "arxiv_id": "2503.09433",
       "relevance": "Benchmark for software vulnerability detection comparing static analyzers, LLMs, and formal verification across 25 CWEs."
     },
     {
       "title": "SWE-PolyBench: A Multi-Language Benchmark for Repository Level Evaluation of Coding Agents",
       "authors": ["M. S. Rashid", "C. Bock", "Y. Zhuang"],
       "year": 2025,
       "relevance": "Multi-language benchmark for evaluating coding agents at repository level across programming languages."
     },
     {
       "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents",
       "authors": ["J. Wei", "Z. Sun", "S. Papay"],
       "year": 2025,
       "relevance": "Benchmark for evaluating browsing agents from OpenAI, testing information retrieval capabilities."
     },
     {
       "title": "Towards an AI Co-Scientist",
       "authors": ["J. Gottweis", "W.-H. Weng", "A. Daryin"],
       "year": 2025,
       "arxiv_id": "2502.18864",
       "relevance": "Google's multi-agent system for automated scientific hypothesis generation and refinement using Gemini 2.0."
     },
     {
       "title": "DARS: Dynamic Action Re-Sampling to Enhance Coding Agent Performance by Adaptive Tree Traversal",
       "authors": ["V. Aggarwal", "O. Kamal", "A. Japesh"],
       "year": 2025,
       "arxiv_id": "2503.14269",
       "relevance": "Method for scaling inference-time compute in coding agents, achieving 55% pass@k on SWE-Bench Lite."
     }
   ]
 }