ai-research-survey

scan.json (22179B)

---

 {
   "paper": {
     "title": "Agentic Artificial Intelligence (AI): Architectures, Taxonomies, and Evaluation of Large Language Model Agents",
     "authors": ["Arunkumar V", "Gangadharan G.R.", "Rajkumar Buyya"],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2601.12560"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided. The paper is a survey with no analysis scripts or data released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper surveys existing literature and does not release a structured corpus or extracted data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification or dependency list is provided. The paper is a survey with no experiments requiring reproducible setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. There are no experiments to reproduce, and no methodology artifact (e.g., search query logs, inclusion/exclusion scripts) is released."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/review paper with no original statistical experiments. No CIs or error bars are applicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No original comparative experiments are conducted; significance tests do not apply to this survey paper."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments are run; effect sizes are not applicable to this survey."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical experiment is conducted; sample size justification does not apply."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments are run; variance reporting does not apply to this survey."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare itself to prior surveys in any systematic way. It briefly mentions prior surveys (Wang et al., Xi et al., Luo et al.) in the introduction but offers no structured comparison of coverage or quality against them."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baseline comparison experiments are conducted; this criterion does not apply."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system is built or ablated; the paper is a survey with no components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments are conducted; multiple metrics do not apply."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of the survey's own outputs is not relevant; no system outputs are produced."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No train/test split is relevant for a survey paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey organizes reviewed systems into six taxonomy dimensions (Core Components, Cognitive Architecture, Learning, Multi-Agent Systems, Environments, Evaluation) and provides multiple tables with per-category breakdowns of architectures and frameworks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 ('Challenges and Future Directions') discusses failure modes including hallucination in action, infinite loops, and latency bottlenecks. Tables 1-4 list critical limitations of each system category."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The survey explicitly notes limitations of current approaches: WebArena shows low success rates (< 15%) on long-horizon tasks; synchronous vs asynchronous agent performance drops from 47% to 11% (Section 7.2); prompt-only defenses are described as brittle."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to propose a unified taxonomy covering Perception, Brain, Planning, Action, Tool Use, and Collaboration, and to review evaluation practices and open challenges. The paper delivers these in Sections 3-8, so the abstract claims are supported."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that describes and synthesizes existing work; it does not make original causal claims about what causes performance improvements."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about 'Agentic AI' and 'LLM-based agents' in general, often reporting results from individual papers (e.g., '30% reduction in bugs' from ChatDev, '50% navigation improvement' from VLM-GroNav) as if they generalize broadly without adequately bounding these to the specific settings and benchmarks in which they were obtained."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/taxonomy paper that presents no original empirical results requiring alternative explanations. NA for pure surveys that do not present original empirical findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are run; model version specification does not apply to this survey paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in its own methodology; it is a survey."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are run; hyperparameter reporting does not apply."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not build or deploy an agentic scaffold; it surveys existing scaffolds."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe any systematic literature search methodology: no search query strings, databases searched, date ranges, inclusion/exclusion criteria, or counts of papers at each screening stage are provided. The review corpus is opaque."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 ('Challenges and Future Directions') provides substantive discussion of open challenges and limitations including hallucination in action, infinite loops, latency/cost, alignment, and theoretical limits."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 discusses challenges in the field broadly but does not address threats to validity of the survey itself, such as selection bias in the reviewed literature, recency bias, or lack of systematic search methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it did NOT cover or what claims it is not making about the scope. No explicit exclusion criteria or out-of-scope domains are stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (list of surveyed papers, extraction tables, coding sheets) is released. The survey's evidence base cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "No systematic literature search process is described. The paper does not explain how papers were identified, what databases were searched, or what time period was covered."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The question of recruitment does not apply to this survey paper."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "There is no documented pipeline from paper collection to inclusion in the survey. No filtering stages or criteria are described, making the selection process entirely opaque."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments or funding section is present in the paper. There is no mention of funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the title page: University College of Engineering (Anna University), National Institute of Technology Tiruchirappalli, and University of Melbourne. The paper does not evaluate tools from any of these institutions' proprietary products."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed. The question is NA due to apparent lack of external funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey and does not evaluate a pre-trained model on any benchmark. Contamination questions do not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model on a benchmark; this criterion does not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not run any benchmarks; contamination is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; inclusion/exclusion criteria for participants are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no original system being evaluated; inference cost of the survey's own method does not apply."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; compute budget for the survey itself is not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "The paper proposes a unified taxonomy decomposing LLM-based agents into six modular dimensions: Core Components, Cognitive Architecture, Learning, Multi-Agent Systems, Environments, and Evaluation.",
       "evidence": "Section 3 (Taxonomy) and Figure 1 present this taxonomy. The taxonomy is described as the organizing structure for the paper.",
       "supported": "strong"
     },
     {
       "claim": "Hierarchical architectures such as ReAcTree maximize task proficiency and reasoning depth but incur exponential increases in token consumption compared to linear chains.",
       "evidence": "Section 7.1 and Figure 4 present a multidimensional architectural comparison. Table 4 shows token complexity estimates for different architectures.",
       "supported": "moderate"
     },
     {
       "claim": "Synchronous agents achieve 47% success on asynchronous tasks while performance drops to 11% in asynchronous settings, showing a critical lack of temporal awareness.",
       "evidence": "Section 7.2 cites Robotouille benchmark [138]. The paper does not conduct this evaluation itself but reports the finding from the cited work.",
       "supported": "moderate"
     },
     {
       "claim": "ChatDev achieves a 30% reduction in bugs compared to single-agent coding.",
       "evidence": "Section 5.3.1 states this claim citing ChatDev [84]. No details on evaluation methodology or sample size are provided in the survey.",
       "supported": "weak"
     },
     {
       "claim": "Supervisory architectures (OVON, Good Parenting) can reduce hallucination rates by up to 100% in controlled environments.",
       "evidence": "Section 5.3.2 cites [88] and [89]. The claim of '100% reduction' is extraordinary and the survey provides no methodological details to assess its basis.",
       "supported": "weak"
     },
     {
       "claim": "VLM-GroNav improves navigation success by 50% by integrating proprioceptive sensing with vision.",
       "evidence": "Section 6.2.3 states this but cites [119] without providing evaluation details, benchmark, or baseline context.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["qualitative", "case-study"],
   "key_findings": "This survey proposes a unified architecture-focused taxonomy for LLM-based agents organized around six dimensions: Core Components (perception, memory, action, profiling), Cognitive Architecture (planning, reflection), Learning, Multi-Agent Systems, Environments, and Evaluation. The paper traces the evolution from simple ReAct-style loops to hierarchical multi-agent systems and reasoning models with inference-time compute scaling. It highlights that modern agent evaluation should use the CLASSic framework (Cost, Latency, Accuracy, Security, Stability) and that security threats such as indirect prompt injection are a primary barrier to real-world deployment. Open challenges identified include hallucination in action, infinite loops, computational cost, and human-agent alignment.",
   "red_flags": [
     {
       "flag": "No systematic literature search methodology",
       "detail": "The paper provides no description of how papers were identified and selected: no search queries, databases, date ranges, or inclusion/exclusion criteria are stated. This makes it impossible to assess coverage or selection bias, and the survey cannot be replicated or updated systematically."
     },
     {
       "flag": "Uncritical laundering of weak empirical claims",
       "detail": "The survey repeats strong-sounding quantitative claims from cited papers (e.g., '30% bug reduction' from ChatDev, '100% hallucination reduction' from Good Parenting, '50% navigation improvement' from VLM-GroNav) without evaluating the methodological quality of the underlying studies or noting their limitations. These numbers are presented as established facts."
     },
     {
       "flag": "No competing interests or funding disclosure",
       "detail": "The paper contains no acknowledgments section, no funding statement, and no competing interests declaration."
     },
     {
       "flag": "Overly broad generalizations",
       "detail": "Results from narrow, controlled benchmarks or controlled environments are regularly described as general findings about 'Agentic AI' without bounding the scope to the tested conditions. For example, hallucination reduction claims are from 'controlled environments' but are cited in sections discussing general deployment."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "ReAct is a foundational agentic pattern for LLM agents combining reasoning and acting; highly relevant to the survey's core topic."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["J. Yang"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "SWE-agent is a prominent benchmark system for LLM-based software engineering agents, directly in the survey scope."
     },
     {
       "title": "A Survey on Large Language Model based Autonomous Agents",
       "authors": ["L. Wang"],
       "year": 2024,
       "relevance": "A major prior survey on LLM-based autonomous agents; relevant for understanding the coverage gap this paper addresses."
     },
     {
       "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework",
       "authors": ["S. Hong"],
       "year": 2024,
       "relevance": "MetaGPT is a widely cited multi-agent software engineering framework using SOPs; directly in scope for this survey."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["G. Wang"],
       "year": 2024,
       "relevance": "Voyager is a flagship embodied LLM agent for open-ended learning in games; relevant to agentic AI capabilities."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["N. Shinn"],
       "year": 2023,
       "relevance": "Reflexion is a key cognitive architecture technique for LLM agent self-improvement via verbal feedback."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["S. Yao"],
       "year": 2024,
       "relevance": "Tree of Thoughts is a major planning architecture for LLM agents; central to the cognitive architecture section."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration",
       "authors": ["G. Li"],
       "year": 2023,
       "relevance": "CAMEL is an early and influential multi-agent framework; directly in scope for multi-agent LLM systems survey."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["C. Wang"],
       "year": 2024,
       "relevance": "AutoGen is a major multi-agent framework for LLM applications; highly relevant to multi-agent systems research."
     },
     {
       "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
       "authors": ["T. Xie"],
       "year": 2025,
       "arxiv_id": "2404.07972",
       "relevance": "OSWorld is a key benchmark for evaluating OS-level agentic AI; directly in scope for this survey's evaluation section."
     },
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "authors": ["G. Mialon"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "GAIA is a benchmark for general AI assistants requiring multi-step tool use; relevant to agent evaluation."
     },
     {
       "title": "Formalizing and Benchmarking Prompt Injection Attacks and Defenses",
       "authors": ["Y. Liu"],
       "year": 2023,
       "arxiv_id": "2310.12815",
       "relevance": "Foundational work on prompt injection attacks against LLM agents; central to the security section."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["X. Liu"],
       "year": 2024,
       "relevance": "AgentBench is a comprehensive multi-environment benchmark for LLM agents; central to the evaluation section of this survey."
     },
     {
       "title": "Large language model agent: A survey on methodology, applications and challenges",
       "authors": ["J. Luo"],
       "year": 2025,
       "arxiv_id": "2503.21460",
       "relevance": "A competing/complementary survey on LLM agents from a methodology perspective; directly relevant for understanding the survey landscape."
     },
     {
       "title": "Top of the CLASS: Benchmarking LLM Agents on Real-World Enterprise Tasks",
       "authors": ["M. Wornow"],
       "year": 2025,
       "relevance": "Introduces the CLASSic evaluation framework (Cost, Latency, Accuracy, Security, Stability) adopted by this survey."
     }
   ]
 }