ai-research-survey

scan-v5.json (18524B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Evaluation and Benchmarking of LLM Agents: A Survey",
     "authors": [
       "Mahmoud Mohammadi",
       "Yipeng Li",
       "Jane Lo",
       "Wendy Yip"
     ],
     "year": 2025,
     "venue": "KDD '25",
     "arxiv_id": "2507.21504",
     "doi": "10.1145/3711896.3736570"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract promises a two-dimensional taxonomy, enterprise-specific challenges, and future research directions — all of which are present in the paper's structure and sections.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a taxonomy and narrative survey; it makes no causal claims about what interventions improve evaluation outcomes.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are framed as organizing existing literature and identifying gaps rather than asserting empirical findings; enterprise challenges are framed as observed gaps, not universal laws.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The two-dimensional taxonomy is presented as the natural organizing structure without acknowledging alternative taxonomic frameworks or justifying why this decomposition is superior.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a taxonomy survey with no empirical measurements; there is no gap between measured proxy and claimed outcome to evaluate.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 on future research directions is forward-looking, not a self-critical limitations section; no limitations or threats-to-validity section exists.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed — the survey's non-systematic paper selection, potential coverage gaps, and enterprise framing bias are not acknowledged.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper never specifies which years, venues, or paper types were included or excluded; boundaries of the review are entirely implicit.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section appears in the paper; all four authors are SAP Labs employees but no funding is disclosed.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors list SAP Labs with city/location explicitly in the paper header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are SAP Labs employees; the survey devotes a full section to enterprise-specific challenges (RBAC, compliance, reliability) that align with SAP's commercial interests without disclosing this potential bias.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or financial interests declaration appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'LLM-based agents' are explicitly defined as 'autonomous or semi-autonomous systems that use LLMs to reason, plan, and act'; taxonomy dimensions and subcategories are defined with examples.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Contributions are stated as two explicit bullet points: (1) a two-dimensional evaluation taxonomy, and (2) identification of enterprise-specific challenges.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper references prior surveys ([121], [107]) and explicitly differentiates its contribution as more holistic and enterprise-focused, though the engagement is brief rather than substantive.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": false,
           "justification": "No search strategy is described anywhere; the paper reads as a curated narrative review with no explanation of how the 127 references were identified.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": false,
           "justification": "No inclusion or exclusion criteria are stated; it is impossible to determine why specific benchmarks and papers were included or why others were omitted.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA flowchart or any other structured review protocol is mentioned or followed.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": false,
           "justification": "No search queries, keywords, or search strings are provided anywhere in the paper.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": false,
           "justification": "No databases or sources (e.g., arXiv, ACM DL, Semantic Scholar) are named as having been searched.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": false,
           "justification": "No screening process with paper counts at each stage is documented; the selection process is entirely opaque.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": false,
           "justification": "Temporal scope, venue coverage, and topic boundaries are never justified; the paper claims to cover 'the emerging field' without bounding what qualifies.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The paper catalogs benchmarks and methods additively without acknowledging any conflicting evidence or disagreement across the reviewed literature.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No quality rubric, risk-of-bias tool, or structured evaluation is applied to any reviewed paper; all cited works are treated as equally reliable.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias is never mentioned; the survey does not acknowledge that available benchmarks and evaluation papers skew toward positive or publishable results.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "There is no meta-analysis, vote counting, or quantitative aggregation of findings across reviewed papers; synthesis is entirely narrative.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "The four future research directions (holistic frameworks, realistic settings, automated evaluation, time/cost-bounded protocols) are connected to gaps documented through the taxonomy review, though the support is qualitative.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Evaluating LLM agents is more complex than evaluating LLMs in isolation because agents operate in dynamic, interactive environments with tools, memory, and coordination.",
       "evidence": "Argued conceptually via analogy (engine vs. car) and supported by citing diverse agent benchmarks that require multi-step, environment-aware evaluation beyond static QA.",
       "supported": "moderate"
     },
     {
       "claim": "Existing surveys focus narrowly on LLM evaluation or specific agent capabilities without a holistic perspective.",
       "evidence": "References [121] and [107] as narrower prior work but does not systematically compare coverage across these surveys.",
       "supported": "weak"
     },
     {
       "claim": "Enterprise applications require evaluation considerations (RBAC, reliability guarantees, compliance) that are rarely addressed in existing literature.",
       "evidence": "Only IntellAgent [45] and TheAgentCompany [97] are cited as partially addressing enterprise constraints; the 'rarely' claim is asserted rather than verified through systematic coverage analysis.",
       "supported": "weak"
     },
     {
       "claim": "Current agents struggle with consistency as measured by the pass^k metric.",
       "evidence": "Directly supported by τ-bench [104] results showing agents fail to maintain consistent performance across repeated trials in retail and airline domains.",
       "supported": "strong"
     },
     {
       "claim": "The two-dimensional taxonomy (evaluation objectives × evaluation process) brings clarity to the fragmented agent evaluation landscape.",
       "evidence": "The taxonomy is mapped to 127 references and visualized in a hierarchical tree and Table 1, but no formal evaluation of the taxonomy's completeness or comparative utility is provided.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "survey",
     "qualitative"
   ],
   "key_findings": "This KDD '25 survey proposes a two-dimensional taxonomy of LLM agent evaluation organized by evaluation objectives (agent behavior, capabilities, reliability, safety) and evaluation process (interaction modes, datasets/benchmarks, metrics computation, tooling, contexts). The paper identifies enterprise-specific evaluation gaps including role-based access control, reliability guarantees across repeated runs, long-horizon interaction assessment, and domain-specific compliance verification — areas underserved by academic benchmarks. Future research directions include holistic multi-dimensional frameworks, more realistic enterprise-like evaluation environments, automated and scalable evaluation techniques, and time/cost-bounded protocols. The survey is non-systematic with no described search methodology, making it a curated overview rather than a rigorous literature review.",
   "red_flags": [
     {
       "flag": "Non-systematic paper selection",
       "detail": "No search strategy, inclusion/exclusion criteria, databases searched, or screening process is described; the 127 references appear hand-curated with no transparency about omissions."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "All reviewed benchmarks and evaluation papers are treated equally with no methodological quality assessment, making it impossible to distinguish rigorous from weak evaluations."
     },
     {
       "flag": "Undisclosed enterprise conflict of interest",
       "detail": "All four authors are SAP Labs employees; the survey devotes Section 5 to enterprise-specific challenges that align with SAP's commercial interests, with no disclosure of this potential framing bias."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section, omitting discussion of coverage gaps, selection bias, recency constraints, or non-systematic methodology."
     },
     {
       "flag": "No publication bias acknowledgment",
       "detail": "The survey does not acknowledge that the available corpus of agent evaluation benchmarks and papers skews heavily toward positive or publishable results."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "relevance": "Central benchmark for evaluating LLMs across diverse task environments; anchor reference for the evaluation objectives dimension"
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Representative software engineering benchmark illustrating task completion evaluation in coding domains"
     },
     {
       "title": "Holistic Evaluation of Language Models (HELM)",
       "relevance": "Reference framework for multi-dimensional evaluation incorporating toxicity, bias, and robustness alongside task performance"
     },
     {
       "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
       "relevance": "Introduces pass^k consistency metric; primary reference for reliability evaluation in enterprise-relevant domains"
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "relevance": "Frequently cited as a realistic dynamic evaluation environment exemplifying online/interactive evaluation mode"
     },
     {
       "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents",
       "relevance": "Core reference for the safety/harm evaluation dimension"
     },
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents",
       "relevance": "Key benchmark for adversarial robustness and security evaluation of agents"
     },
     {
       "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks",
       "relevance": "Enterprise-relevant benchmark with organizational policies; cited for both compliance evaluation and enterprise challenges sections"
     },
     {
       "title": "Survey on Evaluation of LLM-based Agents (Yehudai et al., 2025)",
       "relevance": "Prior related survey that the authors explicitly position their work against as being narrower in scope"
     },
     {
       "title": "HAL: A Holistic Agent Leaderboard for Centralized and Reproducible Agent Evaluation",
       "relevance": "Infrastructure reference for standardized leaderboard-based centralized evaluation"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners designing LLM agent evaluation pipelines can use the taxonomy to structure coverage across behavior, capabilities, reliability, and safety dimensions."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The enterprise-specific challenges section surfaces underappreciated evaluation requirements (RBAC, pass^k consistency) not commonly foregrounded in academic benchmark literature."
     },
     "fear_safety": {
       "score": 1,
       "justification": "The safety section covers harm, toxicity, prompt injection, and compliance risks, but the paper's primary contribution is organizational rather than a safety alarm."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, debate between competing approaches, or conflicting findings are surfaced; the paper is a neutral taxonomy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "The paper offers no artifact, tool, dataset, or interactive system that readers can immediately access or try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "SAP is a well-known enterprise software vendor but not a top-tier AI research lab; KDD '25 venue adds credibility but is not a top-prestige AI venue for surveys."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44120359",
         "title": "Diffusion vs. Autoregressive Language Models: A Text Embedding Perspective",
         "points": 19,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=44120359",
         "created_at": "2025-05-28T20:27:45Z"
       },
       {
         "hn_id": "45472586",
         "title": "Physics of Learning: A Lagrangian perspective to different learning paradigms",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45472586",
         "created_at": "2025-10-04T11:38:44Z"
       },
       {
         "hn_id": "36931866",
         "title": "Universal and Transferable Adversarial Attacks on LLM",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36931866",
         "created_at": "2023-07-30T15:04:08Z"
       },
       {
         "hn_id": "45418635",
         "title": "Can LLMs Be Creative? Paper: Combinatorial Creativity: A New Frontier",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45418635",
         "created_at": "2025-09-29T20:53:22Z"
       },
       {
         "hn_id": "41174642",
         "title": "Case-Based Reasoning for Explainable Depression Detection on Twitter Using LLMs",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41174642",
         "created_at": "2024-08-06T19:55:38Z"
       },
       {
         "hn_id": "36903968",
         "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36903968",
         "created_at": "2023-07-28T07:30:39Z"
       }
     ],
     "top_points": 19,
     "total_points": 29,
     "total_comments": 1
   }
 }