ai-research-survey

scan-v4.json (17555B)

---

 {
   "scan_version": 4,
   "paper_type": "survey",
   "paper": {
     "title": "Evaluation and Benchmarking of LLM Agents: A Survey",
     "authors": ["Mahmoud Mohammadi", "Yipeng Li", "Jane Lo", "Wendy Yip"],
     "year": 2025,
     "venue": "KDD '25",
     "arxiv_id": "2507.21504",
     "doi": "10.1145/3711896.3736570"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a two-dimensional taxonomy, enterprise challenge coverage, and future directions — all delivered in the paper's six sections.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a taxonomy/survey and makes no causal claims requiring study designs.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents its taxonomy as a comprehensive framework for 'the field' without bounding which papers were reviewed, what time range was covered, or what domains are excluded.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents its two-dimensional taxonomy without discussing alternative taxonomic frameworks or why other organizational schemes were not adopted.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a conceptual taxonomy — it does not measure outcomes or report experimental results.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses 'Future Research Directions' but there is no dedicated limitations or threats-to-validity section anywhere in the paper.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed; the paper does not acknowledge potential gaps in coverage, selection bias, or limitations of the taxonomy.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state explicit scope boundaries — no time range, venue filter, paper type, or excluded topic areas are defined.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no funding acknowledgment or disclosure anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are explicitly listed with SAP Labs affiliations (Bellevue and Palo Alto) on the first page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are from SAP Labs, an enterprise software company, and the paper significantly amplifies enterprise-specific evaluation challenges — a framing that aligns with SAP's business interests.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests, patent disclosures, or financial interests statement is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "LLM agents are defined as 'autonomous or semi-autonomous systems that use LLMs to reason, plan, and act'; taxonomy dimensions (behavior, capabilities, reliability, safety) are explained in Section 2.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Two contributions are explicitly bulleted in Section 1: a two-dimensional taxonomy and identification of enterprise-specific challenges.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper positions itself against prior surveys ([107], [121]) and extensively references prior benchmarks; however comparative depth is limited to a single paragraph in the introduction.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": false,
           "justification": "No search strategy is described; it is unclear how the 127 cited papers were identified or what databases were queried.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": false,
           "justification": "No inclusion or exclusion criteria are stated anywhere; papers appear selected ad hoc based on author familiarity.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA flowchart, structured review protocol, or systematic methodology is mentioned or used.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": false,
           "justification": "No search queries or keywords used to identify papers are provided.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": false,
           "justification": "No databases, repositories, or search engines used to find papers are mentioned.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": false,
           "justification": "No screening process, stage counts, or PRISMA-style funnel is documented.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": false,
           "justification": "The scope is not justified beyond a vague statement that prior surveys are 'too narrow'; no rationale for time range, venue selection, or topic boundaries is provided.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The paper presents a descriptive taxonomy without acknowledging conflicting findings or methodological debates among the benchmarks and evaluation approaches reviewed.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No quality rubric, risk-of-bias assessment, or structured evaluation of reviewed papers is performed; all cited papers are treated as equally authoritative.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias is not mentioned; the survey does not acknowledge that published evaluation benchmarks may skew toward positive results or popular approaches.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "The survey is entirely narrative; there is no meta-analysis, vote counting, effect size aggregation, or even a count of how many papers cover each category.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": false,
           "justification": "Future research directions (holistic frameworks, realistic settings, scalable evaluation) are stated as author opinion without citing systematic evidence of their necessity or comparing alternatives.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Evaluating LLM agents is more complex than evaluating LLMs in isolation, requiring assessment across dynamic, interactive environments.",
       "evidence": "Logical argument by analogy (LLM = engine, agent = car) plus enumeration of agent capabilities. No empirical support.",
       "supported": "moderate"
     },
     {
       "claim": "Existing surveys focus narrowly on LLM evaluation or specific capabilities without a holistic perspective.",
       "evidence": "Single citation to [121] plus general assertion; no systematic comparison of prior surveys.",
       "supported": "weak"
     },
     {
       "claim": "Enterprise applications introduce requirements (RBAC, compliance, reliability, long-horizon) rarely addressed in current research.",
       "evidence": "Section 5 describes these challenges with selective citations but no systematic analysis of what fraction of current benchmarks address them.",
       "supported": "weak"
     },
     {
       "claim": "Current agents struggle with consistency as measured by the pass^k metric.",
       "evidence": "Cites τ-bench [104], which showed agents fail consistency tests in retail and airline booking domains.",
       "supported": "moderate"
     },
     {
       "claim": "Task completion (success rate) is the predominant measure of overall agent performance.",
       "evidence": "Supported by citing multiple major benchmarks (SWE-bench, AgentBench, WebArena) all using success rate as their primary metric.",
       "supported": "strong"
     },
     {
       "claim": "LLM-as-a-Judge has gained traction for evaluating subjective and nuanced agent responses.",
       "evidence": "Multiple citations [30, 46, 125, 127] support uptake, though no quantitative comparison with human evaluation baselines is provided.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical", "qualitative"],
   "key_findings": "This survey proposes a two-dimensional taxonomy for LLM agent evaluation organized by evaluation objectives (behavior, capabilities, reliability, safety) and evaluation process (interaction modes, datasets, metrics, tooling, contexts). A significant secondary contribution is identification of enterprise-specific evaluation gaps: role-based access control, reliability guarantees, long-horizon interactions, and compliance requirements are largely unaddressed by current benchmarks. The paper is a narrative taxonomy rather than a systematic review — no search strategy, inclusion criteria, or paper quality assessment is reported, making coverage selection opaque. Future directions include holistic, realistic, and scalable evaluation frameworks, but these are opinion-based rather than evidence-driven.",
   "red_flags": [
     {
       "flag": "No systematic literature review",
       "detail": "No search strategy, inclusion/exclusion criteria, database listing, or PRISMA protocol. Paper selection appears ad hoc based on author familiarity with the field."
     },
     {
       "flag": "Enterprise bias from SAP authors",
       "detail": "All four authors are from SAP Labs, an enterprise software company. The paper disproportionately amplifies enterprise-specific evaluation challenges in a way that aligns with SAP's product direction, without disclosing this potential conflict."
     },
     {
       "flag": "No funding disclosure",
       "detail": "No acknowledgment of funding sources or statement that the work is independently funded."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "All 127 cited benchmarks and papers are treated as equally authoritative with no quality filtering, risk-of-bias assessment, or critical appraisal."
     },
     {
       "flag": "No quantitative synthesis",
       "detail": "Purely narrative taxonomy with no paper counts per category, coverage statistics, or meta-analytic aggregation across reviewed papers."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity discussion; coverage gaps and potential selection biases are entirely unacknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Key benchmark for coding agent evaluation using GitHub issue resolution as ground truth; highly cited in agent evaluation literature"
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "relevance": "Multi-environment benchmark covering coding, web, games — foundational agent evaluation framework"
     },
     {
       "title": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
       "relevance": "Introduces pass^k consistency metric; demonstrates current agents fail reliability requirements in retail and airline domains"
     },
     {
       "title": "Holistic Evaluation of Language Models (HELM)",
       "relevance": "Comprehensive evaluation framework covering accuracy, robustness, bias, toxicity — canonical holistic evaluation reference"
     },
     {
       "title": "AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents",
       "relevance": "Multi-turn agent evaluation with fine-grained progress rate metric; distinguishes trajectory quality from binary success"
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "relevance": "Canonical web agent benchmark used across many evaluation studies in the survey"
     },
     {
       "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents",
       "relevance": "Safety evaluation benchmark specifically targeting harmful agent behaviors"
     },
     {
       "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks",
       "relevance": "Enterprise task evaluation framework requiring agents to follow organizational policy constraints"
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "relevance": "Foundational paper establishing the LLM-as-a-Judge evaluation paradigm discussed extensively in the survey"
     },
     {
       "title": "Survey on Evaluation of LLM-based Agents",
       "relevance": "Prior survey explicitly positioned against as 'too narrow', motivating this paper's broader scope"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly useful as a reference for practitioners and researchers designing LLM agent evaluation systems; comprehensive benchmark coverage."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The taxonomy is well-organized but not surprising; the enterprise angle adds modest novelty but does not challenge conventional wisdom."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Section 3.4 covers harm, toxicity, bias, and compliance with concrete failure examples (CoSafe coreference attacks, ToolEmu failures), raising substantive safety concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; the paper is a straightforward organizational taxonomy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Conceptual taxonomy paper with no demo, tool, dataset, or interactive artifact."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "SAP Labs is a recognized enterprise software company but not a top AI research lab; KDD venue adds credibility."
     }
   },
   "hn_data": {
     "threads": [
       {"hn_id": "44120359", "title": "Diffusion vs. Autoregressive Language Models: A Text Embedding Perspective", "points": 19, "comments": 1, "url": "https://news.ycombinator.com/item?id=44120359", "created_at": "2025-05-28T20:27:45Z"},
       {"hn_id": "45472586", "title": "Physics of Learning: A Lagrangian perspective to different learning paradigms", "points": 3, "comments": 0, "url": "https://news.ycombinator.com/item?id=45472586", "created_at": "2025-10-04T11:38:44Z"},
       {"hn_id": "36931866", "title": "Universal and Transferable Adversarial Attacks on LLM", "points": 3, "comments": 0, "url": "https://news.ycombinator.com/item?id=36931866", "created_at": "2023-07-30T15:04:08Z"},
       {"hn_id": "45418635", "title": "Can LLMs Be Creative? Paper: Combinatorial Creativity: A New Frontier", "points": 2, "comments": 0, "url": "https://news.ycombinator.com/item?id=45418635", "created_at": "2025-09-29T20:53:22Z"},
       {"hn_id": "41174642", "title": "Case-Based Reasoning for Explainable Depression Detection on Twitter Using LLMs", "points": 1, "comments": 0, "url": "https://news.ycombinator.com/item?id=41174642", "created_at": "2024-08-06T19:55:38Z"},
       {"hn_id": "36903968", "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models", "points": 1, "comments": 0, "url": "https://news.ycombinator.com/item?id=36903968", "created_at": "2023-07-28T07:30:39Z"}
     ],
     "top_points": 19,
     "total_points": 29,
     "total_comments": 1
   }
 }