ai-research-survey

scan.json (28986B)

---

 {
   "paper": {
     "title": "AgentBench: Evaluating LLMs as Agents",
     "authors": [
       "Xiao Liu",
       "Hao Yu",
       "Hanchen Zhang",
       "Yifan Xu",
       "Xuanyu Lei",
       "Hanyu Lai",
       "Yu Gu",
       "Hangliang Ding",
       "Kaiwen Men",
       "Kejuan Yang",
       "Shudan Zhang",
       "Xiang Deng",
       "Aohan Zeng",
       "Zhengxiao Du",
       "Chenhui Zhang",
       "Sheng Shen",
       "Tianjun Zhang",
       "Yu Su",
       "Huan Sun",
       "Minlie Huang",
       "Yuxiao Dong",
       "Jie Tang"
     ],
     "year": 2023,
     "venue": "ICLR 2024",
     "arxiv_id": "2308.03688",
     "doi": "10.48550/arXiv.2308.03688"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Datasets, environments, and an integrated evaluation package for AGENTBENCH are released at https://github.com/THUDM/AgentBench' in the abstract. A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All datasets are publicly available' (Section 4.1) and datasets are released as part of the GitHub repository linked in the abstract."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Docker images (Ubuntu Docker for OS) and the Server-Client architecture, but does not provide a requirements.txt, Dockerfile definition, conda environment, or detailed library version listings. Mentioning that tasks run inside Docker is not the same as specifying the environment dependencies. The schema requires 'enough detail to recreate the environment.'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository is provided and the paper describes the evaluation toolkit with a Server-Client architecture (Appendix A) and per-task evaluation setup (Appendices B-I), including that researchers need to 'set up a model server accessible via the HTTP protocol' and use Docker images."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 3 are reported as single point estimates with no confidence intervals or error bars. No uncertainty quantification is provided for any of the benchmark scores."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'gpt-4 presents the best performance on 6 out of 8 datasets', 'codellama-34b...still presents a clear performance gap to gpt-3.5-turbo') without any statistical significance tests. No p-values, t-tests, or other tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While raw numbers are reported (e.g., overall scores 4.01 for gpt-4 vs 0.51 average for OSS LLMs), no formal effect sizes (Cohen's d, etc.) are computed. The comparison lacks a baseline-relative effect size framework."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper explains the benchmark size was set 'to consider non-chat models' and for efficiency (roughly matching MMLU call counts), but does not provide a power analysis or formal justification for why 1,014 test examples is sufficient to reliably distinguish model performance."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states temperature=0 (greedy decoding) was used for all evaluations, meaning only single deterministic runs were performed. No variance, standard deviation, or multiple-run results are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark compares 29 LLMs including both API-based commercial models and open-source models. For the Digital Card Game task, two naive strategies (random and greedy) are used as baselines (Section E.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes the most recent models available at the time of publication, including gpt-4 (0613), claude-2, claude-3 (opus added later), llama-2 (70b), and codellama-34b. The baselines are contemporary."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "AgentBench is a benchmark, not a proposed system with removable components. The paper evaluates models on fixed tasks; there are no system components to ablate. Comparing model families (CodeLlama vs Llama-2) is observational model comparison, not ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used across tasks: Success Rate (OS, DB, HH), F1 (KG), Reward (DCG, WS), Game Progress (LTP), and Step SR (WB). The paper also reports execution outcome categories (Completed, Invalid Format, Invalid Action, TLE, CLE)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses entirely automated evaluation. Appendix F.2 acknowledges that for LTP, automatic evaluation sometimes differs from human evaluation, suggesting human evaluation would be relevant. No human evaluation of LLM agent outputs was conducted despite its relevance to claims about reasoning and decision-making quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates Dev and Test sets for all 8 tasks (Table 2, e.g., OS: 26 dev, 144 test). Main results are reported on the test set (Table 3, titled 'Test set (standard) results')."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-environment scores for all 29 LLMs across 8 environments. Additionally, Table 4 provides per-task breakdown of execution outcome categories, and Appendix J provides validity analysis per model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix J.2 provides detailed failure case analysis including qualitative examples of gpt-3.5-turbo getting stuck in a loop (J.2.2), analysis of repetition as primary cause of TLE (J.2.4), and analysis of invalid format errors with a gpt-4 example (J.2.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly reports that code tuning has 'ambivalent impacts' — improving some tasks (Web Shopping) while hurting others (Digital Card Game). Section 4.3 states 'training on code present ambivalent impacts on different agent tasks.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a significant performance disparity between top commercial LLMs and OSS models ≤70B are supported by Table 3 (average 2.32 vs. 0.51). The claim about 'poor long-term reasoning, decision-making, and instruction following' is supported by Table 4 and Appendix J analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'training on code...improves some agent tasks' and 'high-quality alignment training...could also help improve LLM agents' based on observational comparisons between model families (e.g., CodeLlama vs. Llama-2) that differ on multiple dimensions, not just the factor of interest. These are treated as causal findings without controlled experiments."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states AGENTBENCH will 'serve as a cornerstone for subsequent LLM agent research' and Section 4.3 generalizes about code tuning's impact on agents broadly. The paper does not sufficiently bound claims to the specific 8 tasks, the tested models, or the text-only, English-language evaluation setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not systematically discuss alternative explanations for the observed performance gaps. For example, the lower OSS performance could reflect RLHF differences, data distributions, or parameter count effects — the paper acknowledges some of these in passing but does not systematically consider alternatives to its interpretations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 lists versions for some models (gpt-4: 0613, gpt-3.5-turbo: 0613, claude-instant: v1.1, vicuna-13b: v1.5) but several key models lack version/snapshot information: claude-2 has '-', glm-4 has '-', chat-bison-001 has '-'. Per the schema, marketing names without snapshot dates do not count as specified versions. The evaluation is inconsistent in version specification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates with actual content are provided in Appendices B through I for all 8 environments. These are not just templates with placeholders — they include the actual instructions sent to models (e.g., the complete OS instruction prompt in Appendix B.3, the full KG prompt with API descriptions in Appendix D.2)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'To ensure reproducible results, we set temperature=0 (i.e., greedy decoding) in the inference on all tasks.' The paper also describes the context window management strategy (selecting minimum r such that token count ≤ 3500)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation scaffolding is described in detail: CoT prompting strategy (Section 2), the Server-Client framework with max-flow algorithm (Appendix A), per-round interaction management, context window handling with omission strategy, and task-specific interaction loops (Appendices B-I)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Each task appendix (B-I) describes dataset construction, including filtering criteria. For OS: gpt-4 generated problems 'strictly filtered by passing the unit tests'; 6000 Stack Overflow problems filtered by annotators. For DB: augmented data 'filtered and sampled into final dataset'. Data augmentation bias study in Appendix C.4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion mentions that 'even the strongest gpt-4 is not qualified as a practically usable agent' but this is a finding, not a limitations discussion. No substantive limitations section exists."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as benchmark contamination, the limitation of using greedy decoding only, the lack of multi-run variance, or the restriction to English-only text-only models."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what AGENTBENCH does NOT show. There is no discussion of scope boundaries such as: results apply only to text-only LLMs, only to English-language tasks, only to the specific 8 task types tested, or what agent capabilities fall outside the benchmark's coverage."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets are released at https://github.com/THUDM/AgentBench. The interactive environments, evaluation scripts, and checking pipelines are also released, enabling independent verification of results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Each appendix (B-I) describes in detail how the dataset was collected. For OS: human annotators from Stack Overflow + gpt-4 generated problems with unit tests. For KG: sourced from GrailQA, ComplexWebQuestions, GraphQuestions with criteria that questions require 5+ tool invocations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "The benchmark does not involve human participants. The data collection uses automated environments, existing public datasets, and LLM-based augmentation. The NA criterion for human recruitment does not apply here."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from data collection to final evaluation is documented. For DB: source datasets → gpt-3.5-turbo augmentation → validity filtering → final 300 entries by category. The checking pipelines (scripts) for automated evaluation are also described in detail."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgement section states funding from NSFC (62276148, 61825602), Ministry of Science and Technology of China (2022ZD0118600), Tsinghua University, and New Cornerstone Science Foundation. Zhipu AI is noted as covering 'all GPU and API cost consumed in this study'."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Tsinghua University (majority of authors), Ohio State University (Yu Gu, Yu Su, Huan Sun, Xiang Deng), and UC Berkeley (Sheng Shen, Tianjun Zhang). The connection to Zhipu AI (GLM-4 developer) is disclosed via funding acknowledgment."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Zhipu AI covered 'all GPU and API cost' and has a 'research fund' supporting this work, yet Zhipu AI's glm-4 model is evaluated in the benchmark (ranked 3rd with score 2.89). Zhipu AI has a direct financial interest in glm-4 appearing competitive, which is a non-independent funding relationship."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration of financial interests in the paper. The Zhipu AI funding relationship and the evaluation of Zhipu AI's glm-4 model in the benchmark is not explicitly acknowledged as a potential conflict of interest."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the 29 evaluated models. The models are described with release versions but no training data cutoff information is provided, making it impossible to assess whether benchmark data could have appeared in training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential train/test overlap. The DB task notes data augmentation was used partly 'to avoid leakage from the dataset' (Section C.1), suggesting awareness of the issue, but no systematic contamination analysis is performed for any task."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Two tasks (WebShop and Mind2Web/ALFWorld) are adapted from pre-existing publicly available benchmarks that were available before most models' training cutoffs. The paper does not address whether these benchmarks may have been in the training data of the evaluated models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. Pre-registration is not applicable."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. Inclusion/exclusion criteria for participants are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. Randomization of participant assignment is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. Attrition reporting is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report API costs or inference latency for running the benchmark. It notes that Zhipu AI covered 'all GPU and API cost' but does not quantify these costs. The paper acknowledges that evaluations are 'time-consuming' but provides no estimates."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper mentions GPU costs were covered by Zhipu AI but does not quantify GPU hours, API spend, or hardware used. Running 29 models across 8 environments on 1,014 test examples each represents substantial compute with no quantification."
       }
     }
   },
   "claims": [
     {
       "claim": "There is a significant performance gap between top commercial LLMs (average overall score 2.32) and open-source LLMs ≤70B (average overall score 0.51) on AgentBench.",
       "evidence": "Table 3 and Figure 1(b) show overall AgentBench scores. gpt-4 achieves 4.01, while the best OSS model (codellama-34b) achieves 0.96, and the OSS average is 0.51 vs. API average of 2.32.",
       "supported": "strong"
     },
     {
       "claim": "Poor long-term reasoning, decision-making, and instruction following are the main obstacles for developing usable LLM agents.",
       "evidence": "Table 4 shows Task Limit Exceeded (weak reasoning/decision-making) and Invalid Format/Invalid Action (weak instruction following) as dominant failure modes across 8 tasks. Appendix J.2 provides detailed qualitative analysis.",
       "supported": "moderate"
     },
     {
       "claim": "Code training has ambivalent impacts on LLM agent performance, improving some tasks while hurting others.",
       "evidence": "Section 4.3 compares CodeLlama vs. Llama-2 series across tasks: CodeLlama outperforms Llama-2 on Web Shopping but underperforms on Digital Card Game. Figure 10 and Table 3 show these differences.",
       "supported": "moderate"
     },
     {
       "claim": "High-quality alignment data training (e.g., on GPT-4 generated data) improves LLM agent performance.",
       "evidence": "Section 4.3 compares vicuna-13b (trained on ShareGPT/GPT-4 data) vs. llama-2-13b: vicuna-13b achieves 0.93 vs. 0.77 overall score, 'even performs comparably to 3 times larger codellama-34b'.",
       "supported": "weak"
     },
     {
       "claim": "GPT-4 is not yet qualified as a practically usable agent.",
       "evidence": "Section 2 and conclusion state this explicitly. Table 3 shows GPT-4 achieves 78% SR on House Holding but only 16.6% on Lateral Thinking Puzzles and overall score of 4.01 on a normalized scale where 'practically usable' would require much higher performance.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AgentBench is a multi-dimensional benchmark evaluating 29 LLMs across 8 interactive environments (OS, Database, Knowledge Graph, Digital Card Game, Lateral Thinking Puzzles, House Holding, Web Shopping, Web Browsing). Top commercial LLMs like GPT-4 significantly outperform open-source models ≤70B, with average overall scores of 2.32 vs. 0.51 respectively. The primary failure modes are Task Limit Exceeded (weak reasoning) and Invalid Format/Action (weak instruction following). Code training shows mixed effects: improving procedural tasks like web shopping while degrading performance on tasks requiring general reasoning like digital card games.",
   "red_flags": [
     {
       "flag": "Conflict of interest: funder evaluates own model",
       "detail": "Zhipu AI covered 'all GPU and API cost consumed in this study' and provided a research fund, yet Zhipu AI's glm-4 model is evaluated in the benchmark and ranked 3rd overall (score 2.89, above claude-2). This is a direct conflict of interest that is not explicitly acknowledged as such."
     },
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All benchmark scores are reported as single point estimates with greedy (temperature=0) decoding and no error bars, confidence intervals, or multi-run variance. The paper cannot distinguish whether small performance differences between models are meaningful or within the noise floor."
     },
     {
       "flag": "Contamination not addressed for adapted benchmarks",
       "detail": "Two of eight tasks use adapted versions of existing publicly available benchmarks (WebShop from 2022, Mind2Web/ALFWorld). These were available before most evaluated models' training cutoffs, but the paper does not discuss potential contamination."
     },
     {
       "flag": "Causal claims from observational model comparisons",
       "detail": "Claims that code training 'improves' some tasks and 'harms' others are made by comparing model families (CodeLlama vs. Llama-2) that differ on multiple dimensions beyond just code training. No controlled ablation is performed, making causal attribution unreliable."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Key limitations such as single-run evaluation, English-only tasks, text-only models, restricted action spaces, and potential benchmark contamination are not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu",
         "Nan Du",
         "Izhak Shafran",
         "Karthik R Narasimhan",
         "Yuan Cao"
       ],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Pioneer work combining chain-of-thought reasoning with actions in agent tasks; the CoT+action format adopted in AgentBench is based on ReAct."
     },
     {
       "title": "WebShop: Towards Scalable Real-World Web Interaction with Grounded Language Agents",
       "authors": [
         "Shunyu Yao",
         "Howard Chen",
         "John Yang",
         "Karthik Narasimhan"
       ],
       "year": 2022,
       "relevance": "Simulated online shopping environment adapted as one of AgentBench's 8 evaluation tasks; foundational work on web agent evaluation."
     },
     {
       "title": "Mind2Web: Towards a Generalist Agent for the Web",
       "authors": [
         "Xiang Deng",
         "Yu Gu",
         "Boyuan Zheng",
         "Shijie Chen",
         "Samuel Stevens",
         "Boshi Wang",
         "Huan Sun",
         "Yu Su"
       ],
       "year": 2023,
       "arxiv_id": "2306.06070",
       "relevance": "General web benchmark adapted as one of AgentBench's 8 tasks; evaluates LLMs as web browsing agents."
     },
     {
       "title": "ALFWorld: Aligning Text and Embodied Environments for Interactive Learning",
       "authors": [
         "Mohit Shridhar",
         "Xingdi Yuan",
         "Marc-Alexandre Cote",
         "Yonatan Bisk",
         "Adam Trischler",
         "Matthew Hausknecht"
       ],
       "year": 2020,
       "relevance": "Text-based household environment used as the House-Holding task in AgentBench; foundational agent evaluation benchmark."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": [
         "Jason Wei",
         "Xuezhi Wang",
         "Dale Schuurmans",
         "Maarten Bosma",
         "Fei Xia",
         "Ed Chi",
         "Quoc V Le",
         "Denny Zhou"
       ],
       "year": 2022,
       "relevance": "Chain-of-thought prompting is the core evaluation strategy adopted throughout AgentBench for all 8 tasks."
     },
     {
       "title": "Reflexion: an autonomous agent with dynamic memory and self-reflection",
       "authors": [
         "Noah Shinn",
         "Beck Labash",
         "Ashwin Gopinath"
       ],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Advanced reasoning strategy for LLM agents; discussed as a relevant extension beyond AgentBench's primitive CoT evaluation."
     },
     {
       "title": "InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback",
       "authors": [
         "John Yang",
         "Akshara Prabhakar",
         "Karthik Narasimhan",
         "Shunyu Yao"
       ],
       "year": 2023,
       "arxiv_id": "2306.14898",
       "relevance": "Concurrent work providing an interactive coding benchmark for bash and SQL environments, overlapping with AgentBench's OS and DB tasks."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": [
         "Joon Sung Park",
         "Joseph C. O'Brien",
         "Carrie J. Cai",
         "Meredith Ringel Morris",
         "Percy Liang",
         "Michael S. Bernstein"
       ],
       "year": 2023,
       "arxiv_id": "2304.03442",
       "relevance": "LLM agent work in social simulation contexts; relevant to the survey scope on LLM agent capabilities and evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": [
         "Mark Chen",
         "Jerry Tworek",
         "Heewoo Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark for code generation; foundational code evaluation benchmark that influenced the code-grounded tasks in AgentBench."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": [
         "Sirui Hong",
         "Xiawu Zheng",
         "Jonathan P. Chen"
       ],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent LLM framework; relevant to the LLM-as-Agent research landscape that AgentBench aims to evaluate."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": [
         "Qingyun Wu",
         "Gagan Bansal",
         "Jieyu Zhang"
       ],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent conversation framework for LLM applications; part of the LLM agent ecosystem being evaluated."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": [
         "Percy Liang",
         "Rishi Bommasani",
         "Tony Lee"
       ],
       "year": 2022,
       "arxiv_id": "2211.09110",
       "relevance": "HELM benchmark for comprehensive LLM evaluation; cited as prior work on broad-spectrum LLM benchmarking that AgentBench extends to agent settings."
     }
   ]
 }