ai-research-survey

scan.json (24643B)

---

 {
   "paper": {
     "title": "Automated test generation to evaluate tool-augmented LLMs as conversational AI agents",
     "authors": [
       "Samuel Arcadinho",
       "David Aparício",
       "Mariana S. C. Almeida"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.15934"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link to the ALMITA dataset and all generated datasets: https://github.com/zendesk/almita-dataset (footnote 1, Section 1). This is the dataset; code for the pipeline itself is not explicitly mentioned as released, but the dataset artifacts are available."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "ALMITA and auto-ALMITA datasets are released at https://github.com/zendesk/almita-dataset, as stated in footnote 1: 'ALMITA, along with all other datasets generated using our pipeline and referenced in the paper, are available'."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency listings are provided in the paper. The paper does not describe the software environment used to run the pipeline."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While all prompts are provided in the appendix, there are no step-by-step reproduction instructions, no README with commands to run, and no scripts to replicate the pipeline end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and Supplementary Table 1 report only point estimates (e.g., '92.7', '75.2') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims across models (e.g., 'GPT models outperform the others') but provides no statistical significance tests — comparisons are based solely on raw percentage differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (Cohen's d, odds ratios, etc.) are reported. The paper reports raw percentages but does not contextualize the magnitude of differences between models."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset sizes (1,420 tests for ALMITA, 2,696 for auto-ALMITA) are reported but not justified. No power analysis or justification for why these sample sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any results. All numbers appear to be single-run evaluations with no indication of variability across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares multiple LLMs (GPT-4o, GPT-4, Claude3-sonnet, Mistral-NeMo-Instruct, Llama3.1-8b-Instruct) against each other in Table 2, providing baseline comparisons across models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include GPT-4o (2024), Claude 3 Sonnet (2024), Mistral-NeMo-Instruct, and Llama3.1 — all contemporary models at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 presents an ablation study comparing conversations generated directly from procedures vs. using intermediate graph representations, showing graph representations improve validity from ~68% to ~88%."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 7 evaluation dimensions: reply recall, correct reply, API recall, correct API, correct API parameters, test correctness, and conversation correctness (Section 4.3, Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two annotators independently reviewed each datapoint in ALMITA for correctness (Section 4.1), and the ablation study in Section 4.2 uses manual curation by the same annotators for quality assessment."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a learning/training study — the paper evaluates LLMs on a fixed dataset without any tuning phase that would require train/test separation. The entire ALMITA dataset serves as the evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dimension breakdowns (reply recall, correct reply, API recall, correct API, correct parameters, test correct, conversation correct) rather than just a single aggregate metric."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses failure patterns: GPT-4 tends to call APIs even when unnecessary (low reply recall), all models struggle with full conversation correctness, and the Limitations section (Section 6) discusses annotation limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that all models have very low conversation correctness (4.2%-15.6%), which is a significant negative finding. GPT-4's low reply recall (53.2%) is also reported as a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that LLMs 'perform well in single interactions' but 'struggle to handle complete conversations' — this is supported by Table 2 showing high test correctness (73-89%) but very low conversation correctness (1.6-15.6%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that intermediate graph representations improve conversation quality (Section 4.2). The ablation study provides controlled single-variable manipulation: same procedures, same annotators, only difference is pipeline with/without graphs. This is adequate for the claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the method 'is general and capable of AI agents for different domains' (abstract) but all experiments are in the customer support domain only. The title and abstract claim generality beyond what was tested. Section 1 states 'the same method could be applied, with some changes, to other domains' which slightly hedges, but the abstract does not."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For example, low conversation correctness could be due to error propagation, strict metric definitions, or prompt design — none of these alternatives are explored. The Limitations section mentions that metrics 'may be too strict' but does not substantively discuss this as an alternative explanation for the observed performance gaps."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 footnote specifies exact versions: 'gpt-4-0613, gpt-4o-2024-05-13, anthropic.claude-3-sonnet-20240229-v1:0'. Open-source models are specified by name and size (Llama3.1-8b-Instruct, Mistral-NeMo-Instruct)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "All prompts used in the pipeline are provided in full in Appendix A (A.1 through A.8), including system prompts and user prompts for each pipeline stage and the AI agent evaluation prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling hyperparameters are reported for any of the LLM calls in the pipeline or the evaluation. The noise generator mentions a 20% probability but API calling hyperparameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The 8-stage pipeline is described in detail in Section 3 with each component (intent generator, procedure generator, API extractor, flowgraph generator, conversation graph generator, noise generator, path sampler, conversation generator, test extractor) thoroughly explained with diagrams (Figures 1-4) and algorithms (Algorithm 1)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the full data pipeline with counts at each stage: 84 intents → 168 procedures → 132 after filtering → 70 with valid APIs → 49 valid flowgraphs → 33 valid conversation graphs → 217 conversations → 192 after manual filtering → 1420 tests. Filtering criteria at each stage are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitations' section with substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: using only GPT-4 as the generator may bias evaluation, small number of annotators, single prompt used across models, lack of quantitative diversity evaluation, and metrics that may be too strict. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions focus on customer support, it does not explicitly state what the results do NOT show. It does not bound its claims to specific model versions, specific types of customer support, or acknowledge that results may not transfer to other conversational domains despite claiming generality."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The ALMITA and auto-ALMITA datasets are available at https://github.com/zendesk/almita-dataset, allowing independent verification of the generated tests and evaluation."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection procedure in detail: LLM-generated intents as seeds, multi-stage pipeline with filtering at each step, two-annotator review process. Each stage's criteria and filtering rules are described in Section 3."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited as subjects. The two annotators are co-authors/team members performing curation, not research subjects. The data source is synthetically generated, not a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides complete pipeline statistics showing counts at each stage (generated, after automatic filters, after manual filters). Section 4.1 documents each transformation with specific numbers of items removed and reasons."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, acknowledgments section, or grant information is provided in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed with @zendesk.com email addresses, clearly identifying their affiliation with Zendesk, a customer support platform company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Zendesk employees. Zendesk is a customer support platform that would benefit from demonstrating both the need for AI agent evaluation (justifying their tools) and the effectiveness of automated test generation. The funder (Zendesk, implicitly) has a stake in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is provided. The authors work at Zendesk, which is directly in the customer support AI space, but this potential conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs on the ALMITA benchmark but does not state training data cutoff dates for any of the models. While the benchmark is newly created, the paper does not discuss whether any of the test content could overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The benchmark is synthetically generated, which reduces contamination risk, but this advantage is not explicitly discussed. GPT-4 was used to generate the test data and is also evaluated — this circular dependency is not addressed as a contamination concern."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "ALMITA is a newly created benchmark not available before the models' training cutoffs, so pre-existing benchmark contamination does not apply. However, the fact that GPT-4 generated the data could be considered a form of contamination — this is partially addressed in Section 4.3 ('this may be biased by the use of GPT-4 for test generation') and Limitations."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The two annotators performed dataset curation, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. Annotation was performed by team members as part of dataset construction."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or tokens consumed are reported. The pipeline calls GPT-4 multiple times across 8 stages for each test but provides no cost or latency figures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications are reported. The pipeline involves extensive LLM calls for generation and evaluation but the cost is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Intermediate graph representations improve the validity of generated conversations from ~68% to ~88%.",
       "evidence": "Section 4.2 ablation study: simplified pipeline without graphs yields 34/50 (68%) valid conversations; original pipeline with graphs yields 192/217 (88%) valid conversations, both manually curated by the same annotators.",
       "supported": "moderate"
     },
     {
       "claim": "Tool-augmented LLMs perform well in single interactions (API calling) but struggle with complete conversations.",
       "evidence": "Table 2: API correct exceeds 90% for most models, but conversation correctness ranges from 1.6% (Llama3.1-8b-I) to 15.6% (GPT-4o w/F).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o achieves the highest test correctness (88.9%) among evaluated models on ALMITA.",
       "evidence": "Table 2 shows GPT-4o at 88.9% test correct, highest among the 5 models tested (next highest: Mistral-NeMo-I at 84.7%).",
       "supported": "moderate"
     },
     {
       "claim": "The fully automated pipeline (auto-ALMITA) produces rankings highly correlated with the manually curated dataset (ALMITA), with correlation of 0.98.",
       "evidence": "Section 4.4 and Figure 5: both datasets rank LLMs in the same order with correlation value 0.98 on the test correct metric.",
       "supported": "moderate"
     },
     {
       "claim": "The proposed method is general and capable of evaluating AI agents for different domains beyond customer support.",
       "evidence": "No experiments outside customer support are conducted. This claim (from abstract and Section 1) is purely assertive with no supporting evidence.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper presents an automated pipeline for generating evaluation datasets for tool-augmented conversational AI agents, using intermediate graph structures to improve test quality (88% validity vs 68% without graphs). Evaluation of 5 LLMs on the ALMITA benchmark reveals that while models achieve high single-interaction accuracy (>85% for API calls), full conversation correctness is very low (1.6-15.6%), suggesting current LLMs are inadequate as fully autonomous customer support agents. The automated pipeline produces model rankings highly correlated (0.98) with manually curated datasets, suggesting it can replace expensive human curation.",
   "red_flags": [
     {
       "flag": "Generator-evaluator circularity",
       "detail": "GPT-4 is used to generate the ALMITA test dataset and is also one of the models being evaluated. The paper acknowledges this may bias correct reply scores toward GPT models (Section 4.3) but does not quantify or mitigate this bias."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no confidence intervals, error bars, or variance across runs. LLM outputs are stochastic, so single-run results may not be stable."
     },
     {
       "flag": "Company evaluating its own domain",
       "detail": "All authors are Zendesk employees (a customer support platform). The paper proposes automated evaluation methods for customer support AI agents — a direct product interest. No conflict of interest disclosure is provided."
     },
     {
       "flag": "Small ablation sample",
       "detail": "The ablation study comparing graph vs. no-graph pipelines uses only 50 conversations for the no-graph condition, which may be insufficient to draw robust conclusions about the difference (68% vs 88%)."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any LLM calls, despite the pipeline involving 8+ LLM-dependent stages where these settings could substantially affect output quality."
     }
   ],
   "cited_papers": [
     {
       "title": "Autogen: Enabling next-gen LLM applications via multi-agent conversation framework",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Foundational multi-agent LLM framework relevant to survey's coverage of agentic AI architectures."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Key benchmark for evaluating LLMs as agents with multi-step interactions, directly relevant to the survey's evaluation methodology scope."
     },
     {
       "title": "AgentTuning: Enabling generalized agent abilities for LLMs",
       "authors": ["Aohan Zeng", "Mingdao Liu", "Rui Lu"],
       "year": 2023,
       "arxiv_id": "2310.12823",
       "relevance": "Proposes tuning LLMs for agent capabilities using compiled agent datasets, relevant to survey's coverage of LLM agent capability enhancement."
     },
     {
       "title": "ToolLLM: Facilitating large language models to master 16000+ real-world APIs",
       "authors": ["Yujia Qin", "Shihao Liang", "Yining Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Major work on tool-augmented LLMs and API calling capability, directly relevant to survey's scope on LLM tool use."
     },
     {
       "title": "Gorilla: Large language model connected with massive APIs",
       "authors": ["Shishir G Patil", "Tianjun Zhang", "Xin Wang"],
       "year": 2023,
       "arxiv_id": "2305.15334",
       "relevance": "Early influential work on LLMs connected to APIs, relevant to survey's coverage of tool-augmented models."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang"],
       "year": 2024,
       "relevance": "Addresses rigorous LLM evaluation methodology for code generation, relevant to survey's methodological quality assessment focus."
     },
     {
       "title": "Exploring and evaluating hallucinations in LLM-powered code generation",
       "authors": ["Fang Liu", "Yang Liu", "Lin Shi"],
       "year": 2024,
       "arxiv_id": "2404.00971",
       "relevance": "Studies LLM hallucination in code generation, relevant to survey's coverage of LLM reliability and evaluation."
     },
     {
       "title": "APIGen: Automated pipeline for generating verifiable and diverse function-calling datasets",
       "authors": ["Zuxin Liu", "Thai Hoang", "Jianguo Zhang"],
       "year": 2024,
       "arxiv_id": "2406.18518",
       "relevance": "Automated dataset generation for function-calling evaluation, directly comparable methodology to the paper under review."
     },
     {
       "title": "AgentInstruct: Toward generative teaching with agentic flows",
       "authors": ["Arindam Mitra", "Luciano Del Corro", "Guoqing Zheng"],
       "year": 2024,
       "relevance": "Framework for generating synthetic agent training data, relevant to survey's coverage of agentic AI data generation and evaluation."
     },
     {
       "title": "GAIA: A benchmark for general AI assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Craig Swift"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "Benchmark for evaluating general AI assistant capabilities with human-annotated questions, relevant to survey's evaluation methodology coverage."
     },
     {
       "title": "API-Blend: A comprehensive corpora for training and benchmarking API LLMs",
       "authors": ["Kinjal Basu", "Ibrahim Abdelaziz"],
       "year": 2024,
       "arxiv_id": "2402.15491",
       "relevance": "Combines multiple datasets for API-calling LLM evaluation, relevant to survey's coverage of tool-use benchmarks."
     }
   ]
 }