ai-research-survey

scan.json (27213B)

---

 {
   "paper": {
     "title": "Beyond Task Completion: An Assessment Framework for Evaluating Agentic AI Systems",
     "authors": [
       "Sreemaee Akshathala",
       "Bassam Adnan",
       "Mahisha Ramesh",
       "Karthik Vaidhyanathan",
       "Basil Muhammed",
       "Kannan Parthasarathy"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2512.12791"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL (https://github.com/sa4s-serc/asf) in the introduction and again in Section 4.2 ('Our experiments and results are available on GitHub'), indicating code and experiment trajectories have been released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The synthetic CloudOps data (instances, logs, policies) was generated using Sonnet 4.5 but no dataset download link or release is mentioned. The GitHub repository contains experiment trajectories but the underlying synthetic dataset is not described as separately released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions GPT-4o with temperature 0.7, ChromaDB for embeddings, and text-embedding-3-small, and the Mem0 library with 'default settings', but no requirements.txt, Dockerfile, or conda environment file with library versions is provided in the paper. The setup description is partial."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub link is provided, the paper itself does not include step-by-step reproduction instructions. The GitHub repository may contain them but the paper does not describe the commands or procedure needed to reproduce the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 3-7 are reported as point estimates only (e.g., '33.7%' precision, '37.9%' recall). While Figure 3 shows distribution plots across three runs (which implies spread), no confidence intervals or explicit error bars are reported in the tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "RQ1 frames a comparison between baseline and framework evaluation approaches, but no statistical significance tests are used to support the claim that the framework is more effective. The comparison relies on showing the framework reveals additional metrics, without formal hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No comparative effect sizes are computed. The paper reports descriptive metrics (completion rates, precision/recall) but does not compare two competing methods with effect size statistics."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only three scenarios (S1, S2, S3), each run three times, are used to validate the framework. No justification for why three scenarios or three runs are sufficient is provided; no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each scenario was executed three times and results were 'averaged to ensure reliability', but Tables 3-6 report only the averages. Table 7 reports 'average failures per pillar' but no standard deviation or range is given. Figure 3 shows token/cost distributions visually but without numerical variance statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 explicitly compares a 'Baseline (B)' evaluation (capturing only task completion and tool usage ratio) against the proposed 'Framework (F)' metrics across all three scenarios, directly addressing RQ1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is a standard task-completion + tool-usage-ratio evaluation, which the paper correctly identifies as the current dominant practice. The comparison is fair given the paper's goal of showing what standard metrics miss."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 is labeled 'Pillar ablation' but shows a per-category breakdown of failure counts across pillars, not a proper ablation study. No pillar is removed from the framework to measure its contribution — the table simply counts how many failures each category detects. A true ablation would remove a pillar and show the framework's effectiveness degrades."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The framework uses many metrics including task completion, tool usage ratio, tool sequence correctness, expected tool calls, policy adherence, dependency inquiry, memory P/R/F1, and judge scores (completion, safety, memory, reasoning, overall)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated (LLM-as-Judge and Agent-as-Judge). No human experts validate whether the framework's detected failures are genuine. For a framework claiming to identify behavioral failures, human validation of those findings would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The three CloudOps scenarios were designed based on production experiences at MontyCloud and used directly for evaluation. There is no separate held-out test set; the same scenarios are used for framework validation without a train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per pillar (LLM, Memory, Tools, Environment) in Table 7, and per scenario (S1, S2, S3) throughout. Table 4 further decomposes memory retrieval into Single-hop, Multi-hop, and Temporal types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Table 7 describe concrete failure examples (e.g., 'Skipped policy validation before instance termination', 'Did not recall previous role mappings or configuration changes'). Section 5 also discusses specific failure patterns observed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports substantial failures across all pillars: S1 had only 33% policy adherence, S2 had 13.1% memory recall, S3 had 7.67 average tool failures. The paper explicitly presents these negative results to demonstrate gaps in baseline evaluation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the framework 'reveal[s] behavioral deviations overlooked by conventional metrics', which is supported by Tables 3-7 showing framework metrics surface failures (e.g., 33% policy adherence) hidden by 100% task completion in S2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims (e.g., 'tool sequencing and memory management need particular attention in multi-agent scenarios') derived from observational differences across three scenarios with different complexity. No controlled single-variable manipulation isolates the cause of observed failures."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames the framework as general ('an end-to-end Agent Assessment Framework') but validates it only on CloudOps scenarios with a single LLM (GPT-4o). Section 5.1 acknowledges external validity threats but the title and framing imply broad applicability beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 Threats to Validity notes that 'Ground truth contracts defined by us may not reflect all valid solution paths' and 'Judge-based protocols introduce potential LLM evaluation biases', but these are mostly generic limitations. No alternative explanations for WHY specific failures occurred are systematically considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' without specifying a version or snapshot date. It also mentions 'Sonnet 4.5' for synthetic data generation and 'text-embedding-3-small' for embeddings. 'GPT-4o' without a version or API snapshot date does not count as a specified version per schema criteria."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'structured prompts' for LLM-as-Judge (Table 5) and describes the evaluation protocol, but no actual prompt text is provided in the paper or appendix. Section 3.3 describes that the LLM-as-Judge 'receives execution logs and produces scored assessments' but the actual prompt is not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature (0.7) is reported for GPT-4o, and 'default settings' of Mem0 are mentioned. However, top-p, max tokens, and other LLM API settings are not specified. Mem0 default settings are not enumerated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the agentic scaffolding in detail: MOYA framework, memory using Mem0 with ChromaDB, tool invocation protocols, test case generation, static/dynamic/judge evaluation layers, Agent Card mechanism, and Agent-as-Judge auditor protocols. Figures 1 and 2 provide architectural diagrams."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Synthetic data was 'generated using Sonnet 4.5' based on production issues encountered at MontyCloud, but the actual generation process, prompts used, volume of data, filtering criteria, and preprocessing steps are not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 is titled 'Threats to Validity' and addresses external, internal, and construct validity with substantive discussion beyond a single sentence."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 identifies specific threats: evaluation limited to CloudOps domain (external), uniform temperature 0.7 and single model selection (internal), metrics may not capture all reliability aspects and memory assessment excludes storage efficiency (construct). These are study-specific, not boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 5.1 notes the framework is evaluated only on CloudOps, it does not explicitly state what the results do NOT show. Future work section gestures at extending beyond CloudOps but does not enumerate specific claims that the paper is NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "GitHub link is provided for 'experiments and results' (trajectories) but the underlying synthetic CloudOps data (instances, logs, policies) is not described as publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Synthetic data was 'generated using Sonnet 4.5 due to its ranking on LMArena' based on 'production issues we encountered', but the prompts used, generation criteria, volume, and verification procedures for the synthetic data are not described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "There are no human participants in this study. The data is synthetic CloudOps scenarios, not collected from human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from synthetic data generation through to evaluation metrics is not fully documented. The paper describes the evaluation framework components but does not trace the full data pipeline with intermediate steps and filtering."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper. The industry collaboration with MontyCloud is described in the body, but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the title page: Sreemaee Akshathala, Bassam Adnan, Mahisha Ramesh, Karthik Vaidhyanathan at IIIT-Hyderabad SERC; Basil Muhammed and Kannan Parthasarathy at MontyCloud Inc. The MontyCloud affiliation is prominently disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "MontyCloud Inc. employees are co-authors of the paper, and the framework is validated on MontyCloud's production system (MOYA). MontyCloud has a direct financial interest in the framework's apparent effectiveness, making the funder non-independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests is present in the paper. The MontyCloud affiliation creates an undisclosed potential conflict: two authors are MontyCloud employees evaluating their own system."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is used as the agent backbone but no training data cutoff date is stated for GPT-4o. The paper does not address whether the model's training data could include the evaluation scenarios or benchmark components."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether GPT-4o's training data overlaps with the CloudOps scenarios, policies, or tool descriptions used in evaluation. Since scenarios are based on real production patterns, some training data overlap is plausible."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation scenarios are custom-designed CloudOps workflows (not public benchmarks), so benchmark contamination in the traditional sense (e.g., HumanEval appearing in training data) does not apply here."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "There are no human participants in this study. The evaluation is entirely automated using LLM agents and synthetic CloudOps scenarios."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; IRB or ethics approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or controlled experiment with human assignment; not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or human evaluators; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 (RQ3) explicitly reports inference costs: average $0.0621 per scenario execution run, LLM-as-Judge adds $0.0593 total, Agent-as-Judge $0.9572 total. Figure 4 shows cost breakdowns per evaluation protocol."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 reports average execution time (183.5s per run), token consumption (19,644 input tokens, 1,301 output tokens per average run), and scenario-level cost ($0.0405–$0.0818). The compute budget is quantified in practical terms."
       }
     }
   },
   "claims": [
     {
       "claim": "Conventional task-completion and tool-usage-ratio metrics miss critical behavioral failures in agentic systems.",
       "evidence": "Table 3 shows S2 achieves 100% task completion and 0.97 tool usage ratio under baseline, but framework metrics reveal only 33% tool sequence correctness, 0% dependency inquiry, and 13.1% memory recall. Section 5 elaborates these failures.",
       "supported": "strong"
     },
     {
       "claim": "The proposed four-pillar framework (LLM, Memory, Tools, Environment) effectively surfaces behavioral failures across all pillars.",
       "evidence": "Table 7 shows per-pillar failure counts across all scenarios; tool failures peak at 7.67 in S3, memory failures at 3.67, LLM at 1.67. These failures are invisible in baseline evaluation.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-as-Judge is cost-efficient for continuous monitoring ($0.0593 total overhead, 14.7s) while Agent-as-Judge provides deeper assessment at higher cost ($0.9572, 913.4s).",
       "evidence": "Section 4.2.3 and Figure 4 report these exact cost and time figures across three scenarios.",
       "supported": "strong"
     },
     {
       "claim": "Tool orchestration failures are the dominant failure mode in complex multi-agent scenarios.",
       "evidence": "Table 7 shows Tools pillar has highest failure count in S3 (7.67 average), attributed to 'missed diagnostic or verification steps before applying remediation.'",
       "supported": "moderate"
     },
     {
       "claim": "Memory retrieval failures scale with scenario complexity, with multi-hop and temporal retrieval showing high precision but low recall.",
       "evidence": "Table 4 shows multi-hop precision 100% but recall 26.5%, temporal reasoning precision 100% but recall 29.8% for S3.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "case-study",
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes a four-pillar assessment framework (LLM, Memory, Tools, Environment) for evaluating agentic AI systems beyond task completion. Validated on three CloudOps scenarios using the MOYA multi-agent framework with GPT-4o, it finds that standard task-completion metrics miss significant behavioral failures: a scenario with 100% task completion showed only 13.1% memory recall and 33% tool sequence correctness. The framework identifies tool orchestration and memory management as the highest-failure dimensions in complex multi-agent scenarios, and quantifies the cost-accuracy tradeoff between LLM-as-Judge ($0.0593, 14.7s) and Agent-as-Judge ($0.9572, 913.4s) evaluation protocols.",
   "red_flags": [
     {
       "flag": "Industry co-authorship without conflict-of-interest declaration",
       "detail": "Two of six authors are MontyCloud Inc. employees, and the framework is validated exclusively on MontyCloud's own production system (MOYA). The paper does not include a competing interests statement, and MontyCloud has a financial interest in the framework appearing effective. The results could be selectively framed to favor MontyCloud's system."
     },
     {
       "flag": "Very small evaluation: 3 scenarios, 3 runs each",
       "detail": "The framework is validated on only three scenarios, run three times each (nine total executions). This is an extremely small sample from which to draw conclusions about framework effectiveness. The paper provides no justification for why three scenarios are sufficient."
     },
     {
       "flag": "Overgeneralized framing vs. narrow validation",
       "detail": "The paper presents the framework as general for 'evaluating agentic AI systems' and describes it as 'end-to-end', but validates it only on CloudOps with a single LLM (GPT-4o). Results from one domain and one model cannot support claims about general agentic AI evaluation."
     },
     {
       "flag": "LLM-as-Judge reliability not validated",
       "detail": "The LLM-as-Judge protocol is used to assess agent performance, but the reliability of this evaluation method (inter-rater agreement, calibration against human judgment) is not validated. The paper notes 'Judge-based protocols introduce potential LLM evaluation biases' in the limitations but does not quantify or address this."
     },
     {
       "flag": "No variance statistics despite non-deterministic model",
       "detail": "The paper explicitly motivates the need to handle non-deterministic LLM behavior, yet tables report only averages across three runs with no standard deviations. Wide variability is visible in Figure 3 (e.g., S1 input tokens range 7,784-19,335) but is not numerically reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Are: Scaling up agent environments and evaluations",
       "authors": [
         "Andrews, P.",
         "Benhalloum, A."
       ],
       "year": 2025,
       "relevance": "Directly relevant as a benchmark/evaluation environment for agentic AI systems, cited for structured workflow evaluation methodology."
     },
     {
       "title": "AgentHarm: A benchmark for measuring harmfulness of LLM agents",
       "authors": [
         "Andriushchenko, M.",
         "Souly, A."
       ],
       "year": 2024,
       "arxiv_id": "2410.09024",
       "relevance": "Benchmark for evaluating safety and harmful behavior in LLM agents, relevant to the survey's interest in agentic AI evaluation methodologies."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": [
         "Cemri, M.",
         "Pan, M. Z.",
         "Yang, S."
       ],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes in multi-agent LLM systems, directly relevant to understanding evaluation challenges in agentic AI."
     },
     {
       "title": "AIopslab: A holistic framework to evaluate AI agents for enabling autonomous clouds",
       "authors": [
         "Chen, Y."
       ],
       "year": 2025,
       "relevance": "Closely related evaluation framework for AI agents in cloud operations, providing a comparison point for this paper's CloudOps focus."
     },
     {
       "title": "A survey on LLM-as-a-judge",
       "authors": [
         "Gu, J.",
         "Jiang, X."
       ],
       "year": 2025,
       "relevance": "Survey of LLM-as-a-judge evaluation methodology, foundational for the judge-based evaluation protocols used in this paper."
     },
     {
       "title": "An empirical study of testing practices in open source AI agent frameworks and agentic applications",
       "authors": [
         "Hasan, M. M.",
         "Li, H."
       ],
       "year": 2025,
       "relevance": "Empirical study of testing and evaluation practices in agentic AI frameworks, directly in scope for the survey."
     },
     {
       "title": "Agentic software engineering: Foundational pillars and a research roadmap",
       "authors": [
         "Hassan, A. E."
       ],
       "year": 2025,
       "relevance": "Research roadmap for agentic software engineering, providing context for the evaluation challenges addressed by this paper."
     },
     {
       "title": "Engineering LLM powered multi-agent framework for autonomous CloudOps",
       "authors": [
         "Parthasarathy, K.",
         "Vaidhyanathan, K."
       ],
       "year": 2025,
       "relevance": "Prior work by the same group introducing the MOYA framework used as the evaluation target in this paper."
     },
     {
       "title": "AgentIF: Benchmarking instruction following of large language models in agentic scenarios",
       "authors": [
         "Qi, Y.",
         "Peng, H."
       ],
       "year": 2025,
       "arxiv_id": "2505.16944",
       "relevance": "Benchmark for instruction-following in agentic scenarios, relevant to the LLM pillar evaluation in this framework."
     },
     {
       "title": "A survey on the memory mechanism of large language model based agents",
       "authors": [
         "Zhang, Z.",
         "Bo, X."
       ],
       "year": 2024,
       "relevance": "Survey of memory mechanisms in LLM agents, relevant to the Memory pillar evaluation methodology in this paper."
     },
     {
       "title": "Which agent causes task failures and when? On automated failure attribution of LLM multi-agent systems",
       "authors": [
         "Zhang, S."
       ],
       "year": 2025,
       "relevance": "Automated failure attribution in multi-agent LLM systems, directly relevant to the failure analysis methodology in this paper."
     },
     {
       "title": "Evaluating very long-term conversational memory of LLM agents",
       "authors": [
         "Maharana, A.",
         "Lee, D.-H.",
         "Tulyakov, S.",
         "Bansal, M.",
         "Barbieri, F.",
         "Fang, Y."
       ],
       "year": 2024,
       "arxiv_id": "2402.17753",
       "relevance": "Evaluates memory in LLM agents, directly relevant to the Memory pillar of the proposed assessment framework."
     },
     {
       "title": "Evaluating memory in LLM agents via incremental multi-turn interactions",
       "authors": [
         "Hu, Y.",
         "Wang, Y.",
         "McAuley, J."
       ],
       "year": 2025,
       "arxiv_id": "2507.05257",
       "relevance": "Memory evaluation methodology for LLM agents, relevant to the survey's coverage of agentic AI evaluation."
     }
   ]
 }