ai-research-survey

scan.json (25469B)

---

 {
   "paper": {
     "title": "AgentFM: Role-Aware Failure Management for Distributed Databases with LLM-Driven Multi-Agents",
     "authors": [
       "Lingzhe Zhang",
       "Yunpeng Zhai",
       "Tong Jia",
       "Xiaosong Huang",
       "Chiming Duan",
       "Ying Li"
     ],
     "year": 2025,
     "venue": "FSE '25 (33rd ACM International Conference on the Foundations of Software Engineering)",
     "arxiv_id": "2504.06614",
     "doi": "10.1145/3696630.3728492"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, repository URL, or archive is provided anywhere in the paper. There is no mention of code release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The injected anomaly dataset used for evaluation is not released. The paper uses Apache IoTDB as the test platform but the specific experimental data (injected anomalies, collected traces, metrics, logs) is not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements file, Dockerfile, or dependency listing is provided. The paper mentions using Qwen2.5-72b and Apache IoTDB but gives no environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, script, or step-by-step instructions for reproducing the experiments are provided. The evaluation section (Section 4) describes the experimental design but not how to reproduce it."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, and 3 are reported as point estimates only (e.g., 'Anomaly Detection: F1-Score 95.76%'). No confidence intervals or error bars are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are conducted. The preliminary study (Table 1) and main evaluation (Table 3) compare numbers directly without any statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. There are no baseline comparisons for the main evaluation of AgentFM, so effect size reporting is not possible in any case."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 10 anomaly types with 20 injections each (200 total), but no justification is given for why this sample size was chosen or whether it is sufficient for the precision/recall claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are single-run point estimates. No variance, standard deviation, or results across multiple runs are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The main evaluation of AgentFM (Table 3) includes no baselines. PLELog is used in the preliminary empirical study (Table 1) as a demonstration tool, not as a comparison to AgentFM. AgentFM's detection/diagnosis numbers are reported in isolation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included in the AgentFM evaluation, so this cannot be assessed. The preliminary study uses PLELog (2021) as the sole method, without comparing against more recent approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is provided. The paper proposes a multi-component system (system agents, data agents, task agents, meta-agent) but does not test the contribution of any individual component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports precision, recall, and F1-score for both anomaly detection and failure diagnosis, and additionally includes manual analysis of mitigation solutions (Figure 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states 'the mitigation solutions generated are manually analyzed to determine their usefulness' and 'For failure diagnosis, we manually review the LLM-generated results.' Human evaluation of the system's outputs is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes manually injecting anomalies into Apache IoTDB but does not describe a train/test split. The RAG+CoT approach uses 'historical data as labeled examples,' but it is unclear whether the test examples are separate from the historical examples used to guide the model."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports aggregate precision, recall, and F1 for anomaly detection and failure diagnosis as a whole, but does not break down results by anomaly type (e.g., CPU saturation vs. network partition vs. excessive import)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly notes: 'the model struggles to identify correct results when dealing with complex internal anomalies (e.g., excessive data import).' This is a specific failure case discussion."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No systematic negative results or ablation failures are reported. The single failure mention (excessive data import) is qualitative and not quantified. The paper presents AgentFM's performance favorably without discussing approaches that were tried and abandoned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'preliminary evaluations using Apache IoTDB demonstrate the effectiveness of AgentFM.' The results in Table 3 (F1 of 95.76% for detection, 87.62% for diagnosis) support this modest claim, and the paper consistently uses hedging language ('preliminary')."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims that role-awareness leads to better failure management (the core design argument), and that AgentFM's architecture 'facilitates specialized agents for each role—ensuring a more nuanced and effective approach.' However, without baselines, there is no causal evidence that role-awareness itself improves performance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently describes results as 'preliminary' and explicitly states experiments are conducted on Apache IoTDB only. The conclusion says 'preliminary experiments demonstrate the feasibility of AgentFM' and future work plans are acknowledged, so the scope is appropriately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed performance are considered. The paper does not discuss whether simpler baselines might achieve similar results, or whether the high detection F1 could be inflated by the test design (e.g., 20 injections per anomaly type, relatively simple scenarios)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Qwen2.5-72b' is used but provides no snapshot date, API version, or specific model checkpoint identifier. Model behavior can vary significantly across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 describes that 'all agents follow a common RAG+CoT approach' and that 'prompts vary by task,' but no actual prompt text is provided. The paper describes what prompts do in natural language without showing the actual text sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM hyperparameters are reported. The paper mentions using 'function call feature' for detection but provides no configuration details."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the multi-agent architecture in detail, including the meta-agent orchestration (Figure 1), System Role Manager workflow (Figure 2), data agent preprocessing pipelines (Sections 3.1-3.2), and the sequential task agent execution (Section 3.3). The scaffolding is described at a design level, though implementation details are sparse."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the Metric Agent preprocessing pipeline (noise removal, imputation, conversion to natural language) and Log Agent processing (sequence compression via log parsing, semantic compression via LLM summarization), with formal notation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 5 (Conclusion) mentions only that the model 'struggles to identify correct results when dealing with complex internal anomalies' as future work direction, not as a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed at all. Key threats—such as the evaluation on a single database system, lack of baselines, manual anomaly injection methodology, and the use of manually reviewed mitigation solutions—are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper uses 'preliminary' language throughout, it does not explicitly state what the results do not show. For example, it does not state that results are limited to time-series databases, single-cluster configurations, or the specific anomaly types tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (traces, metrics, logs, injected anomaly records) is made available. The experimental data is entirely inaccessible to independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection procedure: 10 anomaly types (listed explicitly) were manually injected into Apache IoTDB, each 20 times, generating traces/metrics/logs from the running system."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark-style evaluation with manually injected anomalies on a database system; there are no human participants or samples that require recruitment description."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the preprocessing components are described in Section 3.2, the full pipeline from anomaly injection to final evaluation numbers is not documented. It is unclear how many log/metric instances were collected, how anomalous vs. normal windows were defined for detection, and how training examples were selected for RAG."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section states: 'This work is supported by Key-Area Research and Development Program of Guangdong Province, China (NO.2020B010164003).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the title page: five authors are from Peking University and one (Yunpeng Zhai) is from Alibaba Group. Affiliations are clearly disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Guangdong Province government research program) is a government funding body with no apparent financial stake in the AgentFM framework or Apache IoTDB performance results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial disclosure statement. Yunpeng Zhai is affiliated with Alibaba Group, and the paper references Alibaba OceanBase and Alibaba Cloud business impacts, but no conflict-of-interest declaration is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen2.5-72b but does not state the model's training data cutoff date. This is relevant because the model might have been trained on documentation or logs from Apache IoTDB or similar systems."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between Qwen2.5-72b's training data and the Apache IoTDB documentation, configuration files, or known anomaly patterns used in the evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation uses manually injected anomalies on a live Apache IoTDB instance rather than a public benchmark dataset. The test cases were created by the authors and not published prior to the model's training, so benchmark contamination in the traditional sense does not apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a system evaluation paper with no human subjects study; pre-registration is not applicable."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental human study design; randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants experimental study; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, latency, or tokens consumed are reported. AgentFM calls Qwen2.5-72b across multiple agents per failure event, which could be expensive in practice, but costs are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, API spend, hardware configuration, or compute time is stated. The paper does not disclose what compute resources were used to run the Qwen2.5-72b model or the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Different nodes in a distributed database have varying significance for failure management; Node6 achieves the best anomaly detection performance due to hosting the largest number of leader partitions.",
       "evidence": "Table 1 in Section 2.1 shows PLELog-based anomaly detection F1-scores per node ranging from 47.53% (Node4) to 90.73% (Node6) for a manually injected 'excessive data export' anomaly.",
       "supported": "moderate"
     },
     {
       "claim": "Metrics are effective for detecting resource anomalies (CPU, memory) while logs excel at identifying internal database issues (data export/import, configuration errors).",
       "evidence": "Table 2 in Section 2.2 shows anomaly classification results using DTW-based classification on metrics vs. log data across 5 anomaly categories, with an F1 > 50% threshold for identifiability.",
       "supported": "weak"
     },
     {
       "claim": "AgentFM achieves 95.76% F1-score for anomaly detection and 87.62% F1-score for failure diagnosis on Apache IoTDB.",
       "evidence": "Table 3 in Section 4.2 reports precision/recall/F1 for anomaly detection (95.14%/97.03%/95.76%) and failure diagnosis (89.61%/87.04%/87.62%) using Qwen2.5-72b with 200 injected anomalies.",
       "supported": "weak"
     },
     {
       "claim": "AgentFM generates targeted and effective mitigation solutions for system remediation.",
       "evidence": "Section 4.2 and Figure 3 present one example mitigation output (recommendations for CPU resource increase and load balancing), described as 'targeted and effective assistance.'",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "AgentFM proposes a role-aware multi-agent framework for distributed database failure management that distinguishes system roles (node types), data roles (metrics vs. logs), and task roles (detection, diagnosis, mitigation). A preliminary empirical study on Apache IoTDB shows that node role correlates with anomaly detection performance and that metrics and logs identify different anomaly types. Preliminary evaluation using Qwen2.5-72b on 200 manually injected anomalies shows 95.76% F1 for detection and 87.62% F1 for diagnosis, though without baselines for comparison.",
   "red_flags": [
     {
       "flag": "No baseline comparison",
       "detail": "The main evaluation of AgentFM (Table 3) includes no comparison to any prior system or baseline. There is no way to assess whether the reported detection/diagnosis scores represent an improvement over simpler or existing approaches."
     },
     {
       "flag": "Single-run results, no variance",
       "detail": "All performance numbers are reported as single-point estimates with no repeated runs, standard deviation, or confidence intervals. Results may not be stable across different random seeds, prompt orderings, or anomaly injection timings."
     },
     {
       "flag": "Manual mitigation evaluation with single example",
       "detail": "The mitigation solution quality is assessed by manual review, and only a single example is shown (Figure 3). This provides very weak evidence of effectiveness across diverse failure scenarios."
     },
     {
       "flag": "No per-category breakdown despite multiple anomaly types",
       "detail": "Although 10 different anomaly types are injected, results are reported only as aggregate F1 scores. The single qualitative failure mention (excessive data import) suggests the aggregate score masks significant variation across anomaly types."
     },
     {
       "flag": "Model version underspecified",
       "detail": "The paper uses 'Qwen2.5-72b' without a snapshot date or checkpoint identifier. Given that Qwen2.5-72b may have been trained on Apache IoTDB documentation and logs, the model version specification matters for reproducibility and contamination assessment."
     },
     {
       "flag": "Alibaba affiliation without conflict-of-interest disclosure",
       "detail": "Co-author Yunpeng Zhai is affiliated with Alibaba Group, and the paper cites Alibaba Cloud and OceanBase as motivation. No competing interests statement is provided despite the potential financial interest."
     },
     {
       "flag": "Unclear train/test separation for RAG examples",
       "detail": "The RAG+CoT approach uses 'historical data as labeled examples' but it is not clear whether these labeled examples are drawn from the same 200 injected anomalies used for evaluation, which would constitute test leakage."
     }
   ],
   "cited_papers": [
     {
       "title": "Recommending root-cause and mitigation steps for cloud incidents using large language models",
       "authors": ["Toufique Ahmed", "Supriyo Ghosh", "Chetan Bansal", "Thomas Zimmermann", "Xuchao Zhang", "Saravan Rajmohan"],
       "year": 2023,
       "relevance": "Directly related work on using LLMs for cloud incident root cause analysis and mitigation recommendation."
     },
     {
       "title": "Eadro: An end-to-end troubleshooting framework for microservices on multi-source data",
       "authors": ["Cheryl Lee", "Tianyi Yang", "Zhuangbin Chen", "Yuxin Su", "Michael R Lyu"],
       "year": 2023,
       "relevance": "Related work on multimodal (trace+log+metric) failure management for microservice systems, a key comparison point."
     },
     {
       "title": "A Survey of AIOps for Failure Management in the Era of Large Language Models",
       "authors": ["Lingzhe Zhang", "Tong Jia", "Mengxi Jia", "Yifan Wu", "Aiwei Liu", "Yong Yang", "Zhonghai Wu", "Xuming Hu", "Philip S Yu", "Ying Li"],
       "year": 2024,
       "arxiv_id": "2406.11213",
       "relevance": "Survey of LLM-based AIOps for failure management, directly scoping the research landscape this paper contributes to."
     },
     {
       "title": "Building AI Agents for Autonomous Clouds: Challenges and Design Principles",
       "authors": ["Manish Shetty", "Yinfang Chen", "Gagan Somashekar", "Minghua Ma", "Yogesh Simmhan", "Xuchao Zhang", "Jonathan Mace", "Dax Vandevoorde", "Pedro Las-Casas", "Shachee Mishra Gupta"],
       "year": 2024,
       "relevance": "Related work on autonomous cloud operations using AI agents, relevant to the agentic AIOps theme."
     },
     {
       "title": "Exploring llm-based agents for root cause analysis",
       "authors": ["Devjeet Roy", "Xuchao Zhang", "Rashi Bhave", "Chetan Bansal", "Pedro Las-Casas", "Rodrigo Fonseca", "Saravan Rajmohan"],
       "year": 2024,
       "relevance": "LLM-based agent approach for root cause analysis, a direct comparison to AgentFM's diagnosis capability."
     },
     {
       "title": "mABC: multi-Agent Blockchain-Inspired Collaboration for root cause analysis in micro-services architecture",
       "authors": ["Wei Zhang", "Hongcheng Guo", "Jian Yang", "Yi Zhang", "Chaoran Yan", "Zhoujin Tian", "Hangyuan Ji", "Zhoujun Li", "Tongliang Li", "Tieqiao Zheng"],
       "year": 2024,
       "arxiv_id": "2404.12135",
       "relevance": "Multi-agent approach to root cause analysis in microservices, closely related to AgentFM's multi-agent architecture."
     },
     {
       "title": "Semi-supervised log-based anomaly detection via probabilistic label estimation",
       "authors": ["Lin Yang", "Junjie Chen", "Zan Wang", "Weijing Wang", "Jiajun Jiang", "Xuyuan Dong", "Wenbin Zhang"],
       "year": 2021,
       "relevance": "PLELog is used as the baseline anomaly detection method in the preliminary empirical study (Table 1)."
     },
     {
       "title": "Automated root causing of cloud incidents using in-context learning with gpt-4",
       "authors": ["Xuchao Zhang", "Supriyo Ghosh", "Chetan Bansal", "Rujia Wang", "Minghua Ma", "Yu Kang", "Saravan Rajmohan"],
       "year": 2024,
       "relevance": "In-context learning with GPT-4 for cloud incident root causing, directly relevant to LLM-based AIOps."
     },
     {
       "title": "Multivariate Log-based Anomaly Detection for Distributed Database",
       "authors": ["Lingzhe Zhang", "Tong Jia", "Mengxi Jia", "Ying Li", "Yong Yang", "Zhonghai Wu"],
       "year": 2024,
       "relevance": "Prior work from the same group on log-based anomaly detection for distributed databases, a direct predecessor to AgentFM."
     },
     {
       "title": "Robust failure diagnosis of microservice system through multimodal data",
       "authors": ["Shenglin Zhang", "Pengxiang Jin", "Zihan Lin", "Yongqian Sun", "Bicheng Zhang", "Sibo Xia", "Zhengdan Li", "Zhenyu Zhong", "Minghua Ma", "Wa Jin"],
       "year": 2023,
       "relevance": "Multimodal failure diagnosis for microservices, relevant as a comparison system for multi-source data integration approaches."
     }
   ]
 }