ai-research-survey

calibration.json (14607B)

---

 {
   "paper_slug": "agentfm-roleaware-failure-2025",
   "calibration_date": "2026-02-28",
   "model": "opus",
   "total_questions": 50,
   "agreement_count": 50,
   "disagreement_count": 0,
   "agreement_rate": 1.0,
   "disagreements": [],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, Zenodo archive, or any other code release is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The injected anomaly dataset, collected traces, metrics, and logs used for evaluation are not released. Apache IoTDB itself is public but the experimental data is not provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or dependency listing is provided. Only 'Qwen2.5-72b' and 'Apache IoTDB' are mentioned without environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, reproduction scripts, or step-by-step instructions are provided. Section 4 describes the experimental design at a high level but not how to reproduce it."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, and 3 are reported as point estimates only (e.g., 'F1-Score 95.76%'). No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The preliminary study (Table 1) and main evaluation (Table 3) present raw numbers without any testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Without baselines in the main AgentFM evaluation, there is no reference point for effect size computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 10 anomaly types with 20 injections each (200 total) but no justification is provided for why this sample size was chosen or whether it is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are single-run point estimates. No variance, standard deviation, or results across multiple runs are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The main evaluation of AgentFM (Table 3) includes no baseline comparisons. PLELog in Table 1 is used in the preliminary empirical study as a tool to motivate the approach, not as a comparison to AgentFM."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included in the AgentFM evaluation, making this criterion impossible to satisfy. The preliminary study uses only PLELog (2021)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "AgentFM has multiple components (system agents, data agents, task agents, meta-agent) but no ablation study tests the contribution of any individual component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports precision, recall, and F1-score for both anomaly detection and failure diagnosis tasks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'the mitigation solutions generated are manually analyzed to determine their usefulness' and 'we manually review the LLM-generated results.' The system's outputs are evaluated by humans."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The RAG+CoT approach uses 'historical data as labeled examples' for in-context learning, but it is not clear whether the test examples are separated from the historical examples used to guide the model."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Although 10 different anomaly types are injected, Table 3 reports only aggregate precision, recall, and F1 without per-anomaly-type breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 notes: 'the model struggles to identify correct results when dealing with complex internal anomalies (e.g., excessive data import).' This is a specific failure case."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No systematic negative results are reported. The brief mention of difficulty with complex internal anomalies is qualitative and not quantified. No approaches tried and abandoned are described."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'preliminary evaluations using Apache IoTDB demonstrate the effectiveness of AgentFM.' Table 3 results (F1 95.76% detection, 87.62% diagnosis) support this appropriately hedged claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper argues role-awareness leads to better failure management and that AgentFM's architecture 'ensures a more nuanced and effective approach.' Without baselines or ablations, there is no causal evidence that role-awareness itself drives the performance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently uses 'preliminary' language and tests only on Apache IoTDB. The conclusion says 'preliminary experiments demonstrate the feasibility of AgentFM.' Scope is appropriately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for results are discussed. The paper does not consider whether simpler approaches or the LLM's pre-existing knowledge of Apache IoTDB could explain the results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Qwen2.5-72b' but does not specify the variant (base vs. instruct), snapshot date, or checkpoint identifier. Multiple Qwen2.5-72B variants exist with substantially different behaviors."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 describes a 'RAG+CoT approach' and notes that 'prompts vary by task,' but no actual prompt text is provided. Prompts are described only in natural language."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM configuration parameters are reported despite using an LLM API for all agents."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the multi-agent architecture in detail: meta-agent orchestration (Figure 1), System Role Manager (Figure 2), data agent pipelines (Sections 3.1-3.2), and sequential task agent execution (Section 3.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the Metric Agent preprocessing (noise removal, imputation, NL conversion with formal notation) and Log Agent processing (sequence compression via log parsing, semantic compression via LLM summarization)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion briefly mentions future work directions but does not substantively discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Key threats such as single-system evaluation, lack of baselines, manual anomaly injection methodology, and unclear train/test separation are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While 'preliminary' language is used throughout, the paper does not explicitly state what the results do not show or what settings/systems are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (traces, metrics, logs, anomaly injection records) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection: 10 anomaly types are listed explicitly (CPU saturation, IO saturation, memory saturation, etc.), each injected 20 times into Apache IoTDB."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a system evaluation with manually injected anomalies on a database; there are no human participants or samples requiring recruitment description."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While Section 3.2 describes preprocessing components, the full pipeline from anomaly injection to final evaluation numbers is not documented. How normal vs. anomalous windows were defined, how many instances per class were collected, and how RAG examples were selected remain unclear."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section states: 'This work is supported by Key-Area Research and Development Program of Guangdong Province, China (NO.2020B010164003).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed on the title page: five from Peking University and one (Yunpeng Zhai) from Alibaba Group."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Guangdong Province government research program) is a government body with no apparent financial stake in AgentFM's performance or Apache IoTDB."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. Yunpeng Zhai is from Alibaba Group, and the paper references Alibaba Cloud and OceanBase as motivation, but no conflict-of-interest declaration is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen2.5-72b for anomaly detection and diagnosis but does not state its training data cutoff. The LLM's prior knowledge of Apache IoTDB patterns could influence results."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Qwen2.5-72b's training data includes Apache IoTDB documentation, log patterns, or similar system operational knowledge that could affect evaluation results."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation uses manually injected anomalies on a live Apache IoTDB instance, not a published benchmark dataset. The test cases were created by the authors and were not publicly available before the model's training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; pre-registration is not applicable."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental human study design; randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental human study; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, latency, or token consumption is reported despite AgentFM making multiple LLM calls across several agents per failure event."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, hardware configuration, or compute time is stated for running Qwen2.5-72b or the experiments."
       }
     }
   }
 }