ai-research-survey

scan.json (23947B)

---

 {
   "paper": {
     "title": "Detecting Silent Failures in Multi-Agentic AI Trajectories",
     "authors": [
       "Divya Pathak",
       "Harshit Kumar",
       "Anuska Roy",
       "Felix George",
       "Mudit Verma",
       "Pratibha Moogi"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2511.04032",
     "doi": "10.48550/arXiv.2511.04032"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper introduces anomaly detection for multi-agentic AI system trajectories, curating two benchmark datasets (4,275 and 894 traces) from Stock Market and Research Writing assistant systems. XGBoost achieves up to 98% accuracy in supervised settings, while semi-supervised SVDD reaches 96%, suggesting labeled data may not be necessary. Error analysis reveals that subtle drift anomalies without cycles or errors remain the hardest failure type to detect.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Footnote 1 states 'The dataset and curation pipeline will be released after paper acceptance in accordance with organizational policies.' No code is currently available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same footnote: datasets planned for release after acceptance, not currently available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports only point estimates for accuracy, F1, precision, and recall with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims XGBoost outperforms other models based on comparing numbers in Table 2, with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance differences are reported with baseline context, e.g., 'Recall (97.03%) is slightly lower (-0.46%) than SVM (97.49%)' and similar comparisons throughout Section 3.3."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes (4,275 and 894 traces). The number of input prompts (525 and 112) is stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or results across multiple runs are reported. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: supervised (XGBoost, Random Forest, LR, SVM, Naive Bayes), semi-supervised (SVDD, Isolation Forest), and unsupervised (K-Means)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are standard ML methods appropriate for the task. The paper notes that 'established techniques for AI Agents remain unexplored,' so there are no prior agent-specific baselines to compare against."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the feature set or pipeline components. The paper extracts 16 features but does not test subsets."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports accuracy, macro-F1, precision, and recall for each model."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two human annotators labeled the traces with inter-annotator agreement measured via Cohen's kappa (97.6% and 80.6%). This serves as human evaluation ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2: 'split into 70-15-15% for train-validation-test' with hyperparameters tuned on validation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model type (supervised/semi-supervised/unsupervised) and across both datasets. SHAP feature importance analysis provides per-feature breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 provides detailed error analysis of false negatives, with three specific insights about why misclassifications occur (subtle drift resembling normal traces)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "K-Means unsupervised performance is reported as moderate/weak, and the paper explicitly states 'there remains significant room for improvement in the unsupervised case.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 98% and 96% accuracy for XGBoost and SVDD are supported by Table 2 (Stock Market dataset: 98.03% and 96.47%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes detection performance to model capabilities without controlling for confounds. No causal claims are explicitly made, but the error analysis implies path-level features 'cause' better detection without rigorous causal design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract frames this as a study of 'Multi-Agentic AI systems' generally, but results are from only two specific applications (Stock Market and Research Writing assistants). The paper does note the pipeline 'can be readily extended' but does not bound generalization claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results, such as whether the high accuracy is due to the anomalies being too easy (most involve cycles/errors with obvious feature signatures)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper equates detecting anomalous traces (the proxy) with detecting 'silent failures' (the claimed outcome) without discussing the gap. Many silent failures may not manifest as anomalous feature vectors, and not all anomalous traces represent meaningful failures."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "LLMs mentioned are 'gpt-4o, ibm-granite-3-1-8B, meta-llama-3-3-70B' — gpt-4o lacks a version/snapshot date. Granite and Llama include size but not exact version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "System prompts are categorized as 'poor', 'good', and 'strict' with brief descriptions, but the actual prompt text is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 states 'Hyperparameters were tuned via grid search on the validation set' but does not report the actual hyperparameter values used."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent systems are described: Stock Market has 3 agents and 9 tools with named tools; Research Writing has 9 agents and 6 tools with named roles. The agentic trace structure is diagrammed in Figure 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the full pipeline: trace collection via OpenTelemetry, feature extraction (16 features in 5 categories), and labeling criteria with inter-annotator agreement."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Section 4 (Conclusions and Future Plans) mentions areas for improvement but does not substantively discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Future work mentions are about expanding methods, not about threats to the current study's validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what its results do NOT show or what settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is not available. Planned for release after acceptance."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes trace collection via OpenTelemetry instrumentation, with three factors varied (input query, LLM model, system prompt type). Specific numbers of prompts and models are provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — data consists of synthetically generated agentic traces, not human-collected data. The annotators labeled traces but were not study participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Section 2 document the full pipeline: trace collection → feature extraction (16 features) → labeling. Numbers at each stage are provided (e.g., 525 prompts × 3 prompt types × 3 LLMs = pipeline producing 4,275 datapoints)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. Five of six authors are from IBM Research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: IBM Research (5 authors) and IIIT Bangalore (1 author)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. IBM Research authors evaluate IBM Granite models alongside others, creating a potential conflict that is not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. IBM authors evaluate IBM Granite, which could represent a financial interest."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capabilities on a benchmark. The LLMs are used to generate agent traces, not evaluated for their knowledge. The anomaly detection is performed by classical ML models on extracted features."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — classical ML anomaly detection on extracted features, not evaluating pre-trained LLM capabilities."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the benchmark is newly created trace data, and the evaluated models are classical ML, not pre-trained LLMs."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Annotators labeled synthetic trace data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or token usage for the anomaly detection pipeline are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget or hardware information is provided for either trace generation or model training."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented as single values."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Grid search is mentioned but the search budget (number of configurations, ranges) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2: 'Hyperparameters were tuned via grid search on the validation set' — selection on validation set, not test."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple models are compared across two datasets with no statistical tests or multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the datasets and evaluated all methods themselves. No acknowledgment of self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute costs for different methods. Some methods (XGBoost vs K-Means) have very different compute profiles but this is not addressed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the two synthetic agent systems and their artificially generated traces actually represent real-world multi-agent failures. Construct validity is a major concern for synthetic benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The anomaly detection models are classical ML, not scaffolded LLM systems. No scaffold confound applies."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal ordering in train/test split. Traces from the same input prompts with different LLMs/prompts could appear in both train and test."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether extracted features (e.g., error status in spans) trivially reveal the label, which would make detection artificially easy."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Traces generated from the same input prompts with different LLMs/prompt types are likely structurally similar. No discussion of whether train/test split respects prompt-level independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "XGBoost achieves accuracies up to 98% and 94% on the Stock Market and Research Writing datasets respectively.",
       "evidence": "Table 2 shows XGBoost at 98.03% accuracy on Stock Market and 94.81% on Research Writing (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "Semi-supervised SVDD performs comparably to supervised methods, achieving 96% and 89% accuracy.",
       "evidence": "Table 2 shows SVDD at 96.47% and 89.63% on the two datasets (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "This is the first systematic study of anomaly detection in Multi-Agentic AI systems.",
       "evidence": "Section 1 states 'To the best of our knowledge no publicly available datasets currently exist' and 'established techniques for AI Agents remain unexplored.'",
       "supported": "moderate"
     },
     {
       "claim": "Path-level features are the most important for anomaly detection.",
       "evidence": "SHAP analysis in Section 3.3 shows tool_count, total_steps, unique_steps, and agent_count are consistently top-ranked.",
       "supported": "moderate"
     },
     {
       "claim": "False negatives are caused by subtle drift anomalies that resemble normal traces in feature space.",
       "evidence": "Error analysis in Section 3.3 with three insights showing FNs have feature values close to normal traces, supported by t-SNE visualization (Figure 2).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Potential feature leakage",
       "detail": "Error status is extracted directly from trace spans as a feature. Since 'errors' is one of the anomaly types, including error-derived features could make detection trivially easy for a large portion of anomalies. The 98% accuracy may partly reflect this circularity."
     },
     {
       "flag": "Synthetic benchmark validity",
       "detail": "Both datasets are generated from controlled systems with known configurations. Real-world multi-agent systems would have far more diverse and subtle failure modes. The high accuracies may not transfer."
     },
     {
       "flag": "Company evaluating own product",
       "detail": "IBM Research authors include IBM Granite as one of three LLMs without disclosing potential conflicts of interest or funding."
     },
     {
       "flag": "No variance or statistical tests",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or significance tests despite comparing 8 models across 2 datasets."
     },
     {
       "flag": "Train/test independence concern",
       "detail": "With 525 prompts generating 4,275 traces (~8 traces per prompt), traces from the same prompt likely share structural features. Random 70-15-15 split may place correlated traces in both train and test, inflating accuracy."
     },
     {
       "flag": "Low inter-annotator agreement on one dataset",
       "detail": "Cohen's kappa of 80.6% on the Research Writing dataset suggests substantial ambiguity in what constitutes an anomaly, yet the paper treats labels as ground truth."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent llm systems fail?",
       "authors": [
         "Mert Cemri",
         "Melissa Z Pan",
         "Shuyi Yang"
       ],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Directly studies failure modes in multi-agent LLM systems."
     },
     {
       "title": "Multi-agent risks from advanced ai",
       "authors": [
         "Lewis Hammond",
         "Alan Chan",
         "Jesse Clifton"
       ],
       "year": 2025,
       "arxiv_id": "2502.14143",
       "relevance": "Comprehensive analysis of risks in multi-agent AI systems relevant to safety research."
     },
     {
       "title": "SentinelAgent: Graph-based anomaly detection in multi-agent systems",
       "authors": [
         "Xu He",
         "Di Wu",
         "Yan Zhai",
         "Kun Sun"
       ],
       "year": 2025,
       "arxiv_id": "2505.24201",
       "relevance": "Graph-based anomaly detection approach for multi-agent systems, closely related work."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": [
         "Shunyu Yao",
         "Jeffrey Zhao",
         "Dian Yu"
       ],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational prompting pattern used in the agent system designs evaluated in this paper."
     },
     {
       "title": "XGBoost: A scalable tree boosting system",
       "authors": [
         "Tianqi Chen",
         "Carlos Guestrin"
       ],
       "year": 2016,
       "doi": "10.1145/2939672.2939785",
       "relevance": "Core ML method used as top-performing supervised anomaly detector in the study."
     },
     {
       "title": "A unified approach to interpreting model predictions",
       "authors": [
         "Scott M Lundberg",
         "Su-In Lee"
       ],
       "year": 2017,
       "relevance": "SHAP framework used for feature importance analysis of anomaly detection models."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Addresses a real problem (silent agent failures) but datasets aren't released yet and the techniques are standard ML classifiers, not a usable tool."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that semi-supervised methods nearly match supervised ones is mildly interesting but not shocking; otherwise results confirm expected ML baselines."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Silent failures in agentic systems touch on reliability concerns but the paper frames it as an engineering/monitoring problem, not a safety risk."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to specific claims or companies; straightforward benchmarking paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Datasets and code are not yet released ('will be released after paper acceptance'), so nothing to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "IBM Research is a recognized institution but not a top-tier ML hype brand; no famous product involved."
     }
   }
 }