ai-research-survey

scan.json (26640B)

---

 {
   "paper": {
     "title": "Think Locally, Explain Globally: Graph-Guided LLM Investigations via Local Reasoning and Belief Propagation",
     "authors": [
       "Saurabh Jha",
       "Rohan Arora",
       "Bhavya",
       "Noah Zheutlin",
       "Paulina Toro Isaza",
       "Laura Shwartz",
       "Yu Deng",
       "Daby Sow",
       "Ruchi Mahindru",
       "Ruchir Puri"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.17915",
     "doi": "10.48550/arXiv.2601.17915"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EoG, a disaggregated architecture separating LLM-based local abductive reasoning from deterministic graph traversal and belief propagation, achieves 7x higher Majority@k F1 than ReAct baselines on ITBench SRE diagnostic scenarios. The reliability gap between Pass@k and Majority@k is near-eliminated (e.g., GPT-5.1 goes from 22.9/8.6 under ReAct to 88.9/86.1 under EoG). Semantic Belief Propagation provides 7-64% additional improvement over structured traversal alone, with largest gains on weaker models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the implementation is 'available as part of the open-source Codex project' at https://github.com/openai/codex (§B.8, §5). ITBench is also described as open-source."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "ITBench is described as an open-source benchmark (§1 footnote, §3). The evaluation scenarios are part of the public benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions ~5,000 lines of Rust and Codex CLI v0.76 but no dependency or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the architecture and prompts but does not include a README or commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2-4 report point estimates (F1, Recall) with no confidence intervals or error bars despite running 3 trials per scenario."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite claiming EoG outperforms ReAct across multiple models. Comparisons are made by directly comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context: '7× average gain in Majority@k F1' (abstract), and Table 2 shows absolute values for both systems (e.g., GPT-5.1: 8.6→86.1 maj@3 F1). Table 4 reports relative improvement percentages (e.g., +6.9% to +64.3%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 35 scenarios with 3 runs was chosen. No power analysis or discussion of whether this sample is sufficient for the claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper uses Majority@k vs Pass@k to illustrate consistency, no standard deviation, IQR, or variance across the 3 runs is reported in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "ReAct baselines are included across 4 models (Table 2). The ReAct baseline 'incorporates best practices from prior work' and achieves 'higher performance than published baselines on ITBench' (§5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use contemporary models (GPT-5.1, Gemini 3 Flash/Pro, Claude Opus 4.5, Kimi K2) and the ReAct implementation uses the current Codex CLI v0.76 framework."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Two ablation studies: (1) Prompting ablation showing ReAct prompted with EoG algorithm (Table 3), and (2) SBP ablation isolating belief propagation contribution (Table 4). Also ablation of distributed traces as data source."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics reported: Pass@k F1, Pass@k Recall, Majority@k F1, Majority@k Recall for RC Entity, plus Majority@k Recall for RC Reasoning (Table 2). Also input/output token counts."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation uses LLM-as-a-Judge (Appendix D) rather than human evaluation. No human expert validation of the diagnoses is reported."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "ITBench scenarios serve as the test set. The EoG system is evaluated on 35 ITBench SRE scenarios that are independent of the system's development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per model (Table 2), with separate metrics for RC Entity and RC Reasoning. Failure mode analysis broken down by model (Figures 1, 2, 7). MAST failure taxonomy with per-category prevalence rates (§C.1)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure mode analysis in §3 (exploration failures, controller failures, path dependence). Figures 2a-c show GT discovery funnel, evidence discovery failures. §C analyzes plan abandonment, tool repetition, syntactic failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows ReAct prompted with EoG algorithm collapses to near-zero Majority@k, a negative result. The distributed traces ablation showed 'no statistically significant change in results' (§5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims '7× average gain in Majority@k F1 score' and 'improves accuracy and run-to-run consistency' are supported by Table 2 results across multiple models."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about EoG's design improvements are justified through ablation studies: SBP ablation (Table 4) isolates belief propagation contribution, prompting ablation (Table 3) shows the controller is necessary. These are controlled single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on ITBench SRE scenarios but makes broad claims about 'diagnostic investigations' generalizing to 'medical diagnosis', 'security threat analysis', 'FinOps, DevOps, healthcare, and forensics' (§2). While the methodology section references these domains, no evidence is provided outside IT SRE. The title says 'Graph-Guided LLM Investigations' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the observed improvements. Could the gains be due to the specific prompts used rather than the architecture? Could the ITBench scenarios be particularly suited to graph traversal? These alternatives are not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes what it measures (RC Entity F1/Recall, RC Reasoning accuracy on ITBench scenarios) from broader diagnostic capability. The Pass@k vs Majority@k distinction explicitly addresses the gap between sporadic success and consistent reasoning."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'GPT-5.1', 'GPT-OSS-120B', 'Gemini 3 Flash', 'Gemini 3 Pro', 'Kimi K2', 'Claude Opus 4.5', etc. — marketing names without specific API versions or snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text provided for all system components: ReAct agent prompt (§A.2), and all EoG prompts — select_investigation_window, filter_events, filter_spec_changes, bootstrap, explore (πabd), and finalize (§B.5-B.7)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§B.8 states 'All LLM calls use temperature 0 for determinism.' Also reports kthresh for damping, kmax=5 max visits, kcool=2 cooldown, >80K token chunking threshold, and 'high reasoning effort settings' for experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The EoG scaffolding is described in extensive detail: Deterministic Controller, Context Contract (CxC), Abductive Policy, event-driven architecture (§4, §B). MCP tool interfaces documented (Table 1, Table 6). Algorithm pseudocode provided (§4.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing described: Context Contract implements token-aware pagination, tournament-style reduction for relevance filtering (§B.8). Event and spec change filtering with explicit criteria (§B.5.2-B.5.3). Entity normalization format documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The paper has an 'Impact Statement' (§8) but it contains only a generic disclaimer: 'There are many potential societal consequences of our work, none of which we feel must be specifically highlighted here.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. The paper does not address threats like benchmark representativeness, LLM-as-a-Judge reliability, or the small number of runs (k=3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper claims generalizability to healthcare, forensics, FinOps (§2) without stating what the results do NOT show. §B.9.1 lists correctness assumptions but these are formal properties, not scope limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw agent trajectories, detailed per-scenario results, or raw evaluation data are released. Only aggregate metrics in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "ITBench is described as containing 35 SRE scenarios 'spanning misconfigurations, resource exhaustion, and cascading failures' (§5). The evaluation uses 3 runs per scenario with LLM-as-a-Judge evaluation (Appendix D)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses benchmark scenarios from ITBench."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: scenarios run → agent produces JSON output → LLM-as-a-Judge evaluates against ground truth with entity mapping (Appendix D). Entity normalization, alias handling, and scoring rubrics are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section. All authors are from IBM Research but no funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as 'IBM Research, Yorktown Heights, New York, USA' on page 1."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are IBM Research employees. IBM has commercial interests in IT operations tools (IBM Instana is cited). The funder (IBM) has a financial interest in demonstrating AI-driven IT diagnostics."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement. IBM Instana (IBM's commercial observability product) is cited but no conflict is acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the LLMs used (GPT-5.1, Gemini, etc.), despite evaluating on ITBench which could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ITBench scenarios or solutions could appear in the training data of the evaluated models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ITBench was published in 2025. Models like GPT-5.1 and Gemini 3 could have been trained on data including ITBench papers and solutions. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports average input and output token counts per trial per scenario for each agent/model combination (e.g., EoG GPT-5.1: 1837K input, 1.7K output). Note excludes reasoning tokens."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, API costs, or wall-clock time reported. Token counts are per-scenario averages but total cost across all experiments is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0 is used for determinism, but no seed sensitivity analysis is provided. The paper uses k=3 runs to measure Pass@k vs Majority@k, but does not report variance across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: '3 runs per scenario to measure both accuracy and consistency' (§5). k=3 is used throughout for Pass@k and Majority@k."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The damping threshold kthresh, kmax=5, kcool=2 appear to be set without describing how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for how hyperparameters (kthresh, kmax, kcool, token budget thresholds) were selected. No mention of validation set or selection process."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across 8+ models with no correction applied. No significance tests are used at all, so correction is moot, but the issue remains."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors built both EoG and the ReAct baseline. No acknowledgment that their implementation of the baseline may systematically underperform. While they note the baseline 'achieves higher performance than published baselines,' the ReAct implementation choices could still favor EoG."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports token usage for both EoG and ReAct. EoG uses more input tokens (e.g., 1837K vs 779K for GPT-5.1) but far fewer output tokens (1.7K vs 9.3K). The compute difference is visible, though not formally analyzed as a function."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ITBench scenarios are representative of real-world SRE incidents. No analysis of construct validity — whether success on ITBench implies capability in actual production diagnosis."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The scaffold confound is directly addressed: EoG vs ReAct comparisons use the same models, explicitly isolating the scaffolding effect. The prompting ablation (Table 3) further controls for this by giving ReAct the EoG algorithm as instructions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models could have seen ITBench solutions or similar diagnostic patterns during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides hints not available in real SRE usage. The MCP tools abstract away real production complexity."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 35 ITBench scenarios are independent or share structural similarities that could inflate performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied."
       }
     }
   },
   "claims": [
     {
       "claim": "EoG achieves 7× higher Majority@k F1 than ReAct baselines on ITBench SRE scenarios.",
       "evidence": "Table 2 shows EoG maj@3 F1 scores ranging from 64.2-92.1% vs ReAct 5.7-74.3%. Average gain computed across models (§5, abstract).",
       "supported": "strong"
     },
     {
       "claim": "EoG near-eliminates the reliability gap between Pass@k and Majority@k.",
       "evidence": "Table 2: GPT-5.1 gap reduces from 14.3pp (22.9-8.6) under ReAct to 2.8pp (88.9-86.1) under EoG. Similar narrowing across models.",
       "supported": "strong"
     },
     {
       "claim": "Prompting ReAct with EoG algorithm instructions is insufficient — the deterministic controller is necessary.",
       "evidence": "Table 3: ReAct prompted with EoG algorithm achieves 40-71.4% pass@3 but 0-2.9% maj@3, showing instruction-following cannot substitute for external control.",
       "supported": "strong"
     },
     {
       "claim": "Semantic Belief Propagation provides 7-64% additional improvement over structured traversal alone.",
       "evidence": "Table 4: SBP ablation shows relative improvements from +6.9% (GPT-5.1) to +64.3% (GPT-OSS-20B) on maj@3 Recall.",
       "supported": "strong"
     },
     {
       "claim": "ReAct agents fail primarily due to exploration failures (not reasoning failures).",
       "evidence": "Figure 2b shows high 'Evidence not in context' rates across models, while 'Evidence ignored' is small. Figure 2c shows 14-70% improvement from oracle reordering without adding new information (§3.2).",
       "supported": "moderate"
     },
     {
       "claim": "The EoG formalism generalizes to SWE tasks for diagnosing production bugs.",
       "evidence": "Only a citation to Cui et al. (2025) is provided (§4 contributions). No empirical evidence in this paper.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own system",
       "detail": "All authors are from IBM Research. IBM has commercial observability products (IBM Instana is cited). The paper proposes an architecture that could be productized. No conflict of interest statement is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations discussion despite making broad generalization claims to healthcare, forensics, FinOps, etc. The Impact Statement is a generic non-answer."
     },
     {
       "flag": "Small k for consistency claims",
       "detail": "Consistency claims (Majority@k) are based on k=3 runs. With 3 runs, Majority@3 requires 2/3 success — a small sample for claiming 'consistent reasoning.' No confidence intervals are provided."
     },
     {
       "flag": "LLM-as-a-Judge evaluation",
       "detail": "The evaluation uses an LLM judge (Appendix D) with complex entity mapping and semantic scoring. No human validation of the judge's accuracy is reported, despite the authors noting the evaluation has non-trivial complexity (alias groups, regex matching)."
     },
     {
       "flag": "No error bars despite stochastic claims",
       "detail": "The paper's central argument is about reliability and consistency, yet no standard deviations, confidence intervals, or significance tests are reported across the 3 runs."
     }
   ],
   "cited_papers": [
     {
       "title": "ITBench: Evaluating AI agents across diverse real-world IT automation tasks",
       "authors": ["Saurabh Jha", "Rohan R. Arora"],
       "year": 2025,
       "relevance": "Primary benchmark used for evaluation; tests AI agents on real-world IT diagnostic tasks."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao"],
       "year": 2023,
       "relevance": "Primary baseline agent paradigm evaluated and critiqued in this paper."
     },
     {
       "title": "Executable code actions elicit better LLM agents",
       "authors": ["Xingyao Wang"],
       "year": 2024,
       "relevance": "CodeAct framework used as basis for the ReAct baseline implementation."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Characterizes failure modes of multi-agent LLM systems including tool invocation failures."
     },
     {
       "title": "From local to global: A graph RAG approach to query-focused summarization",
       "authors": ["Darren Edge"],
       "year": 2024,
       "relevance": "Graph-RAG approach for knowledge graph reasoning, compared against EoG in Table 5."
     },
     {
       "title": "Exploring LLM-based agents for root cause analysis",
       "authors": ["Devjeet Roy"],
       "year": 2024,
       "relevance": "Prior work on LLM-based IT diagnostics; ReAct baseline incorporates best practices from this work."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F. Liu"],
       "year": 2024,
       "relevance": "Documents how LLMs lose information in long contexts, motivating EoG's bounded context design."
     },
     {
       "title": "GALA: Can graph-augmented large language model agentic workflows elevate root cause analysis?",
       "authors": ["Yining Tian"],
       "year": 2025,
       "relevance": "Graph-augmented LLM diagnostic agent compared in Table 5."
     },
     {
       "title": "STRATUS: A multi-agent system for autonomous reliability engineering of modern clouds",
       "authors": ["Yuchen Chen"],
       "year": 2025,
       "relevance": "Multi-agent cloud reliability system compared in Table 5."
     },
     {
       "title": "AgentCompass: Towards reliable evaluation of agentic workflows in production",
       "authors": ["Nishant Kartik"],
       "year": 2025,
       "relevance": "Addresses reliability evaluation of agentic systems in production, directly relevant to the reliability gap concept."
     },
     {
       "title": "Theorem-of-thought: A multi-agent framework for abductive, deductive, and inductive reasoning in language models",
       "authors": ["Sara Abdaljalil"],
       "year": 2025,
       "relevance": "Uses Bayesian belief propagation over agent reasoning graphs; contrasted with EoG's sequential belief revision approach."
     },
     {
       "title": "When agents go astray: Course-correcting SWE agents with PRMs",
       "authors": ["Sarthak Gandhi"],
       "year": 2025,
       "relevance": "Addresses plan revision failures in SWE agents, related to plan abandonment analysis in this paper."
     }
   ]
 }