ai-research-survey

scan.json (27112B)

---

 {
   "paper": {
     "title": "DoVer: Intervention-Driven Auto Debugging for LLM Multi-Agent Systems",
     "authors": ["Ming Ma", "Jue Zhang", "Fangkai Yang", "Yu Kang", "Qingwei Lin", "Saravan Rajmohan", "Dongmei Zhang"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2512.06749",
     "doi": "10.48550/arXiv.2512.06749"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DoVer, an intervention-driven debugging framework for LLM multi-agent systems, recovers 18-28% of failed trials on GAIA/AssistantBench within Magentic-One and 49% on GSMPlus within AG2. The framework validates or refutes 30-60% of failure hypotheses. Log-based single-step failure attribution suffers from fundamental ground-truth annotation uncertainty, with 14 of 29 GAIA cases exhibiting uncertain labels. Open-source models (Qwen3-8B/32B) can power DoVer with narrowing gaps to GPT-4o.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Project website and code will be available at https://aka.ms/DoVer' — a promise of future release, not a current release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The datasets used (GAIA, AssistantBench, GSMPlus, WW) are publicly available benchmarks. The paper builds on these standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the pipeline conceptually but lacks executable reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 2 and 3 report point estimates (e.g., '17.6%', '27.5%') without confidence intervals or error bars. Table 5 includes ± notation for reproduction experiments but the core DoVer results lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used when comparing DoVer's performance across settings or against baselines (Self-Refine, CRITIC). Differences are asserted by comparing raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: e.g., '17.6% trial success rate' on WW vs '27.5%' on GAIA-Level-1, '49% flip rate' on GSMPlus, and '+15.7% progress made'. These provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the sample sizes used. The number of failed cases (26 per dataset for M1) is determined by the benchmark and system performance, not by power analysis or design choice."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we perform three independent runs for each intervention' but does not report variance across these runs. Only aggregate success rates are shown in Tables 2-3."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 compares DoVer against Self-Refine-style and CRITIC-style baselines, both achieving 0% recovery vs DoVer's 17.6%."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Self-Refine (Madaan et al., 2023) and CRITIC (Gou et al., 2023) are recent and relevant self-improvement methods. The paper also discusses contemporaneous works in Section 2."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 includes ablations on different DoVer underlying models (GPT-4o, Qwen3-8B, Qwen3-32B) and few-shot prompting effects."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Trial Success Rate, Progress Made (milestone-based), and hypothesis validation categories (Validated/Partially Validated/Refuted/Inconclusive)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Milestone progress and 'intervention fulfilled' assessments rely on LLM-as-a-judge (GPT-5). The paper acknowledges this limitation in Section 7 but does not include human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard benchmark validation/test splits (GAIA Level-1 validation set, WW hand-crafted cases, GSMPlus testmini split)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (WW-AB, WW-GAIA, GAIA-Level-1, GSMPlus) and by hypothesis validation category (Tables 2-3). Table 4 shows per-model breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 presents detailed qualitative case studies for Refuted and Inconclusive outcomes, and Appendix D covers Validated and Partially Validated cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that WW-AB achieves '+0% progress made' after intervention, and that 29-67% of cases are Inconclusive. Section 5.5 analyzes why interventions fail."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims ('flips 18-28% of failed trials', 'up to 16% milestone progress', 'validates or refutes 30-60%', '49% on GSMPlus') are all directly supported by Tables 2 and 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The intervention framework is inherently causal — it edits specific steps and re-executes to measure the effect. The ablation design (varying model, few-shot) uses controlled single-variable manipulation. The paper's core design IS a causal methodology for debugging."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations and Generalizability) explicitly bounds the scope: two agent frameworks, specific task types (web-based info seeking, math), sequential orchestration, and specific model families. States results are 'evidence of feasibility rather than universal guarantees.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses LLM stochasticity (mitigated by 3 runs), tool limitations as alternative explanations for Inconclusive cases (Section 5.5), and that intervention failures may reflect sub-agent capability gaps rather than wrong hypotheses."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between Trial Success Rate (binary task completion) and Progress Made (milestone-based partial credit), and acknowledges in Section 4.2 that milestone evaluation relies on LLM-as-a-judge as a proxy for human judgment."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 3 specifies 'GPT-4o-20241120' and 'GPT-5-chat-20250807'. Open-source models specified as 'Qwen3-8B' and 'Qwen3-32B in thinking mode'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix B: Figures 5-11 show the trial segmenter, failure proposer, intervention recommender, milestone extractor, milestone evaluator, and post-intervention classifier prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Footnote 3 states 'All LLM API calls are made through Azure OpenAI using default parameters' but does not specify what those defaults are (temperature, top-p, max tokens)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The DoVer pipeline is described in detail (Section 4): trial segmentation, failure attribution, intervention generation, intervention execution with checkpoint/replay. The Magentic-One and AG2 frameworks are described including their orchestration patterns."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the failure trace collection process, how datasets were constructed (e.g., excluding WW cases from GAIA-Level-1), and Table 1 documents the filtering from total cases to failed cases to intervened cases."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is a dedicated 'Limitations and Generalizability' section with substantial discussion spanning a full page."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: reliance on LLM-as-a-judge for milestone evaluation, restriction to orchestrator-level text interventions, requirement for checkpoint/replay infrastructure, and limitation to sequential orchestration topologies."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what was NOT tested: 'long-running production workloads, domains with strict latency or cost constraints, or settings with safety-critical requirements', asynchronous orchestrators, and sub-agent code modification."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The failure traces, intervention logs, and checkpoint data are not released. Only aggregate results in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes failure trace collection: initial run over all cases, evaluation of outcomes to identify failures, with GPT-4o generating and powering traces. Success rate matches reported M1 numbers."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard benchmarks (GAIA, AssistantBench, GSMPlus)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents the pipeline: total cases → failed cases → intervened cases → intervened trials, with counts at each stage. The filtering from failed to intervened is explained (LLMs may conclude no mistake occurred)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The first author is an intern at Microsoft and other authors are Microsoft employees, but no funding acknowledgment section exists."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly list Microsoft and Chinese Academy of Sciences. The footnote notes 'Work is done during an internship at Microsoft.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work is done at Microsoft, which develops the Magentic-One and AutoGen2 frameworks being evaluated. Microsoft has a commercial interest in demonstrating these frameworks can be improved."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for GPT-4o, GPT-5, or Qwen3 models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GAIA, AssistantBench, or GSMPlus benchmark problems appeared in the training data of the models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GAIA and AssistantBench were published in 2024, and the models (GPT-4o/GPT-5) may have been trained on data including these benchmarks. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock times are reported despite the pipeline making many LLM calls (trial segmentation + failure attribution + intervention generation + re-execution + evaluation, each repeated 3 times)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper runs thousands of GPT-4o and GPT-5 API calls across all experiments without quantifying cost or compute."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper performs 3 runs per intervention but does not report sensitivity to random seeds or show per-seed variation. Only aggregate success rates across all runs are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly states 'we perform three independent runs for each intervention' to reduce impact of LLM stochasticity."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Prompt designs appear manually crafted but the number of iterations or alternatives tried is not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The prompt refinements in Section 3 (step indices + guidance) are presented as iterative improvements but the selection process for the final configuration is not systematically justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft authors evaluate their own DoVer framework on Microsoft's Magentic-One and AutoGen2 systems. The bias of evaluating their own system is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "DoVer requires multiple LLM calls per trial (segmentation + attribution + intervention + re-execution + evaluation × 3 runs), which is substantially more compute than the baselines. This compute difference is not quantified or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides substantial analysis of why log-based failure attribution metrics (step accuracy) have poor construct validity due to ground-truth uncertainty, motivating the shift to outcome-based metrics. The paper questions what the benchmarks actually measure."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly tests DoVer across two different agent frameworks (Magentic-One and AG2) and different underlying models (GPT-4o, Qwen3-8B, Qwen3-32B), separating the scaffold variable from the model variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o/GPT-5 training data includes GAIA, AssistantBench, or GSMPlus benchmark solutions, despite these being published before model training cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The intervention generation stage receives the ground-truth answer as input (visible in Figure 8 prompt). While this is by design for debugging, the paper does not discuss how this ground-truth leakage affects the evaluation's ecological validity."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and test benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "DoVer flips 18-28% of failed trials into successes on GAIA/AssistantBench within Magentic-One.",
       "evidence": "Table 2 shows Trial Success Rate of 17.6% for WW-AB, 17.6% for WW-GAIA, and 27.5% for GAIA-Level-1.",
       "supported": "strong"
     },
     {
       "claim": "DoVer recovers 49% of failed trials on GSMPlus within AG2.",
       "evidence": "Table 2 shows 49.0% Trial Success Rate on 198 intervened trials in GSMPlus setting.",
       "supported": "strong"
     },
     {
       "claim": "DoVer validates or refutes 30-60% of failure hypotheses.",
       "evidence": "Table 3 shows Validated+Refuted rates: 29.2% for WW-AB, 37.4% for WW-GAIA, 71.4% for GAIA-Level-1. The 30-60% range is conservative; GAIA-Level-1 exceeds 60%.",
       "supported": "strong"
     },
     {
       "claim": "Ground-truth uncertainty substantially affects model attribution performance, with 14 of 29 GAIA cases exhibiting uncertain labels.",
       "evidence": "Section 3 and Table 7 document the re-annotation of 29 GAIA cases, finding 14 uncertain. Step accuracy is 24% (GPT-4o) / 7% (GPT-5) on uncertain cases vs 44% / 53% on certain cases.",
       "supported": "strong"
     },
     {
       "claim": "Self-Refine and CRITIC baselines fail to recover any failures (0% recovery) in the WW-GAIA setting.",
       "evidence": "Section 5.3 states 'neither baseline is able to flip any failure into success (0% recovery)' across all 26 failed WW-GAIA cases.",
       "supported": "moderate"
     },
     {
       "claim": "Open-source models (Qwen3-8B/32B) can power DoVer with narrowing gaps to GPT-4o.",
       "evidence": "Table 4 shows Qwen3-8B at 11.3% (0-shot) / 14.3% (3-shot) and Qwen3-32B at 16.9% vs GPT-4o at 17.6% trial success rate.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own frameworks",
       "detail": "Microsoft authors evaluate DoVer on Microsoft's Magentic-One and AutoGen2 frameworks. While testing on two frameworks adds validity, there is an inherent conflict of interest in showing these frameworks can be improved by another Microsoft-developed tool."
     },
     {
       "flag": "Ground-truth answer provided to intervention generator",
       "detail": "The intervention recommender prompt (Figure 8) receives the ground-truth answer as input. While intended for debugging, this means the system has access to the correct answer when generating interventions, which may inflate success rates compared to a realistic deployment scenario."
     },
     {
       "flag": "No cost reporting despite expensive pipeline",
       "detail": "DoVer requires multiple LLM calls per trial (segmentation + attribution + intervention + 3× re-execution + evaluation), which is substantially more compute than the baselines. Without cost reporting, the practical viability cannot be assessed."
     },
     {
       "flag": "Small sample sizes for some settings",
       "detail": "WW-AB (23 intervened cases), WW-GAIA (25), and GAIA-Level-1 (25) are small samples. Point estimates like '17.6% trial success rate' on these sizes have wide uncertainty that is not quantified."
     },
     {
       "flag": "LLM-as-a-judge for key metrics",
       "detail": "Both the Progress Made metric (milestone evaluation) and the 'intervention fulfilled' metric rely on GPT-5 as a judge. The paper acknowledges this limitation but does not validate the LLM judge against human ratings."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Catalogs failure patterns in multi-agent LLM systems across task interpretation, planning, tool interaction, and verification."
     },
     {
       "title": "Which agent causes task failures and when? On automated failure attribution of LLM multi-agent systems",
       "authors": ["Shaokun Zhang", "Ming Yin", "Jieyu Zhang"],
       "year": 2025,
       "relevance": "Introduces the Who&When (WW) dataset and single-agent/step failure attribution methodology for multi-agent systems."
     },
     {
       "title": "TRAIL: Trace reasoning and agentic issue localization",
       "authors": ["Darshan Deshpande", "Varun Gangal"],
       "year": 2025,
       "arxiv_id": "2505.08638",
       "relevance": "Creates turn-level traces with fine-grained taxonomy for debugging agent systems, showing strong models struggle at trace debugging."
     },
     {
       "title": "Magentic-one: A generalist multi-agent system for solving complex tasks",
       "authors": ["Adam Fourney", "Gagan Bansal"],
       "year": 2024,
       "arxiv_id": "2411.04468",
       "relevance": "Primary multi-agent framework used for DoVer experiments; centrally-orchestrated multi-agent architecture."
     },
     {
       "title": "Interactive debugging and steering of multi-agent AI systems",
       "authors": ["Will Epperson", "Gagan Bansal"],
       "year": 2025,
       "doi": "10.1145/3706598.3713581",
       "relevance": "AGDebugger: human-in-the-loop debugging tool for multi-agent systems with rewind/edit/re-execute capabilities."
     },
     {
       "title": "Agent-as-a-judge: Evaluate agents with agents",
       "authors": ["Mingchen Zhuge", "Changsheng Zhao"],
       "year": 2024,
       "arxiv_id": "2410.10934",
       "relevance": "Evaluation framework for agent systems using agents as evaluators, relevant to LLM-as-a-judge methodology."
     },
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier"],
       "year": 2024,
       "relevance": "Primary benchmark used for DoVer evaluation; tests general AI assistant capabilities across multiple difficulty levels."
     },
     {
       "title": "AssistantBench: Can web agents solve realistic and time-consuming tasks?",
       "authors": ["Ori Yoran", "Samuel Joseph Amouyal"],
       "year": 2024,
       "doi": "10.18653/V1/2024.EMNLP-MAIN.505",
       "relevance": "Benchmark for realistic web agent tasks, used as second evaluation dataset for DoVer."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation framework",
       "authors": ["Qingyun Wu", "Gagan Bansal"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent conversation framework used as second agent framework for DoVer generality evaluation."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon"],
       "year": 2023,
       "relevance": "Self-improvement baseline method compared against DoVer; achieves 0% recovery on multi-agent failures."
     },
     {
       "title": "Aegis: Automated error generation and attribution for multi-agent systems",
       "authors": ["Fanqi Kong", "Ruijie Zhang"],
       "year": 2025,
       "arxiv_id": "2509.14295",
       "relevance": "Specialized failure-tracer model for multi-agent systems; orthogonal approach that could strengthen DoVer's attribution stage."
     },
     {
       "title": "Can agents fix agent issues?",
       "authors": ["Alfin Wijaya Rahardja", "Junwei Liu"],
       "year": 2025,
       "arxiv_id": "2505.20749",
       "relevance": "AgentIssue-Bench: benchmark for agent-based repair of real agent system issues, finding low resolution rates."
     },
     {
       "title": "Evaluating agent-based program repair at Google",
       "authors": ["Pat Rondon", "Renyao Wei"],
       "year": 2025,
       "doi": "10.1109/ICSE-SEIP66354.2025.00038",
       "relevance": "Industrial evaluation of agent-based program repair on production bugs at Google scale."
     }
   ]
 }