ai-research-survey

scan.json (23142B)

---

 {
   "paper": {
     "title": "Interactive Debugging and Steering of Multi-Agent AI Systems",
     "authors": ["Will Epperson", "Gagan Bansal", "Victor Dibia", "Adam Fourney", "Jack Gerrits", "Erkang Zhu", "Saleema Amershi"],
     "year": 2025,
     "venue": "CHI '25 (International Conference on Human Factors in Computing Systems)",
     "arxiv_id": "2503.02068",
     "doi": "10.1145/3706598.3713581"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["qualitative", "case-study"],
   "key_findings": "Formative interviews with 5 agent developers revealed core debugging challenges: difficulty reviewing long multi-turn conversations, lack of interactive debugging support, and slow iteration on agent configuration. A user study (n=14) of AGDebugger showed participants used three steering strategies: adding specific instructions (14/24 edits), simplifying instructions (5/24), and modifying agent plans (5/24). Message resetting was rated most helpful (4.9/5), but only 2 of 8 participants successfully steered agents to the correct answer.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'AGDebugger is available as an open source tool at https://github.com/microsoft/agdebugger' (Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No study data (interview transcripts, survey responses, user study logs, edit histories) is released or linked."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the user study or running AGDebugger on the same tasks are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows mean scores with 95% confidence intervals for system and feature usability ratings."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares AGDebugger to a baseline condition in Part 1 and reports Likert ratings, but no statistical significance tests are performed on any comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (Cohen's d, etc.) are reported. Only raw Likert means and counts of edit types are given."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why n=5 (formative), n=6 (Part 1), and n=8 (Part 2) were chosen. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only means and 95% CIs are shown in Figure 7. No standard deviations or variance across participants is reported in tables or text."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Part 1 compares AGDebugger against a 'reduced version of the system that lacked the ability to reset messages or the overview visualization' (Section 6.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline represents current developer workflow (reading logs without interactive features), which is appropriate for this type of tool evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has three features (message sending, resetting/editing, overview visualization) but no ablation study isolates their individual contributions. Part 1 only compares full system vs. reduced version."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple measures are collected: system usability ratings (3 questions), feature-level ratings (3 features), edit type classification, number of edits, time to first edit, task success rate, and qualitative feedback."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire evaluation is human-centered: participants used the tool and provided ratings, think-aloud observations, and interview responses."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a qualitative user study, not a benchmark evaluation with train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by feature (message sending 4.1/5, resetting 4.9/5, overview 4.1/5 in Figure 7) and by edit type (14 add, 5 simplify, 5 modify in Figure 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 discusses open challenges including non-resettable actions, steering requiring deep implementation knowledge, and difficulty determining if edits had effect. Only 2/8 participants succeeded."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that only 2 of 8 participants steered agents to the correct answer (Section 6.3), no participants used the agent configuration feature, and participants were frustrated when agents didn't respond to edits (Section 7.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are well-supported: formative interviews identifying challenges (Section 4), three key features of AGDebugger (Section 5), common steering strategies (Section 6.4), and importance of message resets (Section 6.3, 4.9/5 rating)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies AGDebugger's features cause better debugging ('AGDebugger Facilitates Interactive Steering and Debugging' — Section 6.3 heading), but the Part 1 comparison has only n=6 with no statistical testing and confounded conditions (counterbalanced but not properly powered)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on only 2 GAIA Level-1 tasks with the Magentic-One team but makes general claims about 'debugging multi-agent AI systems' and 'interfaces for debugging increasingly important agentic workflows' (abstract). Section 6.5 notes this limitation but the title and abstract are not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for findings. E.g., the high rating for message resetting could be due to novelty effect, or the 2/8 success rate could reflect task difficulty rather than tool limitations. Section 6.5 mentions limitations but not alternative explanations for results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Likert ratings and task success (2/8 correct answers) but frames this as evidence the tool is 'helpful for debugging.' No discussion of whether Likert ratings and short-session success are adequate proxies for real-world debugging effectiveness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not specify which LLM model or version the Magentic-One agents use. No mention of GPT-4 version, temperature, or API details."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The agent prompts (Orchestrator, Web Surfer, etc.) are not provided in the paper or appendix. Only high-level descriptions of agent roles are given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for the LLM calls made by the agents."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the AutoGen framework (Python classes, message passing, runtime queue), Section 3.3 describes the 5-agent Magentic-One team with roles, tools, and communication patterns. Section 5.2.1 details checkpointing."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how interview notes were processed, how qualitative codes were developed, or how survey responses were aggregated beyond 'thematic analysis of interview notes' (Section 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 'User Study Limitations' provides a dedicated subsection discussing limitations of the study design."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 identifies specific threats: 30-minute session may be insufficient, only two GAIA tasks tested, tool not tested during active agent development. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 states 'we tested the system with users on only two benchmark tasks from GAIA' and 'it remains to be seen how the tool assists developers working on other types of tasks.' Also notes concurrent development use was 'not fully captured.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (survey responses, interview transcripts, task recordings, edit logs) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4 and 6.1 describe data collection: semi-structured interviews (1 hour, detailed notes), survey questionnaires, task recordings, think-aloud observations, and exit interviews. Questions are in Appendices A and B."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states formative interview participants were 'recruited within a large technology corporation' at Microsoft with AutoGen experience. Section 6.1 describes Part 1 and Part 2 recruitment from the same company with specified backgrounds."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The analysis pipeline is vaguely described: 'thematic analysis of interview notes' (Section 4), 'aggregated the Likert scale ratings' and 'qualitatively coded interview transcripts' (Section 6.1). No detail on coding process, inter-rater reliability, or transformation steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgment of financial support. The Acknowledgments section thanks individuals but does not mention funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: one CMU author and six Microsoft Research authors. The tool is built on AutoGen, a Microsoft open-source project."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft Research authors are evaluating a tool built on Microsoft's AutoGen framework using Microsoft's Magentic-One agent team. Microsoft has a commercial interest in the success of its agent ecosystem. This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a debugging tool, not a pre-trained model's capability on a benchmark. The agents are tools being debugged, not being evaluated for knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates a debugging interface, not model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the GAIA tasks are used as debugging targets, not to evaluate model capability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the formative interviews or the user study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite conducting interviews and user studies with human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports formative participants: 3 research scientists, 1 software engineer, 1 engineering manager. Section 6.1 reports Part 1: 4 grad students, 2 research scientists. Part 2: 3 research scientists, 5 grad students, with agent experience levels (none to extensive)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Formative interviews required 'experience building multi-agent applications' with AutoGen (Section 4). Part 1 required 'backgrounds in computer science and experience working with LLMs' (Section 6.1). Part 2 required being 'experienced with LLMs.'"
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 states 'The order of log reviews and system conditions was randomized and counterbalanced' for Part 1. Part 2 participants were 'assigned just one task' though assignment method is not detailed."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Participants could clearly see whether they were using AGDebugger or the reduced baseline version, and no evaluator blinding is mentioned."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is reported. It is unclear whether all recruited participants completed the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No costs reported for the LLM API calls made during debugging sessions, despite agents making many LLM calls (71-90 messages per task)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget is stated for running the agent teams or the debugging sessions."
       }
     }
   },
   "claims": [
     {
       "claim": "Agent developers face three core challenges: difficulty reviewing long conversations, lack of interactive debugging support, and need for better tooling to iterate on configurations.",
       "evidence": "Formative interviews with 5 developers described in Section 4, with themes synthesized via thematic analysis of interview notes.",
       "supported": "moderate"
     },
     {
       "claim": "Participants frequently made three types of modifications: adding specific instructions (14/24), simplifying instructions (5/24), and modifying agent plans (5/24).",
       "evidence": "Section 6.4 reports categorization of all 24 edits across 8 Part 2 sessions, with examples shown in Figure 9.",
       "supported": "strong"
     },
     {
       "claim": "Message resetting was the most highly rated feature (4.9/5) and every participant used it.",
       "evidence": "Section 6.3 and Figure 7 report Likert ratings with 95% CIs. Figure 8 shows all participants edited messages.",
       "supported": "strong"
     },
     {
       "claim": "AGDebugger effectively supported participants in iteratively steering agents towards correct behavior.",
       "evidence": "Only 2 of 8 participants steered agents to the correct answer (Section 6.3). Most participants improved understanding but not outcomes.",
       "supported": "weak"
     },
     {
       "claim": "Edits to earlier messages in the conversation were more effective for steering agents.",
       "evidence": "Section 7.1 notes 'Both of the participants who successfully steered the agents towards producing the correct answer editing messages towards the beginning of the conversation rather than at the end.' Based on n=2 successes.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "Six of seven authors are from Microsoft Research, and AGDebugger is built on Microsoft's AutoGen framework using Microsoft's Magentic-One agent team. The user study evaluates the tool's helpfulness without independent evaluation. This conflict is not disclosed or discussed."
     },
     {
       "flag": "Very small sample sizes",
       "detail": "Formative interviews n=5, Part 1 n=6, Part 2 n=8. All recruited from within one company (Microsoft). No statistical tests are applied to any comparisons. The 2/8 success rate makes claims about steering effectiveness hard to support."
     },
     {
       "flag": "Convenience sampling from single organization",
       "detail": "All participants across both studies were recruited from Microsoft, where AutoGen was developed. This creates potential bias — participants may be more familiar with and favorable toward the underlying framework."
     },
     {
       "flag": "Overclaiming from qualitative evidence",
       "detail": "The abstract claims AGDebugger 'effectively supported participants in iteratively steering the agents towards correct behavior,' but only 2/8 succeeded at the steering task. The tool helped understanding more than outcomes."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Core multi-agent framework that AGDebugger extends; foundational work on LLM-powered agent collaboration."
     },
     {
       "title": "Magentic-One: A Generalist Multi-Agent System for Solving Complex Tasks",
       "authors": ["Adam Fourney", "Gagan Bansal", "Hussein Mozannar"],
       "year": 2024,
       "relevance": "The specific 5-agent team used in the user study; demonstrates state-of-art multi-agent performance on GAIA."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["Gregoire Mialon", "Thomas Scialom", "Clémentine Fourrier"],
       "year": 2024,
       "relevance": "Benchmark used to source debugging tasks; standardized evaluation of AI agent capabilities."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major coding agent benchmark requiring multi-step reasoning and tool use."
     },
     {
       "title": "OpenDevin: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Competing platform for multi-agent AI development; compared as related work."
     },
     {
       "title": "AutoGen Studio: A No-Code Developer Tool for Building and Debugging Multi-Agent Systems",
       "authors": ["Victor Dibia", "Jingya Chen", "Gagan Bansal"],
       "year": 2024,
       "arxiv_id": "2408.15247",
       "relevance": "Prior agent debugging interface from same team; AGDebugger extends its debugging capabilities."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Language Model Society",
       "authors": ["Guohao Li", "Hasan Abed Al Kader Hammoud"],
       "year": 2023,
       "arxiv_id": "2303.17760",
       "relevance": "Foundational multi-agent framework using role-playing for LLM collaboration."
     },
     {
       "title": "ChainForge: A Visual Toolkit for Prompt Engineering and LLM Hypothesis Testing",
       "authors": ["Ian Arawjo", "Chelse Swoopes", "Priyan Vaithilingam"],
       "year": 2023,
       "arxiv_id": "2309.09128",
       "relevance": "Visual tool for debugging LLM pipelines; closest prior work on LLM debugging interfaces."
     },
     {
       "title": "PromptChainer: Chaining Large Language Model Prompts through Visual Programming",
       "authors": ["Tongshuang Wu", "Ellen Jiang", "Aaron Donsbach"],
       "year": 2022,
       "relevance": "Visual programming for LLM chains; identified cascading error problem that AGDebugger addresses."
     },
     {
       "title": "Software engineering for machine learning: a case study",
       "authors": ["Saleema Amershi", "Andrew Begel", "Christian Bird"],
       "year": 2019,
       "doi": "10.1109/ICSE-SEIP.2019.00042",
       "relevance": "Foundational SE4ML study on challenges in developing ML systems; motivates need for debugging tools."
     },
     {
       "title": "WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models",
       "authors": ["Hongliang He", "Wenlin Yao", "Kaixin Ma"],
       "year": 2024,
       "relevance": "Web browsing agent evaluated on complex web tasks; demonstrates the type of agent behavior AGDebugger debugs."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2024,
       "relevance": "Documents LLM attention limitations over long contexts, directly relevant to why agent steering is difficult."
     }
   ]
 }