ai-research-survey

scan.json (27931B)

---

 {
   "paper": {
     "title": "AnimAgents: Coordinating Multi-Stage Animation Pre-Production with Human–Multi-Agent Collaboration",
     "authors": [
       "Wen-Fan Wang",
       "Chien-Ting Lu",
       "Jin Ping Ng",
       "Yi-Ting Chiu",
       "Ting-Ying Lee",
       "Miaosen Wang",
       "Bing-Yu Chen",
       "Xiang 'Anthony' Chen"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.17906"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. The system is described in detail but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset, interview transcripts, survey responses, or interaction logs are released. The paper describes collecting qualitative and quantitative data from participants but does not provide any download links."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the system is built on AutoGen Core 0.6.1, uses FastAPI and Next.js, GPT-4.1-mini, ChromaDB, and Flux.1-Kontext-pro (Section 5.4), but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The system architecture is described but there are no instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as Likert-scale distributions with p-values from Wilcoxon signed-rank tests, but no confidence intervals or error bars are provided for the main survey results in Figures 9 and 10."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Wilcoxon signed-rank tests throughout (Section 6.1.4), reporting p-values (p < .01 and p < .05) for comparisons between AnimAgents and the baseline. A paired t-test is also used for task completion time (Section 7.1.3). One-sample Wilcoxon signed-rank test is used for the open-ended task (Section 6.1.4)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports p-values and percentage preferences but does not report standardized effect sizes (e.g., Cohen's d, r, or rank-biserial correlation). The 13% reduction in task time (Section 7.1.3) provides some baseline context but is not a standardized effect size measure."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The summative study uses N=16 and the field study uses N=4. No power analysis or justification for these sample sizes is provided. The paper does not acknowledge whether N=16 is sufficient for the statistical claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports mean ± standard deviation for interaction metrics (e.g., 'Total messages 22.00±7.47' for Baseline, '27.12±7.96' for AnimAgents). Task completion time is also reported with means (Section 7.1.3)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A single-agent baseline is included that uses the same AutoGen framework, LLM, and T2I model. It is described as 'intentionally stronger than typical industry workflows' (Section 6). The open-ended task also compares against participants' prior GenAI workflows."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline uses the same contemporary models (GPT-4.1-mini, Flux.1-Kontext-pro) and framework (AutoGen Core 0.6.1). The paper acknowledges no other MAS supports tailored human-AI collaboration for animation pre-production, justifying why no MAS baseline is included (Section 6)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The system has multiple components (Core Agent, Specialized Agents, stage-specific boards, block-based interactions) but none are individually removed to measure their contribution. The comparison is only against a single-agent baseline, not ablated versions of AnimAgents."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: 17 Likert-scale survey items across three research dimensions (Figures 9 and 10), task completion time, interaction log analysis with four message categories (Table 3), and qualitative interview data."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire evaluation is human-centered: 16 participants rated both systems on Likert scales, provided qualitative feedback in interviews, and 4 participants conducted a week-long field study with diary documentation and follow-up interviews."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study evaluating a creative tool, not a benchmark evaluation with train/test splits. The concept of held-out test sets does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across three research dimensions (RQ1: Coordination & Continuity, RQ2: Organization & Traceability, RQ3: Agency & Creative Exploration) with per-question breakdowns in Figures 9 and 10 (17 individual questions), and per-category interaction analysis in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple failure cases are discussed: Core Agent misrouting tasks (Section 7.3.1, 9.1.2), stage rigidity issues (Section 7.1.1, P20, P23), divergence from scripts for long text inputs (Section 7.1.2, P15), memory retrieval failures (Section 9.1.3), and limitations in original animation contexts (Section 8.2.1, P24)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several non-significant results are reported: Q12 (full creative control) p = .06, Q11 (convergence on useful directions) p = .10 (Section 7.3). The field study reveals AnimAgents is less suitable for original animation with high narrative depth requirements (Section 8.2.1). Revisions decreased (Table 3), which the paper interprets positively but represents a trade-off."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AnimAgents 'significantly outperformed a strong single-agent baseline' in 'coordination, consistency, information management, and overall satisfaction (p < .01),' which is supported by the Wilcoxon signed-rank test results in Figure 9 (Q1-Q5, Q6-Q7, Q15-Q16). The field deployment claim is supported by Section 8."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a within-subjects experimental design (Section 6.1) with counterbalanced system order and topics, which is an adequate design for causal inference about the system's effects. The causal claims (e.g., 'AnimAgents significantly improved participants' ability to coordinate') are supported by this controlled comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract present AnimAgents broadly as a system for 'animation pre-production' without bounding generalization. The study involves only 16 participants from Taiwan's animation community. The paper's conclusions in Section 10 claim improvements in 'coordination, information management, and creative agency' without explicitly bounding these to the tested population, culture, or project types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.1 discusses alternative explanations including: stage rigidity as a limitation (Section 9.1.1), LLM instability creating errors that could confound results (Section 9.1.2), memory retrieval failures affecting performance (Section 9.1.3). The discussion also considers that single-agent systems may be more flexible for unexpected changes (Section 9.1.1). Section 9.5 notes reliance on self-report as a limitation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'GPT-4.1-mini' and 'GPT-4.1-nano' (Section 5.4.1) with a footnote linking to the model page, but these are marketing names without a snapshot date or API version. 'Flux.1-Kontext-pro' is similarly specified only by product name."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the Core Agent's system prompt contents at a high level (Section 5.2.1: 'base system prompt defining its high-level responsibilities, stage definitions, management rules, delegation logic, and user interaction guidelines') but does not provide the actual prompt text. Stage-specific prompts and Specialized Agent prompts are described functionally but not reproduced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported: no temperature, top-p, max tokens, or other LLM API settings are mentioned anywhere in the paper."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in considerable detail: Section 5.2 describes the Core Agent and Specialized Agent architecture, task delegation flow, result validation, parallel execution, and direct communication modes. Section 5.4 covers memory management (indexing-retrieval-reading), error handling with CancellationTokens, and the image generation pipeline. Figure 4 provides a system architecture diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "For the formative study, thematic analysis is described (Section 4.3). For the summative study, message classification into four categories is documented (Section 7.4). Interview transcription and coding procedures are described (Section 6.1.4). The survey instrument design process and question derivation from research questions are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.5 is titled 'Limitation and Future Work' and discusses study design limitations, centralized orchestration issues, cross-board relation challenges, and potential for role-play. Additional limitations are discussed throughout Section 9 (Sections 9.1-9.4)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: self-report methodology without external expert reviews (Section 9.5), centralized orchestration causing misdirected tasks, lack of inter-agent communication, cross-board lineage becoming challenging with many blocks, and the specific issue that only small short-term deployments were tested while animation pre-production spans weeks or months (Section 9.3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses limitations, it does not explicitly state what the results do NOT show or what populations/settings are excluded. It acknowledges 'large-scale adoption remains untested' (Section 9.3) but does not systematically bound the scope of its claims regarding cultural context, project types, team sizes, or participant experience levels."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available: survey responses, interview transcripts, interaction logs, and task outputs are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection procedures are described for all three studies: formative study interviews (1.5-2 hours, Section 4.2), summative study procedure (3 hours including tutorials, tasks, questionnaires, and interviews, Section 6.1.2), and field study (week-long deployment with diaries and 60-minute interviews, Section 8.1)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Recruitment methods are described: formative study recruited 'through personal referrals in the local animation community and direct email to studios' (Section 4.1). For the summative study, 5 participants from the formative study returned, and others were recruited (Section 6.1.1). Field study participants included 2 summative study volunteers and 2 recruited through 'direct outreach, including a system demo and interview' (Section 8.1)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The analysis pipeline is documented: formative study used thematic analysis with initial coding by one author refined with co-authors (Section 4.3). Summative study used Wilcoxon signed-rank tests for Likert data (Section 6.1.4). Interaction logs were classified into four categories (Section 7.4). Interview coding was led by first author with animation studio experience and refined with two researchers (Section 6.1.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 'Acknowledgments' states: 'This work was supported by the National Science and Technology Council' (Section 10/Acknowledgments)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed: seven authors from National Taiwan University, one from Google DeepMind, and one from UCLA. These are listed on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Science and Technology Council is a government funding body that has no financial interest in the outcome of the research on animation pre-production tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. One author is affiliated with Google DeepMind, and the system uses GPT-4.1 (OpenAI) — no statement about whether any authors hold patents, equity, or other financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a human-AI creative tool through user studies, not a pre-trained model's capability on a benchmark. The LLMs are used as components of the system, not evaluated for their knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model knowledge is conducted. The study evaluates a collaborative system through user studies, so train/test overlap is not relevant."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. The study uses user studies with creative tasks, not standardized benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration link (OSF, AsPredicted, etc.) is mentioned for any of the three studies."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for any of the three studies involving human participants (formative study with 12, summative study with 16, field study with 4)."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides detailed demographics: age, years of experience, industry, role (creative director vs. independent animator), and GenAI tools used for all 24 participants across all three studies."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The paper states participants were 'professional animation creators' but does not specify formal inclusion or exclusion criteria. No screening process is described beyond being in the animation field."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "The summative study uses a within-subjects design where 'system order and topics were counterbalanced' (Section 6.1.2), which describes the randomization/counterbalancing procedure for the controlled experiment."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Participants could clearly see which system they were using (AnimAgents has a multi-board interface vs. the single-board baseline), and no mention is made of whether this was considered or how it might affect results."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is provided. The paper does not state whether all recruited participants completed all tasks or whether any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or per-session costs are reported, despite the system making multiple calls to GPT-4.1-mini, GPT-4.1-nano, and Flux.1-Kontext-pro per user session."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications for running the system are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "AnimAgents significantly outperformed the single-agent baseline in coordination, consistency, information management, and overall satisfaction (all p < .01).",
       "evidence": "Within-subjects study with 16 professional creators; Wilcoxon signed-rank tests on 7-point Likert scale responses shown in Figure 9 (Q1-Q5 for coordination, Q6-Q9 for organization, Q15-Q16 for satisfaction). Most comparisons reach p < .01, some at p < .05.",
       "supported": "moderate"
     },
     {
       "claim": "AnimAgents reduced task completion time by approximately 13% compared to the baseline (Mean 34.4 vs 39.6).",
       "evidence": "Paired t-test on task completion times for 16 participants in the within-subjects study (Section 7.1.3), p < .01.",
       "supported": "moderate"
     },
     {
       "claim": "AnimAgents shifted interaction patterns toward higher-level control, with more directive messages (22% vs 10%) and fewer revisions (30% vs 40%).",
       "evidence": "Log analysis of user interactions classified into four categories (Table 3, Section 7.4). Directive messages increased for 13/16 participants.",
       "supported": "moderate"
     },
     {
       "claim": "In the open-ended task, 100% of participants preferred AnimAgents for cross-stage coordination, 94% for enhancing overall creative process, and 94% for overall satisfaction.",
       "evidence": "One-sample Wilcoxon signed-rank test against neutral midpoint on 7-point preference Likert scale (Figure 10, Section 6.1.4). All 16 participants rated the open-ended task.",
       "supported": "moderate"
     },
     {
       "claim": "AnimAgents' value diverges by context: increasing efficiency in commercial studios but facing limitations in original works.",
       "evidence": "Field study with 4 participants (Section 8.2). P5 (commercial) valued speed; P24 (original) found narrative depth and visual quality insufficient. Qualitative evidence only from 4 participants.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "case-study",
     "qualitative"
   ],
   "key_findings": "AnimAgents, a human-multi-agent collaborative system for animation pre-production, significantly outperformed a single-agent baseline in a within-subjects study with 16 professional creators on measures of coordination, consistency, information management, and overall satisfaction (p < .01), while reducing task completion time by approximately 13%. Interaction log analysis showed users shifted toward more directive, high-level control and fewer corrective revisions with AnimAgents. A week-long field study with 4 creators revealed context-dependent value: the system increased efficiency in commercial projects but was insufficient for original animation requiring high narrative depth and visual rigor.",
   "red_flags": [
     {
       "flag": "Small sample size for statistical claims",
       "detail": "The summative study has N=16 and the field study has N=4. No power analysis justifies these sizes. With 16 participants, the Wilcoxon signed-rank test has limited power, and the many comparisons (17 questions) increase the risk of Type I errors with no correction for multiple comparisons mentioned."
     },
     {
       "flag": "Self-report only evaluation",
       "detail": "The evaluation relies entirely on participant self-reports via Likert scales and interviews. No external expert evaluation of output quality, no blind evaluation of creative artifacts, and no objective measures of coordination quality are included. The paper acknowledges this limitation in Section 9.5."
     },
     {
       "flag": "Recruitment from local network",
       "detail": "Participants were recruited through 'personal referrals in the local animation community' and direct studio outreach, all from Taiwan. This convenience sampling may introduce selection bias and limits generalizability to other cultural contexts and animation industries."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "The paper reports 17 separate statistical tests on Likert-scale items (Figure 9) without applying any correction for multiple comparisons (e.g., Bonferroni, Holm, FDR). This inflates the familywise error rate substantially."
     },
     {
       "flag": "Overlap between formative and summative participants",
       "detail": "5 of the 16 summative study participants (P8-P12) had previously participated in the formative study that informed the system's design, potentially biasing their evaluation toward the system since their feedback shaped it."
     }
   ],
   "cited_papers": [
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework for collaborative software development, directly relevant to survey scope on agentic AI systems."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Agentic system for automated software engineering tasks, core paper in agentic AI for coding."
     },
     {
       "title": "MapCoder: Multi-Agent Code Generation for Competitive Problem Solving",
       "authors": ["Md. Ashraful Islam", "Mohammed Eunus Ali", "Md Rizwan Parvez"],
       "year": 2024,
       "relevance": "Multi-agent approach to code generation, relevant to survey scope on LLM-based multi-agent systems for programming."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Multi-agent conversation framework used as the backbone for AnimAgents, foundational work in agentic AI."
     },
     {
       "title": "CodeAgent: Enhancing Code Generation with Tool-Integrated Agent Systems for Real-World Repo-level Coding Challenges",
       "authors": ["Kechi Zhang", "Jia Li", "Ge Li"],
       "year": 2024,
       "doi": "10.18653/v1/2024.acl-long.737",
       "relevance": "Tool-integrated agent system for code generation, relevant to survey scope on agentic AI for software engineering."
     },
     {
       "title": "Agent Laboratory: Using LLM agents as research assistants",
       "authors": ["Samuel Schmidgall", "Yusheng Su", "Ze Wang"],
       "year": 2025,
       "relevance": "LLM multi-agent system for scientific research tasks, relevant to survey scope on agentic AI capabilities."
     },
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM-based autonomous agents, relevant to the survey's coverage of agentic AI systems."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Zhiheng Xi", "Wenxiang Chen", "Xin Guo"],
       "year": 2025,
       "relevance": "Survey on LLM-based agents covering capabilities and limitations, relevant to the broader survey scope."
     },
     {
       "title": "Multi-agent collaboration mechanisms: A survey of LLMs",
       "authors": ["Khanh-Tung Tran", "Dung Dao", "Minh-Duong Nguyen"],
       "year": 2025,
       "relevance": "Survey of multi-agent collaboration mechanisms for LLMs, directly relevant to understanding agentic AI coordination."
     },
     {
       "title": "MAGIS: LLM-Based Multi-Agent Framework for GitHub Issue Resolution",
       "authors": ["Wei Tao", "Yucheng Zhou", "Yanlin Wang"],
       "year": 2024,
       "relevance": "Multi-agent framework for software engineering (GitHub issue resolution), relevant to agentic AI for coding."
     },
     {
       "title": "Generative AI at work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey Raymond"],
       "year": 2025,
       "relevance": "Empirical study of generative AI productivity impacts in the workplace, relevant to the survey's coverage of AI productivity research."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "relevance": "Analysis of failure modes in multi-agent LLM systems, relevant to understanding reliability and limitations of agentic AI."
     }
   ]
 }