ai-research-survey

scan.json (35019B)

---

 {
   "paper": {
     "title": "MagicAgent: Towards Generalized Agent Planning",
     "authors": [
       "Xuhui Ren",
       "Shaokang Dong",
       "Chen Yang",
       "Qing Gao",
       "Yunbin Zhao",
       "Yongsheng Liu",
       "Xinwei Geng",
       "Xiang Li",
       "Demei Yan",
       "Yanqing Li",
       "Chenhao Huang",
       "Dingwei Zhu",
       "Junjie Ye",
       "Boxuan Yue",
       "Yingnan Fu",
       "Mengzhe Lv",
       "Zezeng Feng",
       "Boshen Zhou",
       "Bocheng Wang",
       "Xuanjing Huang",
       "Yu-Gang Jiang",
       "Tao Gui",
       "Qi Zhang",
       "Yunke Zhang"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.19000"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MagicAgent introduces a lightweight synthetic data framework for diverse planning tasks and a two-stage SFT+RL training paradigm to address multi-task interference. MagicAgent-32B and MagicAgent-30B-A3B achieve strong results across five public benchmarks (Worfbench, NaturalPlan, τ2-Bench, BFCL-v3, ACEBench), outperforming many sub-100B and several ultra-scale models. The proposed χPO online RL algorithm matches or exceeds prior methods on ALFWorld with a smaller model (Qwen2.5-3B). A global load-balancing strategy with z-loss stabilizes MoE training for heterogeneous agent tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The MagicAgent series models will be released soon' (footnote, page 1). This is a promise of future release, not an actual release. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The synthetic training data pipeline is described in detail (Sections 3.1-3.5) but the actual generated training data is not released. The in-house MagicEval benchmarks are proprietary. Evaluation uses public benchmarks (NaturalPlan, Worfbench, etc.) but the training data is not shared."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 8 lists training hyperparameters (optimizer, learning rate, bf16 precision) and mentions 32 GPUs (4 machines × 8 GPUs) with NVIDIA A800s for inference (Table 7). However, no requirements.txt, Dockerfile, or detailed software dependency list is provided — not enough detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts to replicate experiments. The paper describes the methodology but does not provide step-by-step instructions for reproducing results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 2, 3, and 4 report point estimates only (e.g., '80.3%', '69.7%') with no confidence intervals or error bars. Even the ALFWorld results (Table 5), which use 5 seeds, only report Succ.* (best) and Succ. (average) without CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported anywhere in the paper. Claims like 'substantially outperforming' and 'surpassing GPT-5.2 by approximately 19.5%' are based solely on comparing raw numbers without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout: e.g., '27.0% relative improvement over the base Qwen3-32B model (48.8%)' (Section 6.1.2), 'reduction of 28.9% to 59.6% in Average TTFT' (Section 6.4, Table 7). Tables provide all baseline numbers for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sizes, number of training examples, or number of experimental runs. The sizes of the synthetic training datasets are not stated, nor is there any power analysis or discussion of whether the benchmark sizes are adequate for the claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 2-4) appear to be single-run numbers with no variance reported. For ALFWorld (Table 5), they state '5 random seeds' but report only Succ.* (mean of best) and Succ. (average after convergence) without standard deviation, IQR, or any spread measure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines in Tables 2-5: ultra-scale models (GPT-5.2, Kimi-K2, DeepSeek-V3.1, GLM-4.7, Qwen3-235B, Qwen3-MAX) and large-scale models (Qwen3-32B, Llama3.3-70B, ERNIE-4.5, Olmo-3.1, GLM-4.7-Flash). For ALFWorld, additional RL baselines (AgentGym, SFT, GRPO, GiGPO, RLVMR, EPO)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-5.2 (2025), DeepSeek-V3.1, Kimi-K2, GLM-4.7, Qwen3 series — all state-of-the-art models contemporary with the paper's 2026 publication date."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 presents ablation of MoE optimization strategies (z-loss, global-batch LBL, micro-batch LBL, variance loss, expert capacity). Section 6.3 compares χPO components. Figure 8 analyzes entropy dynamics. Both dense and MoE variants are compared."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics across benchmarks: F1 Chain and F1 Graph (Worfbench), accuracy per subtask (NaturalPlan: Trip/Meeting/Calendar), domain-specific accuracy (τ2-Bench: Retail/Airline), Live/Non-Live accuracy (BFCL-v3), English/Chinese accuracy (ACEBench), plus Step/Embedding/LLM metrics for MagicEval-Plan and Tool Name/Argument accuracy for MagicEval-Tool."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated using benchmark metrics and automated judges (GPT-5.2 as LLM-as-a-Judge for MagicEval-Plan). No human evaluation of model outputs is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluations follow official benchmark protocols. ALFWorld explicitly uses separate 'seen' and 'unseen' validation splits (Section 6.3). Public benchmarks (NaturalPlan, Worfbench, BFCL-v3, ACEBench, τ2-Bench) have their own test splits, and the paper states 'we strictly followed the official instructions' (Section 6.1.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed breakdowns: NaturalPlan by Trip/Meeting/Calendar, τ2-Bench by Retail/Airline, BFCL-v3 by Live/Non-Live, ACEBench by English/Chinese, MagicEval by General/Dependency/Condition/Context Inheritance. Tables 2-4 provide per-subtask numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure case analysis or error analysis is presented. The paper discusses training dynamics (Figure 8) and ablation results but never examines what types of planning tasks MagicAgent fails on or why."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 shows ablation configurations that fail to maintain balanced performance: 'variance loss and sequence-level LBL exhibit suboptimal performance on τ2-Bench: retail and airline scores are mostly below 45' (Section 6.4). The paper explicitly notes that 'most mainstream training tricks fail to maintain balanced performance.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (75.1% Worfbench, 55.9% NaturalPlan, 57.5% τ2-Bench, 86.9% BFCL-v3, 81.2% ACEBench) match the numbers derived from Table 2 (e.g., Worfbench: average of F1 Chain 80.3 and F1 Graph 69.7 ≈ 75.0; NaturalPlan: average of Trip/Meeting/Calendar). Claims of outperforming sub-100B models are supported by Table 2 comparisons."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about training components ('our planning-centric training can effectively enhance the performance'). Table 6 provides controlled ablation studies (single-variable manipulation of training strategies with identical data/hyperparameters), which constitutes adequate causal justification for component-level claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Towards Generalized Agent Planning' but results are limited to five specific benchmarks. The abstract claims models 'substantially outperform existing sub-100B models' broadly, without bounding this to the tested benchmarks. No discussion of what planning domains or task types were NOT tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. The improvements could be due to the larger synthetic training data volume rather than the proposed training methodology, or due to data similarity to benchmark formats. No threats-to-validity or confound analysis is provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures accuracy on specific benchmarks (Worfbench F1, NaturalPlan accuracy, etc.) but frames results as 'generalized agent planning' capability. The gap between benchmark performance and actual generalized planning ability is not discussed. No acknowledgment that benchmark accuracy may not reflect real-world planning effectiveness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names without version snapshots or API versions: 'GPT-5.2', 'Kimi-K2-Instruct', 'DeepSeek-V3.1', 'GLM-4.7', 'Qwen3-32B', 'Qwen3-30B-A3B-Instruct-2507'. While some model names include date suffixes (e.g., '-2507'), exact model snapshot IDs or API versions are not provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "System prompts and training prompts are described in natural language (e.g., 'heterogeneous, task-specific prompts to further distinguish tasks') but never shown in full. Appendices C-G show data format examples, not the actual prompts used for data generation or model inference."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 provides comprehensive training hyperparameters: learning rates (1e-5 SFT, 1e-6 RL), optimizer settings (Adam β1=0.9, β2=0.95), generation parameters (temperature=0.9, top_p=0.96, top_k=20), batch sizes, parallelism configuration, MoE loss coefficients, and more."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent data synthesis pipeline is described in detail: Reasoning Agent, Planner, Simulator Agent roles (Section 3.2, Figure 3). The ReAct paradigm for inference is described (Section 3.2). State transition mechanisms and workflow are documented with figures (Figures 2-6)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: SimHash deduplication (Section 3.1), schema validation enforcing strict structural format, semantic verification via GPT-5.2, negative sampling with distractor tools (Section 3.2), rejection sampling for constraint scheduling (Section 3.3), DAG verification (Section 3.4), NovelSum metric for sample selection (Section 4.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists in the paper. The conclusion (Section 7) mentions future work ('advancing long-horizon reasoning and code-augmented task completion') but does not discuss limitations of the current approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no consideration of specific methodological limitations, potential confounds, or scenarios where the approach might fail."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what planning tasks, domains, or scenarios were NOT tested, nor what claims the authors are NOT making. The broad title ('Generalized Agent Planning') is not bounded by explicit caveats."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw training data, synthetic datasets, or MagicEval benchmark data is released. Only processed results are shown in tables. The synthetic data generation process is described but the actual data is not available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.5 describe data collection in detail: 5,000+ tools aggregated from open-source APIs and internal repository (Section 3.1), tool dependency graphs constructed, atomic plan synthesis pipeline, constraint satisfaction framework for scheduling data (Section 3.3), and long-horizon trajectory generation process (Section 3.5)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For MagicEval benchmarks, the paper states data was collected from 'actual user interaction logs from our production environment' (Section 6.2.1) with PII removed, but does not describe what users, how many, what time period, or whether this introduces selection bias. For training data, the sampling of 5,000+ tools is described only as 'from open-source APIs and our internal repository' without detailing the selection process."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described (tool curation → graph construction → atomic synthesis → trajectory composition → filtering), but counts at each filtering stage are not provided. The paper does not state how many examples were generated, how many passed each filter, or how many were retained in the final training set."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section, acknowledgments, or grant information is provided in the paper. The work appears to be funded by Honor Device Co. (corporate employer of 16 of 24 authors) but this is not explicitly disclosed as funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Honor Device Co., Ltd (superscript 1) and Fudan University (superscript 2). The corporate affiliation is transparent on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Honor Device Co. is both the employer of the majority of authors and the company deploying MagicAgent in 'Honor's intelligent assistant' (Section 7). The company has a direct financial interest in demonstrating superior performance of their model. This is a clear non-independent funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. No patent disclosures or equity declarations despite the model being commercially deployed by Honor."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The base model is described as 'Qwen3 series' (Section 4.1) but no training data cutoff date is stated for either the Qwen3 base model or the synthetic training data. It is unknown whether the base model's training data includes benchmark examples."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any of the evaluation benchmarks (NaturalPlan, Worfbench, BFCL-v3, ACEBench, τ2-Bench) or their constituent problems appeared in the Qwen3 base model's training data or in the synthetic training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation benchmarks are publicly available and predate the paper. No contamination analysis is performed. For example, NaturalPlan (2024), Worfbench (2025), and ALFWorld (2020) could plausibly be in Qwen3's training data, but this is never addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve model evaluation on automated benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The MagicEval data uses anonymized production logs but this is not a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in experiments."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in experiments."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in experiments."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in experiments."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in experiments."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 reports inference efficiency metrics: Average TTFT (time to first token), Average TPOT (time per output token), and Average Latency across concurrency levels 1/10/50/100, comparing dense vs MoE models on 4× NVIDIA A800 GPUs with 1024 input/output tokens."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table 8 mentions 32 GPUs (4 machines × 8 GPUs) and A800 GPUs for inference, but total GPU hours, training wall-clock time, and total computational cost are not stated. The training budget for the synthetic data generation (LLM calls to GPT-5.2 for verification) is also unstated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "For ALFWorld (Section 6.3), experiments use 5 random seeds and report Succ.* and Succ. metrics. However, main benchmark results (Tables 2-4) do not report any seed sensitivity. It is unclear whether main results are single-run or averaged, and no standard deviations are provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld experiments explicitly state '5 random seeds' (Section 6.3). However, for the main benchmark evaluations (Tables 2-4), the number of runs is never stated. It is unclear whether results are from a single run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Table 8 lists final hyperparameters but does not describe any search process. No information on how many configurations were tried, what search method was used, or total compute spent on hyperparameter tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final training configuration is presented without justification for how it was selected. The MoE ablation (Table 6) compares strategies but doesn't describe how the final joint configuration was arrived at or validated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is possible. The paper makes many comparisons across models, benchmarks, and conditions without any statistical testing framework."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors (Honor employees) evaluate their own MagicAgent system against baselines using their own evaluation code and in-house benchmarks. No acknowledgment of author-evaluation bias or self-comparison bias. Baseline results appear to use the authors' own evaluation runs."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MagicAgent-32B is compared against ultra-scale models like GPT-5.2 and Qwen3-235B without discussing compute budget differences. While Table 7 compares dense vs MoE inference efficiency, no performance-vs-compute analysis is provided for the main results."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the selected benchmarks actually measure 'generalized agent planning' as claimed. The paper does not question whether Worfbench F1, NaturalPlan accuracy, or BFCL-v3 scores are valid proxies for the claimed capability of generalized planning."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper uses official evaluation protocols for all benchmarks, it does not explicitly discuss scaffolding as a variable. For τ2-Bench, it notes that GLM uses 'an optimized user simulator' but does not analyze whether different evaluation setups affect comparability. The scaffold confound is not treated as an explicit factor."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The Qwen3 base model's training data likely includes published benchmarks (ALFWorld 2020, NaturalPlan 2024, Worfbench 2025), but the temporal relationship between training data and benchmarks is never analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The paper does not analyze whether the synthetic training data generation process (which uses benchmark-style task formulations) could leak structural information about the evaluation benchmarks."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test data share structural similarities. The synthetic data uses similar task types (tool calling, scheduling, workflow orchestration) to the evaluation benchmarks, creating a potential overlap in task distributions, but this is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, n-gram overlap analysis, membership inference tests, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "MagicAgent-32B achieves state-of-the-art among sub-100B models with 75.1% on Worfbench, 55.9% on NaturalPlan, 57.5% on τ2-Bench, 86.9% on BFCL-v3, and 81.2% on ACEBench, surpassing several closed-source ultra-scale models.",
       "evidence": "Table 2 shows MagicAgent-32B outperforming all large-scale models and several ultra-scale models (GPT-5.2, DeepSeek-V3.1) across these benchmarks. Specific numbers: Worfbench F1 Chain 80.3%/F1 Graph 69.7%, NaturalPlan Trip 48.6%/Meeting 57.7%/Calendar 61.5%.",
       "supported": "moderate"
     },
     {
       "claim": "The two-stage SFT+RL training paradigm mitigates the 'seesaw effect' in multi-task learning, enabling synergistic utilization of heterogeneous planning data.",
       "evidence": "Section 4 describes the paradigm conceptually. Table 6 ablates MoE optimization strategies, showing that joint optimization (LBLgbl + z_loss) achieves the best balanced performance. However, no direct measurement of the seesaw effect (e.g., showing per-task performance degradation without the proposed method) is provided.",
       "supported": "weak"
     },
     {
       "claim": "χPO achieves state-of-the-art on ALFWorld, matching or exceeding larger models with Qwen2.5-3B: 91.7% IID Succ.*, 91.8% OOD Succ.*, 78.2% IID Succ., 75.9% OOD Succ.",
       "evidence": "Table 5 shows χPO with Qwen2.5-3B matching EPO's IID Succ.* (91.7%) while exceeding it on OOD Succ.* (91.8% vs 89.6%), IID Succ. (78.2% vs 75.8%), and OOD Succ. (75.9% vs 75.4%). Results are averaged over 5 random seeds.",
       "supported": "moderate"
     },
     {
       "claim": "Global batch load balancing combined with z-loss achieves the best balanced MoE training for heterogeneous agent tasks.",
       "evidence": "Table 6 compares seven strategies. BP + LBLgbl + z_loss achieves the best τ2-Bench performance (52.6% retail, 52.5% airline) with competitive Worfbench/NaturalPlan scores. Figure 9 shows MaxVio stabilization. Figure 10 shows task-specific expert specialization.",
       "supported": "moderate"
     },
     {
       "claim": "MagicAgent achieves 98.0% step accuracy, 97.7% tool name accuracy, and 87.3% tool argument accuracy on in-house MagicEval benchmarks.",
       "evidence": "Tables 3 and 4 show these results under the General scenario. MagicAgent-32B substantially outperforms all baselines across all MagicEval settings (General, Dependency, Condition, Context Inheritance).",
       "supported": "weak"
     },
     {
       "claim": "The MoE variant (MagicAgent-30B-A3B) reduces inference latency by up to 54.7% compared to the dense model (MagicAgent-32B) while maintaining competitive performance.",
       "evidence": "Table 7 shows latency reductions across concurrency levels: 54.7% at concurrency=1, 39.9% at 10, 35.8% at 50, 40.6% at 100. Table 2 shows MagicAgent-30B-A3B achieves comparable benchmark scores to MagicAgent-32B.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Honor Device Co. employees (16 of 24 authors) develop and evaluate MagicAgent, which is deployed in 'Honor's intelligent assistant' (Section 7). The company has a direct commercial interest in demonstrating superior performance. No conflicts of interest statement is provided."
     },
     {
       "flag": "No error bars on main results",
       "detail": "Tables 2, 3, and 4 report point estimates only with no uncertainty quantification. It is unclear whether results are single-run or averaged. Without variance information, it is impossible to assess whether performance differences are meaningful."
     },
     {
       "flag": "Proprietary in-house benchmarks",
       "detail": "MagicEval-Plan and MagicEval-Tool benchmarks (Tables 3-4) are proprietary and cannot be independently verified or reproduced. The paper shows near-perfect scores (98%) on these benchmarks while competitors score much lower, but external validation is impossible."
     },
     {
       "flag": "Models and data not released",
       "detail": "Both the trained models and synthetic training data are not released. The paper only promises future model release ('will be released soon'). Without access to models or data, no claims can be independently verified."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations section, threats to validity, or discussion of scenarios where the approach might fail. For a paper making broad claims about 'generalized agent planning,' the absence of scope boundaries is concerning."
     },
     {
       "flag": "Missing contamination analysis",
       "detail": "The Qwen3 base model's training data likely includes several evaluation benchmarks (ALFWorld published 2020, NaturalPlan 2024). The synthetic training data uses similar task formats to the benchmarks. No decontamination or overlap analysis is performed."
     },
     {
       "flag": "Cherry-picked baseline comparisons",
       "detail": "Several baseline entries in Table 2 show suspiciously poor performance (DeepSeek-V3.1-nothink at 1.4% on NaturalPlan Trip, Qwen3-MAX at 1.2%) suggesting the 'nothink' mode may be disadvantageous. Baselines are run with thinking disabled while MagicAgent may benefit from its specialized training."
     }
   ],
   "cited_papers": [
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R Narasimhan", "Yuan Cao"],
       "year": 2022,
       "relevance": "Foundational agent framework (Reasoning + Acting) used as the inference paradigm in MagicAgent and as a baseline in ALFWorld experiments."
     },
     {
       "title": "ToolLLM: Facilitating large language models to master 16000+ real-world APIs",
       "authors": ["Yujia Qin", "Shihao Liang", "Yining Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Large-scale tool-learning dataset using real-world APIs with depth-first search trajectory collection; directly relevant to tool-augmented planning."
     },
     {
       "title": "AgentTuning: Enabling generalized agent abilities for LLMs",
       "authors": ["Aohan Zeng", "Mingdao Liu", "Rui Lu"],
       "year": 2024,
       "relevance": "SFT-based approach for adapting LLMs to agentic tasks; a key prior work in agent model fine-tuning."
     },
     {
       "title": "Agent-FLAN: Designing data and methods of effective agent tuning for large language models",
       "authors": ["Zehui Chen", "Kuikun Liu", "Qiuchen Wang"],
       "year": 2024,
       "arxiv_id": "2403.12881",
       "relevance": "Agent fine-tuning methodology using curated instruction corpora and interaction trajectories."
     },
     {
       "title": "AgentGym: Evolving large language model-based agents across diverse environments",
       "authors": ["Z. Xi", "Y. Ding", "W. Chen"],
       "year": 2024,
       "arxiv_id": "2406.04151",
       "relevance": "RL-based agent optimization framework used as a baseline in ALFWorld experiments."
     },
     {
       "title": "WebRL: Training LLM web agents via self-evolving online curriculum reinforcement learning",
       "authors": ["Zehan Qi", "Xiao Liu"],
       "year": 2024,
       "arxiv_id": "2411.02337",
       "relevance": "Self-supervised RL method for web agents; relevant to online RL training methodology."
     },
     {
       "title": "Natural plan: Benchmarking LLMs on natural language planning",
       "authors": ["Huaixiu Steven Zheng", "Swaroop Mishra", "Hugh Zhang"],
       "year": 2024,
       "arxiv_id": "2406.04520",
       "relevance": "Multi-constraint scheduling benchmark (Trip, Meeting, Calendar) used as primary evaluation in this paper."
     },
     {
       "title": "Benchmarking agentic workflow generation",
       "authors": ["Shuofei Qiao", "Runnan Fang", "Zhisong Qiu"],
       "year": 2025,
       "arxiv_id": "2410.07869",
       "relevance": "WorfBench benchmark for procedural logic orchestration; used as primary evaluation benchmark in this paper."
     },
     {
       "title": "τ2-bench: Evaluating conversational agents in a dual-control environment",
       "authors": ["Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan"],
       "year": 2025,
       "arxiv_id": "2506.07982",
       "relevance": "Long-horizon tool execution benchmark (Retail/Airline domains) used as primary evaluation in this paper."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reinforcement learning with verifiable rewards methodology; foundational approach for the RLVR design in MagicAgent."
     },
     {
       "title": "EPO: Entropy-regularized policy optimization for LLM agents reinforcement learning",
       "authors": ["Wujiang Xu", "Wentian Zhao", "Zhenting Wang"],
       "year": 2025,
       "arxiv_id": "2509.22576",
       "relevance": "Entropy-regularized RL algorithm for agents; direct competitor and inspiration for the proposed χPO algorithm. Used as primary baseline on ALFWorld."
     },
     {
       "title": "MagNet: Multi-turn tool-use data synthesis and distillation via graph translation",
       "authors": ["Fan Yin", "Zifeng Wang", "I-Hung Hsu"],
       "year": 2025,
       "arxiv_id": "2503.07826",
       "relevance": "Bidirectional translation methodology for tool-augmented data synthesis; directly used in MagicAgent's data generation pipeline."
     },
     {
       "title": "ALFWorld: Aligning text and embodied environments for interactive learning",
       "authors": ["Mohit Shridhar", "Xingdi Yuan", "Marc-Alexandre Côté"],
       "year": 2020,
       "relevance": "Interactive household task environment used for online RL evaluation of χPO algorithm."
     }
   ]
 }