ai-research-survey

scan-v5.json (29308B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "MagicAgent: Towards Generalized Agent Planning",
     "authors": [
       "Xuhui Ren",
       "Shaokang Dong",
       "Cheng Yang",
       "Qingying Gao",
       "Yunbin Zhao",
       "Yongsheng Liu",
       "Xinwei Geng",
       "Xiang Li",
       "Demei Yan",
       "Yanqing Li",
       "Chenhao Huang",
       "Dingwei Zhu",
       "Junjie Ye",
       "Boxuan Yue",
       "Yingnan Fu",
       "Mengzhe Lv",
       "Zezeng Feng",
       "Boshen Zhou",
       "Bocheng Wang",
       "Xuanjing Huang",
       "Yu-Gang Jiang",
       "Tao Gui",
       "Qi Zhang",
       "Yunke Zhang"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.19000",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the model 'even surpasses leading closed-source models,' but Table 2 shows MagicAgent-32B scoring 62.0 on τ²-Bench Retail while GPT-5.2 scores 75.2, and Kimi-K2 outperforms on several BFCL-v3 and ACEBench subtasks. The absolute accuracy numbers are supported but the sweeping 'surpass' claim is overstated.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies in Section 6.4 (Table 6) compare individual training tricks (z-loss, various LBL strategies, variance loss) against the joint optimization strategy; Section 6.3 (Table 5) compares GRPO vs χPO vs EPO under controlled settings. These ablations provide adequate causal grounding for the design choice claims.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title and framing claim 'generalized agent planning,' but evaluation is limited to 5 specific benchmark types (hierarchical decomposition, tool-augmented planning, multi-constraint scheduling, procedural logic orchestration, long-horizon tool execution) with no explicit acknowledgment that this does not cover the broader space of agentic tasks.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered, such as whether the gains come primarily from the Qwen3 base model quality rather than the training pipeline, or whether simply fine-tuning on the public benchmark data would yield similar results.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark accuracy scores (WorfBench F1, NaturalPlan accuracy) are equated with 'generalized agent planning' capability without discussing the gap between benchmark performance and real-world agentic behavior; the connection between these proxies and the claimed generalization ability is asserted rather than argued.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 7 (Conclusions) briefly mentions future work directions but does not discuss limitations of the current study.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper — no mention of benchmark contamination, in-house evaluation bias, synthetic data quality limitations, or generalizability beyond tested task types.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show; the sweeping framing around 'generalized' planning implies broader applicability than the 5 tested task types with no explicit scope caveat.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. The institutional affiliation is Honor Device Co., Ltd, a commercial entity, but no formal funding statement appears.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly disclosed: Honor Device Co., Ltd (affiliation 1) and Fudan University (affiliation 2) are listed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The majority of authors are from Honor Device Co., Ltd, and the conclusion explicitly states the model 'has been deployed across multiple high-value scenarios under Honor's intelligent assistant' — the developing company is evaluating its own commercial product.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "'Generalized planning' — the paper's central claim — is never formally defined; the paper distinguishes it from 'single-point optimization' but provides no criterion for what would or would not count as generalized. The five task types are described but the scope boundary of 'generalized' is left implicit.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists four contributions: (1) scalable synthetic data framework, (2) two-stage multi-task optimization, (3) load-balanced MoE strategy, and (4) SOTA performance results.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 reviews related work across LLM agent models, training paradigms (SFT, RL), and MoE training, positioning MagicAgent relative to AgentTuning, AgentFLAN, AgentGym, WebRL, and others, explaining how the proposed approach addresses gaps in prior work.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "The paper states 'The MagicAgent series models will be released soon' — this is a promise of future release, not an actual release.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The synthetic training data pipeline is described but the data itself is not released. The in-house MagicEval benchmark is proprietary. Public benchmarks used for evaluation are available but the paper does not release its own datasets.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Table 8 specifies training hyperparameters and GPU hardware (NVIDIA A800), but no environment specification file (requirements.txt, Dockerfile, or equivalent) is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes the methodology and refers to official codebases for benchmark evaluation, but provides no step-by-step instructions to reproduce training or inference results for MagicAgent itself.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported in any of the main results tables (Tables 2, 3, 4, 5, 6, 7). The ALFWorld experiment uses 5 random seeds but reports only mean best and mean steady-state without standard deviation.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any comparative claims despite numerous model comparisons with sometimes small differences (e.g., χPO vs EPO in Table 5).",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Percentage improvements are cited in the results discussion (e.g., '27.0% relative improvement over the base Qwen3-32B model' on τ²-Bench Retail, '19.5% and 30.8%' improvement over GPT-5.2 on WorfBench).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification or power analysis is provided for the size of any evaluation benchmark. The number of evaluation examples per benchmark is not even reported.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or spread statistics are reported for any main results. Even Table 5's ALFWorld results across 5 seeds report only best and average values without standard deviation.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Extensive baselines are included: 6 ultra-scale closed-source models (GPT-5.2, Kimi-K2, GLM-4.7, DeepSeek-V3.1, Qwen3-235B, Qwen3-MAX) and 6 large-scale open-source models (Tables 2-5).",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include very recent (2025-2026) models such as Kimi-K2 (2025), GLM-4.7 (2025), Qwen3-235B-A22B-Instruct-2507, and GPT-5.2 (December 2025), representing the current state-of-the-art.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.4 (Table 6) provides an ablation study comparing 7 individual MoE optimization strategies against the joint strategy. Section 6.3 (Table 5) compares multiple RL training algorithms (GRPO, EPO, χPO, GiGPO, RLVMR).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics are used: WorfBench uses F1 Chain and F1 Graph; NaturalPlan uses Trip, Meeting, and Calendar accuracy; τ²-Bench uses Retail and Airline; BFCL-v3 uses Live and Non-Live; ACEBench uses En and Zh; MagicEval uses Step, Embedding, and LLM metrics.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation of system outputs is conducted. MagicEval uses LLM-as-a-Judge (GPT-5.2) for qualitative assessment, which is automated rather than human evaluation.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Public benchmarks use their official test sets; ALFWorld evaluation uses seen (IID) and unseen (OOD) splits; the in-house MagicEval uses production interaction logs not used in training.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "All benchmarks are broken down by subtask: NaturalPlan (Trip/Meeting/Calendar), τ²-Bench (Retail/Airline), BFCL-v3 (Live/Non-Live), ACEBench (En/Zh), and MagicEval (General/Dependency/Condition/Context Inheritance).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No failure cases or error analysis are discussed. The paper only reports success metrics without examining where or why the model fails.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 6 reports that variance loss and sequence-level LBL perform worse than the joint optimization strategy on τ²-Bench (retail and airline scores mostly below 45%), and that BP+LBLseq degrades WorfBench performance relative to the baseline.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Baselines are listed with identifiers (e.g., Qwen3-32B-nothink, Qwen3-30B-A3B-Instruct-2507) but no snapshot dates or commit hashes are provided for training base models. The paper says 'We adopt the Qwen3 series as the base model' without specifying the exact checkpoint used.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "Appendices show data format examples (with actual content in Chinese, partially obscured) but the actual evaluation prompts and system instructions used for benchmarking are not provided.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 8 (Appendix A) provides comprehensive hyperparameters: learning rate (1e-5 SFT, 1e-6 RL), batch size, gradient accumulation, temperature (0.9), top-p (0.96), top-k (20), weight decay, scheduler type, MoE parallel settings, and reward weighting coefficients.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The agentic scaffolding is described in detail: Section 3.2 describes the ReAct-based reasoning-action-observation format; Section 4.2.3 details the think-action entropy smoothing; Table 1 specifies the unified data format for each task type.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Sections 3.1-3.5 document preprocessing and filtering for each data type: SimHash deduplication, schema validation, and semantic verification (via GPT-5.2) for hierarchical decomposition; JSON schema validation, semantic auditing, and negative sampling for tool-augmented planning.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No raw training data or evaluation data is released. Training data is synthetic and proprietary; in-house MagicEval data is from production logs and not released.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The synthetic data collection pipeline is described in substantial detail across Sections 3.1-3.5, including tool graph construction, atomic plan synthesis, filtering steps, and scaling strategies. In-house MagicEval is described as anonymized production user interaction logs.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants are involved; standard synthetic and benchmark data are used.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full synthetic data pipeline is documented end-to-end in Sections 3.1-3.5, from tool curation through graph construction, atomic synthesis, trajectory composition, and quality filtering. In-house data collection is briefly described as anonymized production logs.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The training data cutoff for the Qwen3 base model is never stated. Fine-tuning uses synthetic data but the base model's pretraining cutoff, which affects benchmark contamination risk, is not mentioned.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the synthetic training data could overlap with benchmark test cases, or whether the Qwen3 base model's pretraining included the benchmark data.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Public benchmarks (WorfBench, NaturalPlan, BFCL-v3, ACEBench) predate this paper and may have been included in Qwen3's pretraining corpus; this potential contamination is never acknowledged or addressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 7 provides detailed inference efficiency comparisons (TTFT, TPOT, average latency) between dense and MoE models at concurrency levels 1, 10, 50, and 100 on 4 NVIDIA A800 GPUs.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Table 8 states 32 GPUs (4 machines × 8 GPUs) and 1 training epoch, but total GPU-hours, wall-clock time, or FLOPs consumed for training are not reported.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "MagicAgent-32B achieves 75.1% average on WorfBench, substantially outperforming GPT-5.2 (49.6%) and all other tested sub-100B and ultra-scale models.",
       "evidence": "Table 2 shows F1 Chain 80.3% and F1 Graph 69.7% for MagicAgent-32B vs F1 Chain 60.1% and F1 Graph 39.1% for GPT-5.2. Figure 1 presents aggregated averages.",
       "supported": "strong"
     },
     {
       "claim": "The proposed χPO online RL algorithm outperforms EPO on the ALFWorld benchmark.",
       "evidence": "Table 5 shows χPO (Qwen2.5-3B) achieving IID Succ.* 91.7%/Succ. 78.2% and OOD Succ.* 91.8%/Succ. 75.9% vs EPO's 91.7%/75.8% and 89.6%/75.4%.",
       "supported": "moderate"
     },
     {
       "claim": "Joint optimization of global-batch LBL and z-loss outperforms all individual MoE training strategies across benchmarks.",
       "evidence": "Table 6 ablation shows BP+LBLgbl+z_loss achieving balanced performance across WorfBench, NaturalPlan, and τ²-Bench subtasks where individual strategies each show weaknesses.",
       "supported": "moderate"
     },
     {
       "claim": "MagicAgent-30B-A3B (MoE) achieves up to 54.7% inference latency reduction compared to MagicAgent-32B (dense) with competitive accuracy.",
       "evidence": "Table 7 reports average latency reduction of 54.7% at concurrency 1 (7.582s vs 16.722s) while benchmark scores remain within 1-2pp.",
       "supported": "strong"
     },
     {
       "claim": "The two-stage SFT+RL training mitigates the 'seesaw effect' where gains in one planning task degrade others.",
       "evidence": "Section 6.4 shows joint training outperforms naive multi-task approaches in ablation, and Figure 8b shows steady reward increases in both offline and online RL.",
       "supported": "weak"
     },
     {
       "claim": "MagicAgent substantially outperforms existing sub-100B models across all five planning benchmarks.",
       "evidence": "Tables 2-4 show MagicAgent-32B consistently above Qwen3-32B, Llama3.3-70B, and other large-scale models. E.g., WorfBench F1 Chain: 80.3% vs best competing 65.8%.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "MagicAgent-32B and MagicAgent-30B-A3B achieve top performance among sub-100B models on five agent planning benchmarks (WorfBench, NaturalPlan, τ²-Bench, BFCL-v3, ACEBench), with particular dominance on WorfBench procedural logic orchestration (80.3% F1 Chain vs GPT-5.2's 60.1%). The proposed χPO online RL algorithm marginally outperforms EPO on ALFWorld while using a 3B rather than 7B model. The MoE variant (MagicAgent-30B-A3B) achieves comparable task performance with 35-55% inference latency reduction versus the dense model. However, MagicAgent does not uniformly surpass closed-source models — GPT-5.2 outperforms on τ²-Bench Retail (75.2% vs 62.0%), and several ultra-scale models match or exceed performance on specific subtasks.",
   "red_flags": [
     {
       "flag": "Reproducibility blocked",
       "detail": "Code and training data are not released ('will be released soon'), and the in-house MagicEval benchmark is proprietary. Third-party verification of the results is currently impossible."
     },
     {
       "flag": "Overstated generalization claim",
       "detail": "The paper claims 'generalized agent planning' in its title and throughout, but 'generalized' is never formally defined and evaluation is restricted to 5 specific pre-selected benchmark types."
     },
     {
       "flag": "Circular evaluation with GPT-5.2",
       "detail": "GPT-5.2 is used as the quality validator for synthetic training data generation AND as an LLM-as-a-Judge for in-house MagicEval evaluation, while simultaneously being a primary benchmark baseline — creating potential bias toward GPT-5.2-aligned outputs."
     },
     {
       "flag": "No statistical rigor",
       "detail": "No confidence intervals, error bars, or significance tests anywhere in the paper despite numerous model comparisons with sometimes small margins (e.g., χPO vs EPO differ by 2.4pp IID Succ. and 0.5pp OOD Succ.)."
     },
     {
       "flag": "Company evaluating own product",
       "detail": "The majority of authors are from Honor Device Co., Ltd; the conclusion explicitly states the model is deployed in Honor's commercial assistant, and no independent evaluation or conflict-of-interest disclosure is present."
     },
     {
       "flag": "No limitations section",
       "detail": "Despite making broad claims about 'generalized' planning and surpassing large closed-source models, the paper includes no limitations or threats-to-validity section."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "Public benchmarks (WorfBench, NaturalPlan, BFCL-v3, etc.) predated this paper and may be in Qwen3's pretraining corpus; neither the base model's training cutoff nor potential train-test overlap are mentioned."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "relevance": "Core ReAct scaffolding paradigm used as evaluation framework for ALFWorld and as basis for trajectory data generation in tool-augmented planning module."
     },
     {
       "title": "Benchmarking Agentic Workflow Generation (WorfBench)",
       "relevance": "Primary benchmark for procedural logic orchestration evaluation; also provides source data for DAG-based procedural logic orchestration training data."
     },
     {
       "title": "Natural Plan: Benchmarking LLMs on Natural Language Planning",
       "relevance": "Primary benchmark for multi-constraint scheduling evaluation (Trip, Meeting, Calendar subtasks)."
     },
     {
       "title": "ALFWorld: Aligning Text and Embodied Environments for Interactive Learning",
       "relevance": "Online embodied RL environment used to evaluate χPO against competing RL algorithms."
     },
     {
       "title": "EPO: Entropy-Regularized Policy Optimization for LLM Agents Reinforcement Learning",
       "relevance": "Direct baseline for χPO online RL algorithm; χPO extends EPO's exploration-exploitation framework with information bottleneck and think-action disentanglement."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "relevance": "Validates the RLVR (Reinforcement Learning with Verifiable Rewards) approach that MagicAgent's unified reward function builds upon."
     },
     {
       "title": "AgentTuning: Enabling Generalized Agent Abilities for LLMs",
       "relevance": "Key prior work on agent-specific fine-tuning; direct comparison baseline and represents the SFT-only paradigm that MagicAgent extends with RL."
     },
     {
       "title": "AgentGym: Evolving Large Language Model-Based Agents Across Diverse Environments",
       "relevance": "Prior RL-for-agents framework used as baseline in ALFWorld evaluation (Table 5)."
     },
     {
       "title": "τ²-Bench: Evaluating Conversational Agents in a Dual-Control Environment",
       "relevance": "Primary benchmark for long-horizon tool execution evaluation across Retail and Airline domains."
     },
     {
       "title": "Magnet: Multi-Turn Tool-Use Data Synthesis and Distillation via Graph Translation",
       "relevance": "Related work on bidirectional translation methodology for tool-augmented planning data generation that MagicAgent's framework draws upon."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Model is already deployed in production at Honor's commercial intelligent assistant, directly applicable to any tool-calling or task-planning agent system."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The core result (curated fine-tuning on diverse planning data improves benchmark performance) is expected; χPO is a modest algorithmic novelty over EPO."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, alignment, or risk discussion anywhere in the paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Claims to outperform GPT-5.2 on WorfBench by ~25pp absolute are notable, but the framing is straightforward product improvement rather than provocative."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Model is deployed in Honor's assistant (usable indirectly) and will be released; Appendix H shows three concrete demo application scenarios."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Honor Device Co., Ltd is a large Chinese phone manufacturer and Fudan University is a prominent institution, but neither is a tier-1 AI research lab."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46991734",
         "title": "RL on GPT-5 to write better kernels",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=46991734",
         "created_at": "2026-02-12T17:22:31Z"
       },
       {
         "hn_id": "47004259",
         "title": "Fine-Tuning GPT-5 for GPU Kernel Generation",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47004259",
         "created_at": "2026-02-13T16:04:35Z"
       }
     ],
     "top_points": 4,
     "total_points": 8,
     "total_comments": 1
   }
 }