ai-research-survey

scan-v4.json (36637B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Disentangling Causal Importance from Emergent Structure in Multi-Expert Orchestration",
     "authors": [
       "Sudipto Ghosh",
       "Sujoy Nath",
       "Sunny Manchanda",
       "Tanmoy Chakraborty"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.04291",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: routing-importance divergence by Tables 10-11 and Figures 4-5; asynchronous emergence by Figures 6-7; masking collapse by Table 8 and Figure 8; task-dependent structure by per-task breakdowns throughout.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about expert importance and addresses them through gradient-based attribution (Section 2.2) and controlled ablation/masking experiments (Section 4.5, Appendix F). FAQ Q1 explicitly acknowledges these are 'causal signals' not 'formal causal graphs,' appropriately hedging. Masking single experts (single-variable manipulation) is adequate for the claims made.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and framing ('Multi-Expert Orchestration') are broad, but the paper only evaluates one specific differentiable orchestrator architecture (Oθ) with one specific training setup, plus a brief cascade analysis in Appendix I. The claim that 'any differentiable routing policy shall suffice' (Section 2.1) is asserted without testing on other architectures. Results on 3 benchmarks with specific model families are presented as general orchestration findings.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper considers multiple alternative explanations: FAQ Q3 clarifies high importance doesn't mean better accuracy; Q4 explains why frequently routed experts may have low attribution (interaction hubs vs causal necessity); Q8 discusses when masking doesn't reduce performance (redundancy). Section 4.2 discusses interaction-level dependencies as alternative to intrinsic strength.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "FAQ Q1 explicitly distinguishes between gradient sensitivity (the proxy measured) and formal causal structure (the outcome claimed), stating INFORM provides 'causal signals' not 'full causal mechanisms.' FAQ Q2 and Q3 distinguish between influence on routing decisions (measured) and expert correctness/quality (what might be inferred). The paper is clear about what its measurements do and do not show.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Appendix A (FAQ) addresses some limitations indirectly (Q1, Q7, Q8), and Appendix H discusses failure modes of orchestration, but neither constitutes a dedicated limitations section about the study methodology itself.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "FAQ Q1 specifically states INFORM does not 'recover causal structure in the formal sense.' FAQ Q7 notes the method requires white-box access and cannot be applied to API-based systems. FAQ Q8 discusses when masking may not reduce performance due to redundancy. These are specific to this study, not generic disclaimers.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. FAQ Q7 mentions black-box inapplicability, but there is no systematic statement of scope boundaries — e.g., whether findings generalize beyond the specific orchestrator architecture, beyond 8B-scale models, or beyond the three benchmarks tested. The title implies broad generality without bounding.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. One author is affiliated with DRDO (Defence Research and Development Organisation, India), which is a government defense agency, but no funding statement is provided.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: IIT Delhi (Yardi School of AI, Department of Electrical Engineering) and DRDO Young Scientist Laboratory. The paper does not evaluate any product made by these institutions, so no product-affiliation conflict exists.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be verified. One author is affiliated with DRDO (defense organization), but no explicit funding or sponsorship statement is provided.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are precisely defined: 'relational importance' (total incoming routing mass u_j(x)), 'intrinsic importance' (gradient norm of log-probability w.r.t. expert representation I(E_i)), 'orchestration' (sequential expert selection mechanism), and 'expert zoo' (fixed consortium of frozen LLMs) are all formally defined with equations.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states its contribution: INFORM, an interpretability framework that treats orchestration as analyzable computation enabling decoupling of interaction structure, sequencing, and causal attribution, with four explicit research questions (RQ1–RQ4) stated in the introduction.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a structured comparison of INFORM against six prior systems (MetaGPT, AutoGen, FrugalGPT, RouteLLM, etc.) along specific dimensions; Appendix B extends the related work across MoE, multi-agent collaboration, and interpretability literature with clear positioning.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. The orchestrator and INFORM analysis code are not released.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses publicly available benchmarks: GSM8K (Cobbe et al., 2021), HumanEval (Chen et al., 2021), and MMLU (Hendrycks et al., 2021). No proprietary data was collected.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions training on 'a single NVIDIA A100 80GB GPU' (Appendix C) and lists hyperparameters in Table 4, but does not provide a requirements.txt, Dockerfile, or library version specifications sufficient to recreate the environment.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions, scripts, or README are provided. The architecture and training objective are described but not in enough detail to reproduce without significant reverse engineering.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Table 8 reports 'mean KL divergence ± 95% CI computed across training epochs.' Table 11 reports 'mean and the standard deviation' with ± notation. Box plots with whiskers appear in Figures 6 and 7.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Table 10 reports Spearman's ρ with two-sided p-values and Kendall's τ for rank correlation. Table 11 reports Wilcoxon signed-rank test p-values and paired t-test p-values for masking ablation comparisons. Appendix J provides detailed statistical tests.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports KL divergence magnitudes (e.g., routing divergence of 2.366 vs sequence divergence of 0.428 on MMLU in Table 8), relative comparisons ('5.5× higher routing KL divergence'), speedup ratios ('~3.5× speedup'), and absolute performance differences ('~1.4% performance gain').",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification is given for why 10 experts were chosen, why 100 held-out examples were used for hyperparameter tuning, or why 5 training epochs were run. No power analysis is mentioned.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 8 reports ± 95% CI across epochs. Table 11 reports mean ± standard deviation. Figure 11 states results are 'mean accuracy... over 3 runs.' Box plots in Figures 6 and 7 show spread across samples.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 compares against MetaGPT. Figures 9 and 10 compare against a Uniform Baseline, Relational-Only, and Intrinsic-Only variants. Table 9 includes individual model baselines (LLaMA 3.1 8B, Qwen3 8B, DeepSeek-R1 8B).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "MetaGPT (Hong et al., 2024, ICLR 2024) is a contemporary multi-agent framework. Table 1 also positions against RouteLLM (Ong et al., 2025), IRT-Router (Song et al., 2025), and other recent systems.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.5 and Appendix F present extensive ablations: static collaboration graph (Figure 11a), static inference sequence (Figure 11b), masking intrinsically important experts (Table 8, Figure 8), and Relational-Only vs Intrinsic-Only routing variants (Figures 9, 10).",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports accuracy (GSM8K, MMLU), Pass@1 (HumanEval), KL divergence (routing and sequence), entropy (collaboration and sequence), Gini coefficient (centralization), Spearman's ρ, Kendall's τ, and gradient norms (intrinsic importance).",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation is performed. All evaluation of the orchestration system and its outputs is automated. The interpretability claims (e.g., whether INFORM reveals 'meaningful' structure) could benefit from human assessment but none is provided.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 states the orchestrator is 'evaluated on a held-out subset of the test set.' Appendix C states 'A held-out set of 100 examples were used to determine the weights of the loss terms,' separating tuning from evaluation.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are consistently broken down by task (GSM8K, HumanEval, MMLU), by training epoch (Tables 8, 10; Figures 3-8), and by expert (Figures 4, 5, 13, 14). Different tasks show qualitatively different behaviors.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix H presents an extensive discussion of failure modes: hub over-centralization, routing-attribution misalignment, early commitment to suboptimal initializers, overconfidence under semantically damaged inputs, and redundancy masking structural dependence. Task-specific failure profiles are also analyzed.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 10 shows weak and statistically insignificant rank correlation between routing dominance and causal importance — a key negative finding. Relational-Only and Intrinsic-Only ablations show degraded performance (Figure 9). Appendix H documents systematic failure modes of the orchestrator.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model versions are given: 'LLaMA-3.1 8B', 'Qwen-3 8B', 'DeepSeek-R1 8B' (with 'distilled DeepSeek-R1-0528-Qwen3-8B variant' specified in Appendix D) for the homogeneous consortium, and 'LLaMA-3.2 1B', 'Qwen2.5 3B', 'Mistral 7B' for the heterogeneous one. Oracle: 'GPT OSS 20B' (Agarwal et al., 2025).",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "The full prompt template is provided in Appendix E, including the expert coordination instructions and template structure. The template shows the actual text sent to models, including role instructions and task formatting.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 provides comprehensive hyperparameters: learning rate, batch size, epochs, warmup ratio, gradient clipping, hidden dimension, attention heads, dropout, Gumbel-Softmax temperatures, and all loss coefficients. Tables 6 and 7 list per-expert decoding temperatures. Appendix D specifies top_p settings.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 describes the orchestration architecture in detail: the Interaction Module (routing adapter, conditional transition matrix with query-key attention and cosine similarity prior), the Selection Module (Gumbel-Softmax sampling with position penalty), and the multi-step expert chaining. Figure 1 provides a visual overview. Appendix C details the training objective.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 states 'initial 512 tokens generated by each expert is used as an input to the orchestrator.' Appendix C describes the held-out set used for tuning. The paper uses standard benchmark splits for GSM8K, HumanEval, and MMLU with standard evaluation protocols.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No raw experimental data (routing matrices, gradient attributions, per-sample results) is released. Only aggregated statistics are reported in figures and tables. The underlying benchmarks are public, but the orchestrator outputs and intermediate analysis data are not available.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The data consists of standard public benchmarks (GSM8K, HumanEval, MMLU), which are well-documented in their respective papers. The paper describes how expert outputs are generated (512 initial tokens per expert) and how the orchestrator is evaluated on held-out test subsets.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data sources are standard public benchmarks.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline from input to evaluation is documented: benchmarks → expert generation (512 tokens each, Tables 6-7 for decoding configs) → BERT encoder → orchestrator routing → expert chaining → evaluation. Appendix C details the training pipeline. Table 4 specifies all parameters.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates are stated for any of the models used (LLaMA 3.1, Qwen3, DeepSeek-R1, Mistral 7B, GPT OSS 20B). These models were likely trained on data that includes the benchmarks used for evaluation.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of potential train/test overlap. The expert LLMs may have been trained on GSM8K (2021), HumanEval (2021), and MMLU (2021) solutions, which could affect both individual expert performance and orchestration dynamics. This is not addressed.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "All three benchmarks (GSM8K, HumanEval, MMLU) were published in 2021, well before the training cutoffs of the 2024-2025 era models used. Benchmark contamination risk is significant and not discussed. If experts have memorized answers, the orchestration task is fundamentally different from what the paper analyzes.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Table 3 reports average model calls (1.44 for INFORM vs 5.00 for MetaGPT) and claims '~3.5× speedup,' but no actual wall-clock time, API costs, tokens consumed, or cost per example is reported. The model call count is a proxy, not an actual cost measurement.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper states training was on 'a single NVIDIA A100 80GB GPU' (Appendix C) and lists 5 training epochs with batch size 2, but does not report total GPU hours, training wall-clock time, or total computational budget.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No explicit seed sensitivity analysis. Figure 11 mentions results are 'over 3 runs' for the ablation study, but the main results in Table 9 report 'best accuracy' values without showing sensitivity across random seeds.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "Figure 11 states 'over 3 runs' for ablation results. However, the main performance results in Table 9 report 'best accuracy' without stating how many runs produced them. Table 8 reports values 'computed across training epochs,' which is across epochs, not independent runs. Inconsistent across experiments.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": true,
           "justification": "Table 5 shows a hyperparameter sweep 'through magnitudes {10⁻², 10⁻¹, 10⁰}' for λ_oracle, λ_symmetry, and λ_sparsity. Appendix C states 'A held-out set of 100 examples were used to determine the weights of the loss terms.' The search method and budget are documented.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "Table 5 shows the sweep results, but Table 9 reports 'best accuracy' across training epochs without explaining the selection criterion (e.g., best validation epoch, last epoch, or cherry-picked). The 'best' label implies cherry-picking across epochs.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "Table 10 reports 15 correlation tests across tasks and epochs, and Table 11 reports multiple paired comparisons, all without correction for multiple comparisons (no Bonferroni, Holm, or Benjamini-Hochberg mentioned).",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors compare their INFORM orchestrator against MetaGPT using their own experimental setup (assigning same-size models to MetaGPT roles). No discussion of author-evaluation bias or the risk of inadvertently optimizing their system's presentation relative to the baseline.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 compares INFORM (1.44 avg calls) vs MetaGPT (5.00 avg calls) at matched expert capabilities. Figure 12 plots MMLU accuracy as a function of total parameters (consortium scaling), showing performance relative to monolith models at different scales.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses GSM8K, HumanEval, and MMLU without discussing whether these benchmarks actually measure what INFORM claims to analyze — i.e., whether orchestration dynamics on these tasks generalize to real-world multi-expert collaboration. No construct validity discussion is provided.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The MetaGPT comparison (Table 3) uses a different orchestration mechanism (SOPs vs learned routing) with the same underlying models, but the scaffold confound is not explicitly discussed. The different frameworks introduce confounds beyond routing policy (e.g., MetaGPT's role definitions, communication patterns) that could explain performance differences.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Not discussed. All three benchmarks (GSM8K, HumanEval, MMLU, all 2021) predate the training data of the models used (LLaMA 3.1, Qwen3, DeepSeek-R1, all 2024-2025). Solutions to these benchmarks are widely available online and likely in training data.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Not discussed. The initial 512 tokens generated by each expert could contain memorized solutions that leak answer information to the orchestrator. If experts have seen benchmark problems during training, the orchestration task changes fundamentally.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Not discussed. No analysis of whether evaluation examples share structural properties with training data or whether benchmark problems are independent of each other.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination is mentioned.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Routing dominance is a poor proxy for functional necessity: rank correlation between routing mass and gradient-based importance is weak and mostly statistically insignificant across tasks and epochs.",
       "evidence": "Table 10 shows Spearman's ρ ranging from 0.152–0.648 across tasks and epochs, with most p-values > 0.1; only one epoch on MMLU achieves p=0.043.",
       "supported": "moderate"
     },
     {
       "claim": "Masking the single most intrinsically important expert on MMLU induces ~5.5× higher routing KL divergence than sequencing KL divergence, confirming interaction hub dependence.",
       "evidence": "Table 8 shows MMLU routing KL = 2.366 ± 0.497 vs sequence KL = 0.428 ± 0.072 (ratio ≈ 5.5); GSM8K also shows routing > sequencing divergence.",
       "supported": "strong"
     },
     {
       "claim": "Orchestration behaviors emerge asynchronously: expert centralization increases before routing confidence stabilizes, indicating the system learns 'who to trust' before 'how confidently to route.'",
       "evidence": "Figures 6a and 6c show Gini coefficient rising before routing entropy fully decreases across tasks; the decoupling is described quantitatively but without a formal statistical test.",
       "supported": "moderate"
     },
     {
       "claim": "INFORM achieves better performance than MetaGPT on HumanEval (87.1 vs 85.9 pass@1) with 3.5× fewer model calls (1.44 vs 5.00 average calls).",
       "evidence": "Table 3 directly reports these numbers; however, the comparison uses role-specific MetaGPT with same-sized models, which may not represent MetaGPT's intended use case.",
       "supported": "moderate"
     },
     {
       "claim": "Intrinsic expert importance is sparse and task-dependent: only a small subset of experts consistently exert causal influence, and which experts matter differs by task.",
       "evidence": "Figures 4 and 13 show sparse attribution heatmaps across epochs for all three tasks; qualitative consistency is clear but cross-task comparison relies on visual inspection.",
       "supported": "moderate"
     },
     {
       "claim": "The full INFORM model (combining relational and intrinsic signals) outperforms relational-only and intrinsic-only variants on all three benchmarks.",
       "evidence": "Figure 9 shows consistent performance ordering (full > intrinsic-only > relational-only > uniform baseline) across GSM8K, MMLU, and HumanEval, though no error bars are shown.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational",
     "ablation-study"
   ],
   "key_findings": "INFORM reveals a systematic divergence between relational importance (routing frequency/mass) and intrinsic importance (gradient-based causal attribution) in learned multi-expert LLM orchestration: frequently selected experts often function as interaction hubs with limited causal influence, while sparsely routed experts can be structurally critical. Orchestration dynamics emerge asynchronously—expert centralization precedes routing confidence stabilization—and expert ordering remains non-deterministic at inference time. Targeted masking of intrinsically important experts induces disproportionate routing collapse (up to 5.5× greater KL divergence than sequencing disruption on MMLU), validating that gradient attribution captures functional necessity invisible to accuracy metrics alone. The heterogeneous expert consortium shows slower, more volatile convergence than the homogeneous setting, indicating that mixed-capacity orchestration is harder to stabilize.",
   "red_flags": [
     {
       "flag": "No code released",
       "detail": "The custom INFORM orchestrator architecture is described mathematically but no implementation is released, making reproduction dependent entirely on reimplementing from equations and hyperparameter tables."
     },
     {
       "flag": "Weak statistical support for central claim",
       "detail": "The core claim that routing dominance misrepresents causal importance rests on rank correlations (Table 10) that are statistically insignificant in 14 of 15 task-epoch combinations; the one significant result (MMLU Epoch 2, p=0.043) does not replicate across epochs."
     },
     {
       "flag": "No variance on main performance results",
       "detail": "Table 9 performance values (MMLU, GSM8K, HumanEval accuracy/pass@1) are single-point estimates; Figure 11 caption mentions 'mean over 3 runs' but no standard deviations are shown, obscuring reliability of the comparisons."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "GSM8K, HumanEval, and MMLU all predate the training cutoffs of LLaMA 3.1, Qwen3, and DeepSeek-R1; the paper does not discuss potential contamination, which could inflate performance baselines."
     },
     {
       "flag": "Single orchestrator architecture tested",
       "detail": "All interpretability findings derive from one custom BERT-based orchestrator with Gumbel-Softmax routing; whether the routing-attribution divergence is a general phenomenon or an artifact of this specific design is not tested."
     },
     {
       "flag": "No limitations section",
       "detail": "There is no dedicated limitations or threats-to-validity section; methodological caveats are dispersed across a FAQ appendix and not synthesized into a coherent assessment of where findings may not hold."
     },
     {
       "flag": "Inconsistent MetaGPT comparison",
       "detail": "The MetaGPT comparison (Table 3) uses INFORM at what appears to be Epoch 4 (87.1 pass@1 matches Table 9 Epoch 4), while Epoch 5 achieves 88.4, suggesting the comparison may not use the best-performing INFORM checkpoint."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "relevance": "Multi-agent orchestration framework; used as context for the problem of opaque coordination policies"
     },
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "relevance": "Direct baseline comparison for multi-expert coordination efficiency; represents rigid, structured orchestration"
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "relevance": "Cascade-based orchestration baseline; INFORM is applied to interpret FrugalGPT-style systems in Appendix I"
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "relevance": "Contemporary learned routing approach compared in Table 1; represents performance-driven routing without interpretability"
     },
     {
       "title": "Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity",
       "relevance": "Foundational MoE routing work; motivates the distinction between selection frequency and causal importance"
     },
     {
       "title": "LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion",
       "relevance": "Output aggregation approach contrasted with INFORM's sequential orchestration analysis"
     },
     {
       "title": "On the Resilience of LLM-Based Multi-Agent Collaboration with Faulty Agents",
       "relevance": "Related work on failure propagation in multi-agent systems; motivates need for interpretability of expert influence"
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "relevance": "Multi-agent coding system representing role-based orchestration; context for the multi-expert collaboration paradigm"
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "relevance": "HumanEval benchmark paper; primary evaluation dataset for code generation in this study"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The INFORM framework could help practitioners debug multi-agent LLM systems, but requires white-box access and significant adaptation to apply beyond the specific orchestrator studied."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The core finding that routing frequency diverges from causal importance — popular experts aren't necessarily important ones — is a genuinely counterintuitive insight for anyone building multi-agent systems."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or misuse concerns are raised; the paper is purely about interpretability of routing mechanisms."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mildly questions the opacity of existing multi-agent frameworks like MetaGPT and AutoGen, but does so academically without any pointed controversy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code release, no demo, no tool — purely an analytical framework described in a paper with no public implementation mentioned."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From IIT Delhi and DRDO — recognized in India but not household names in the global tech/AI community; no famous product or lab involved."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }