ai-research-survey

scan.json (35597B)

---

 {
   "paper": {
     "title": "Disentangling Causal Importance from Emergent Structure in Multi-Expert Orchestration",
     "authors": [
       "Sudipto Ghosh",
       "Sujoy Nath",
       "Sunny Manchanda",
       "Tanmoy Chakraborty"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.04291"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. The orchestrator and INFORM analysis code are not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: GSM8K (Cobbe et al., 2021), HumanEval (Chen et al., 2021), and MMLU (Hendrycks et al., 2021). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training on 'a single NVIDIA A100 80GB GPU' (Appendix C) and lists hyperparameters in Table 4, but does not provide a requirements.txt, Dockerfile, or library version specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, scripts, or README are provided. The architecture and training objective are described but not in enough detail to reproduce without significant reverse engineering."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports 'mean KL divergence ± 95% CI computed across training epochs.' Table 11 reports 'mean and the standard deviation' with ± notation. Box plots with whiskers appear in Figures 6 and 7."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 reports Spearman's ρ with two-sided p-values and Kendall's τ for rank correlation. Table 11 reports Wilcoxon signed-rank test p-values and paired t-test p-values for masking ablation comparisons. Appendix J provides detailed statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports KL divergence magnitudes (e.g., routing divergence of 2.366 vs sequence divergence of 0.428 on MMLU in Table 8), relative comparisons ('5.5× higher routing KL divergence'), speedup ratios ('~3.5× speedup'), and absolute performance differences ('~1.4% performance gain')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 10 experts were chosen, why 100 held-out examples were used for hyperparameter tuning, or why 5 training epochs were run. No power analysis is mentioned."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 reports ± 95% CI across epochs. Table 11 reports mean ± standard deviation. Figure 11 states results are 'mean accuracy... over 3 runs.' Box plots in Figures 6 and 7 show spread across samples."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares against MetaGPT. Figures 9 and 10 compare against a Uniform Baseline, Relational-Only, and Intrinsic-Only variants. Table 9 includes individual model baselines (LLaMA 3.1 8B, Qwen3 8B, DeepSeek-R1 8B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MetaGPT (Hong et al., 2024, ICLR 2024) is a contemporary multi-agent framework. Table 1 also positions against RouteLLM (Ong et al., 2025), IRT-Router (Song et al., 2025), and other recent systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 and Appendix F present extensive ablations: static collaboration graph (Figure 11a), static inference sequence (Figure 11b), masking intrinsically important experts (Table 8, Figure 8), and Relational-Only vs Intrinsic-Only routing variants (Figures 9, 10)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy (GSM8K, MMLU), Pass@1 (HumanEval), KL divergence (routing and sequence), entropy (collaboration and sequence), Gini coefficient (centralization), Spearman's ρ, Kendall's τ, and gradient norms (intrinsic importance)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation of the orchestration system and its outputs is automated. The interpretability claims (e.g., whether INFORM reveals 'meaningful' structure) could benefit from human assessment but none is provided."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states the orchestrator is 'evaluated on a held-out subset of the test set.' Appendix C states 'A held-out set of 100 examples were used to determine the weights of the loss terms,' separating tuning from evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are consistently broken down by task (GSM8K, HumanEval, MMLU), by training epoch (Tables 8, 10; Figures 3-8), and by expert (Figures 4, 5, 13, 14). Different tasks show qualitatively different behaviors."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H presents an extensive discussion of failure modes: hub over-centralization, routing-attribution misalignment, early commitment to suboptimal initializers, overconfidence under semantically damaged inputs, and redundancy masking structural dependence. Task-specific failure profiles are also analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 shows weak and statistically insignificant rank correlation between routing dominance and causal importance — a key negative finding. Relational-Only and Intrinsic-Only ablations show degraded performance (Figure 9). Appendix H documents systematic failure modes of the orchestrator."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: routing-importance divergence by Tables 10-11 and Figures 4-5; asynchronous emergence by Figures 6-7; masking collapse by Table 8 and Figure 8; task-dependent structure by per-task breakdowns throughout."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about expert importance and addresses them through gradient-based attribution (Section 2.2) and controlled ablation/masking experiments (Section 4.5, Appendix F). FAQ Q1 explicitly acknowledges these are 'causal signals' not 'formal causal graphs,' appropriately hedging. Masking single experts (single-variable manipulation) is adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and framing ('Multi-Expert Orchestration') are broad, but the paper only evaluates one specific differentiable orchestrator architecture (Oθ) with one specific training setup, plus a brief cascade analysis in Appendix I. The claim that 'any differentiable routing policy shall suffice' (Section 2.1) is asserted without testing on other architectures. Results on 3 benchmarks with specific model families are presented as general orchestration findings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper considers multiple alternative explanations: FAQ Q3 clarifies high importance doesn't mean better accuracy; Q4 explains why frequently routed experts may have low attribution (interaction hubs vs causal necessity); Q8 discusses when masking doesn't reduce performance (redundancy). Section 4.2 discusses interaction-level dependencies as alternative to intrinsic strength."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "FAQ Q1 explicitly distinguishes between gradient sensitivity (the proxy measured) and formal causal structure (the outcome claimed), stating INFORM provides 'causal signals' not 'full causal mechanisms.' FAQ Q2 and Q3 distinguish between influence on routing decisions (measured) and expert correctness/quality (what might be inferred). The paper is clear about what its measurements do and do not show."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are given: 'LLaMA-3.1 8B', 'Qwen-3 8B', 'DeepSeek-R1 8B' (with 'distilled DeepSeek-R1-0528-Qwen3-8B variant' specified in Appendix D) for the homogeneous consortium, and 'LLaMA-3.2 1B', 'Qwen2.5 3B', 'Mistral 7B' for the heterogeneous one. Oracle: 'GPT OSS 20B' (Agarwal et al., 2025)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt template is provided in Appendix E, including the expert coordination instructions and template structure. The template shows the actual text sent to models, including role instructions and task formatting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides comprehensive hyperparameters: learning rate, batch size, epochs, warmup ratio, gradient clipping, hidden dimension, attention heads, dropout, Gumbel-Softmax temperatures, and all loss coefficients. Tables 6 and 7 list per-expert decoding temperatures. Appendix D specifies top_p settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the orchestration architecture in detail: the Interaction Module (routing adapter, conditional transition matrix with query-key attention and cosine similarity prior), the Selection Module (Gumbel-Softmax sampling with position penalty), and the multi-step expert chaining. Figure 1 provides a visual overview. Appendix C details the training objective."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'initial 512 tokens generated by each expert is used as an input to the orchestrator.' Appendix C describes the held-out set used for tuning. The paper uses standard benchmark splits for GSM8K, HumanEval, and MMLU with standard evaluation protocols."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Appendix A (FAQ) addresses some limitations indirectly (Q1, Q7, Q8), and Appendix H discusses failure modes of orchestration, but neither constitutes a dedicated limitations section about the study methodology itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "FAQ Q1 specifically states INFORM does not 'recover causal structure in the formal sense.' FAQ Q7 notes the method requires white-box access and cannot be applied to API-based systems. FAQ Q8 discusses when masking may not reduce performance due to redundancy. These are specific to this study, not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. FAQ Q7 mentions black-box inapplicability, but there is no systematic statement of scope boundaries — e.g., whether findings generalize beyond the specific orchestrator architecture, beyond 8B-scale models, or beyond the three benchmarks tested. The title implies broad generality without bounding."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (routing matrices, gradient attributions, per-sample results) is released. Only aggregated statistics are reported in figures and tables. The underlying benchmarks are public, but the orchestrator outputs and intermediate analysis data are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data consists of standard public benchmarks (GSM8K, HumanEval, MMLU), which are well-documented in their respective papers. The paper describes how expert outputs are generated (512 initial tokens per expert) and how the orchestrator is evaluated on held-out test subsets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from input to evaluation is documented: benchmarks → expert generation (512 tokens each, Tables 6-7 for decoding configs) → BERT encoder → orchestrator routing → expert chaining → evaluation. Appendix C details the training pipeline. Table 4 specifies all parameters."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. One author is affiliated with DRDO (Defence Research and Development Organisation, India), which is a government defense agency, but no funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: IIT Delhi (Yardi School of AI, Department of Electrical Engineering) and DRDO Young Scientist Laboratory. The paper does not evaluate any product made by these institutions, so no product-affiliation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be verified. One author is affiliated with DRDO (defense organization), but no explicit funding or sponsorship statement is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (LLaMA 3.1, Qwen3, DeepSeek-R1, Mistral 7B, GPT OSS 20B). These models were likely trained on data that includes the benchmarks used for evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The expert LLMs may have been trained on GSM8K (2021), HumanEval (2021), and MMLU (2021) solutions, which could affect both individual expert performance and orchestration dynamics. This is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All three benchmarks (GSM8K, HumanEval, MMLU) were published in 2021, well before the training cutoffs of the 2024-2025 era models used. Benchmark contamination risk is significant and not discussed. If experts have memorized answers, the orchestration task is fundamentally different from what the paper analyzes."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports average model calls (1.44 for INFORM vs 5.00 for MetaGPT) and claims '~3.5× speedup,' but no actual wall-clock time, API costs, tokens consumed, or cost per example is reported. The model call count is a proxy, not an actual cost measurement."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states training was on 'a single NVIDIA A100 80GB GPU' (Appendix C) and lists 5 training epochs with batch size 2, but does not report total GPU hours, training wall-clock time, or total computational budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No explicit seed sensitivity analysis. Figure 11 mentions results are 'over 3 runs' for the ablation study, but the main results in Table 9 report 'best accuracy' values without showing sensitivity across random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Figure 11 states 'over 3 runs' for ablation results. However, the main performance results in Table 9 report 'best accuracy' without stating how many runs produced them. Table 8 reports values 'computed across training epochs,' which is across epochs, not independent runs. Inconsistent across experiments."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 shows a hyperparameter sweep 'through magnitudes {10⁻², 10⁻¹, 10⁰}' for λ_oracle, λ_symmetry, and λ_sparsity. Appendix C states 'A held-out set of 100 examples were used to determine the weights of the loss terms.' The search method and budget are documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 shows the sweep results, but Table 9 reports 'best accuracy' across training epochs without explaining the selection criterion (e.g., best validation epoch, last epoch, or cherry-picked). The 'best' label implies cherry-picking across epochs."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Table 10 reports 15 correlation tests across tasks and epochs, and Table 11 reports multiple paired comparisons, all without correction for multiple comparisons (no Bonferroni, Holm, or Benjamini-Hochberg mentioned)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their INFORM orchestrator against MetaGPT using their own experimental setup (assigning same-size models to MetaGPT roles). No discussion of author-evaluation bias or the risk of inadvertently optimizing their system's presentation relative to the baseline."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares INFORM (1.44 avg calls) vs MetaGPT (5.00 avg calls) at matched expert capabilities. Figure 12 plots MMLU accuracy as a function of total parameters (consortium scaling), showing performance relative to monolith models at different scales."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GSM8K, HumanEval, and MMLU without discussing whether these benchmarks actually measure what INFORM claims to analyze — i.e., whether orchestration dynamics on these tasks generalize to real-world multi-expert collaboration. No construct validity discussion is provided."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The MetaGPT comparison (Table 3) uses a different orchestration mechanism (SOPs vs learned routing) with the same underlying models, but the scaffold confound is not explicitly discussed. The different frameworks introduce confounds beyond routing policy (e.g., MetaGPT's role definitions, communication patterns) that could explain performance differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. All three benchmarks (GSM8K, HumanEval, MMLU, all 2021) predate the training data of the models used (LLaMA 3.1, Qwen3, DeepSeek-R1, all 2024-2025). Solutions to these benchmarks are widely available online and likely in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The initial 512 tokens generated by each expert could contain memorized solutions that leak answer information to the orchestrator. If experts have seen benchmark problems during training, the orchestration task changes fundamentally."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether evaluation examples share structural properties with training data or whether benchmark problems are independent of each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination is mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Routing dominance is a poor proxy for functional necessity — frequently selected experts are not always causally important.",
       "evidence": "Table 10 shows weak and statistically insignificant Spearman rank correlation between routing dominance and intrinsic importance across all tasks and epochs (max ρ = 0.648 with p=0.043 on MMLU Epoch 2, most others p > 0.1). Figures 4 and 5 visually show divergence between gradient attribution and routing mass patterns.",
       "supported": "strong"
     },
     {
       "claim": "Masking the single most intrinsically important expert induces 5.5× higher routing KL divergence than sequencing divergence on MMLU.",
       "evidence": "Table 8 reports KL(Routing) = 2.366 ± 0.497 vs KL(Sequence) = 0.428 ± 0.072 on MMLU (ratio ~5.5×). Figure 8c confirms routing divergence consistently exceeds sequencing divergence across epochs on MMLU.",
       "supported": "strong"
     },
     {
       "claim": "Orchestration behaviors emerge asynchronously: centralization precedes stable routing confidence.",
       "evidence": "Figures 6a and 6c show Gini coefficient (centralization) increasing before collaboration entropy fully stabilizes across tasks. Section 4.3 states 'centralization increases before routing entropy fully stabilizes.'",
       "supported": "moderate"
     },
     {
       "claim": "Expert ordering remains non-deterministic — entropy decreases but stays above zero.",
       "evidence": "Figure 6b shows sequence entropy decreasing across training epochs but remaining significantly above zero on all three tasks. Table 2 shows the starting-expert distribution concentrating but not collapsing to a single expert.",
       "supported": "strong"
     },
     {
       "claim": "INFORM orchestrator achieves 87.1% Pass@1 on HumanEval with 1.44 average model calls vs MetaGPT's 85.9% with 5.00 calls (~3.5× speedup with ~1.4% gain).",
       "evidence": "Table 3 reports these specific numbers. However, the comparison is between a learned adaptive orchestrator and MetaGPT's fixed SOP framework — they are architecturally quite different, so the comparison isolates orchestration style, not just routing efficiency.",
       "supported": "moderate"
     },
     {
       "claim": "A consortium of two 8B Qwen3 experts (~16B total parameters) matches the accuracy of a 70B monolith model on MMLU.",
       "evidence": "Figure 12 shows the consortium curve crossing the 70B monolith accuracy line (~86%) at ~16B total parameters. However, the orchestrator was trained with oracle distillation from GPT OSS 20B, meaning the training signal includes a much larger model's knowledge. The comparison is between a distilled/orchestrated system and a standalone model.",
       "supported": "weak"
     },
     {
       "claim": "Routing sensitivity is task-dependent: numerical token removal affects GSM8K most, while sentence shuffling and reasoning removal affect HumanEval and MMLU most.",
       "evidence": "Figure 3 shows perturbation-specific entropy shifts across tasks, with 'remove_numbers' causing the largest shift on GSM8K (Figure 3a) and 'shuffle_sentences' causing larger shifts on HumanEval and MMLU (Figures 3b, 3c).",
       "supported": "strong"
     },
     {
       "claim": "Masking intrinsically important experts induces substantially greater routing collapse than masking frequently routed experts.",
       "evidence": "Table 11 reports intrinsic masking KL = 1.38 ± 0.40 vs frequent masking KL = 0.11 ± 0.15 on GSM8K (paired t p=0.0065). However, on HumanEval the pattern reverses (intrinsic = 0.30 ± 0.19, frequent = 0.81 ± 0.46, p=0.0223). MMLU shows the expected direction (2.37 vs 1.40) but is not statistically significant (p=0.267).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "INFORM reveals a systematic divergence between routing frequency (relational importance) and gradient-based causal attribution (intrinsic importance) in multi-expert LLM orchestration. Experts that dominate routing often function as interaction hubs with limited causal influence, while sparsely routed experts can be structurally critical. Orchestration dynamics emerge asynchronously during training, with centralization preceding stable routing confidence, and expert ordering remaining non-deterministic. Targeted ablations confirm that masking intrinsically important experts disrupts routing structure disproportionately compared to masking frequently selected peers, though this effect is task-dependent.",
   "red_flags": [
     {
       "flag": "Cherry-picked best-epoch results",
       "detail": "Table 9 reports 'best accuracy' across training epochs rather than final-epoch or averaged results. This allows selecting the most favorable epoch for each task independently, inflating reported performance numbers."
     },
     {
       "flag": "Oracle distillation confounds scaling comparison",
       "detail": "Figure 12 claims a consortium of 2×8B experts matches a 70B monolith on MMLU, but the orchestrator was trained with oracle distillation from GPT OSS 20B. The consortium benefits from knowledge of a much larger model during training, making the parameter-count comparison misleading."
     },
     {
       "flag": "No contamination discussion despite using well-known benchmarks",
       "detail": "GSM8K, HumanEval, and MMLU (all 2021) are likely in the training data of the 2024-2025 era models used. If experts have memorized benchmark answers, the orchestration task is fundamentally different from coordinating genuine reasoning — the orchestrator may be routing to memorized answers rather than reasoning chains."
     },
     {
       "flag": "Unequal comparison with MetaGPT",
       "detail": "Table 3 compares INFORM (a learned, trained orchestrator with oracle distillation) against MetaGPT (a fixed SOP framework). These are architecturally very different systems. The claim of '~3.5× speedup' conflates the benefits of adaptive routing with the overhead of rigid role-based workflows."
     },
     {
       "flag": "Small held-out set for hyperparameter tuning",
       "detail": "Only 100 held-out examples were used for determining loss weights (Appendix C). This small set may not be representative, risking overfitting the orchestrator's training configuration to a small sample."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": [
         "Qingyun Wu",
         "Gagan Bansal",
         "Jieyu Zhang"
       ],
       "year": 2024,
       "relevance": "Foundational multi-agent LLM framework for conversational orchestration, directly compared as a coordination approach in Table 1."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": [
         "Sirui Hong",
         "Mingchen Zhuge",
         "Jonathan Chen"
       ],
       "year": 2024,
       "relevance": "State-of-the-art rigid multi-agent coordination framework used as the primary performance baseline in Table 3."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": [
         "Isaac Ong",
         "Amjad Almahairi",
         "Vincent Wu"
       ],
       "year": 2025,
       "relevance": "LLM routing framework for cost-performance trade-off, positioned in Table 1 as having moderate interpretability."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": [
         "Lingjiao Chen",
         "Matei Zaharia",
         "James Zou"
       ],
       "year": 2024,
       "relevance": "Cost-efficient cascade routing approach analyzed in Appendix I using INFORM's causal attribution methods."
     },
     {
       "title": "LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion",
       "authors": [
         "Dongfu Jiang",
         "Xiang Ren",
         "Bill Yuchen Lin"
       ],
       "year": 2023,
       "relevance": "Output aggregation framework for multi-LLM ensembles that treats expert contributions as exchangeable, contrasted with INFORM's sequential approach."
     },
     {
       "title": "Mixture-of-agents enhances large language model capabilities",
       "authors": [
         "Junlin Wang",
         "Jue Wang",
         "Ben Athiwaratkun"
       ],
       "year": 2025,
       "relevance": "Multi-LLM mixture-of-agents approach demonstrating capability gains from model collaboration."
     },
     {
       "title": "Multi-Agent Collaboration via Evolving Orchestration",
       "authors": [
         "Yufan Dang",
         "Chen Qian",
         "Xueheng Luo"
       ],
       "year": 2025,
       "relevance": "Evolving orchestration framework for multi-agent collaboration, directly relevant to learned orchestration mechanisms."
     },
     {
       "title": "IRT-router: Effective and interpretable multi-LLM routing via item response theory",
       "authors": [
         "Wei Song",
         "Zhenya Huang",
         "Cheng Cheng"
       ],
       "year": 2025,
       "relevance": "Interpretable LLM router using Item Response Theory, positioned as having high interpretability in Table 1."
     },
     {
       "title": "MapCoder: Multi-Agent Code Generation for Competitive Problem Solving",
       "authors": [
         "Md. Ashraful Islam",
         "Mohammed Eunus Ali",
         "Md Rizwan Parvez"
       ],
       "year": 2024,
       "relevance": "Multi-agent code generation system demonstrating agentic collaboration for programming tasks."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": [
         "Mark Chen",
         "Jerry Tworek",
         "Heewoo Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the HumanEval benchmark used as one of the three primary evaluation benchmarks in this paper."
     },
     {
       "title": "Wisdom and Delusion of LLM Ensembles for Code Generation and Repair",
       "authors": [
         "Fernando Vallecillos-Ruiz",
         "Max Hort",
         "Leon Moonen"
       ],
       "year": 2025,
       "relevance": "Studies LLM ensemble degeneration into static ensembles, a failure mode INFORM aims to diagnose."
     },
     {
       "title": "Can Dependencies Induced by LLM-Agent Workflows Be Trusted?",
       "authors": [
         "Yu Yao",
         "Yiliao Song",
         "Yian Xie"
       ],
       "year": 2025,
       "relevance": "Studies trustworthiness of dependencies in LLM-agent workflows, directly related to INFORM's analysis of orchestration dependencies."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The INFORM framework could help practitioners debug multi-agent LLM systems, but requires white-box access and significant adaptation to apply beyond the specific orchestrator studied."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The core finding that routing frequency diverges from causal importance — popular experts aren't necessarily important ones — is a genuinely counterintuitive insight for anyone building multi-agent systems."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or misuse concerns are raised; the paper is purely about interpretability of routing mechanisms."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mildly questions the opacity of existing multi-agent frameworks like MetaGPT and AutoGen, but does so academically without any pointed controversy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code release, no demo, no tool — purely an analytical framework described in a paper with no public implementation mentioned."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From IIT Delhi and DRDO — recognized in India but not household names in the global tech/AI community; no famous product or lab involved."
     }
   }
 }