ai-research-survey

scan.json (23839B)

---

 {
   "paper": {
     "title": "Power and Limitations of Aggregation in Compound AI Systems",
     "authors": [
       "Nivasini Ananthakrishnan",
       "Meena Jagadeesan"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.21556"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["theoretical", "case-study"],
   "key_findings": "This paper provides a theoretical framework characterizing when aggregation in compound AI systems expands the set of elicitable outputs beyond what a single model can produce. Three necessary mechanisms are identified: feasibility expansion, support expansion, and binding set contraction. Strengthened versions of these mechanisms are proven both necessary and sufficient for elicitability-expansion (Theorems 4.3, 4.4). Empirical illustration using GPT-4o-mini on a toy reference-generation task demonstrates all three mechanisms, though the feasibility expansion effect is very small (ℓ1 gap [0.03, 0.15]).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/nivasini/aggregation_compound_ai, referenced in footnotes on pages 7 and 16."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The code repository enables reproduction of all experimental data. The only external dataset used (AlpacaEval helpful-base) is publicly available. Experimental outputs are generated from the released code."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies the model (gpt-4o-mini-2024-07-18) and embedding model (all-mpnet-base-v2) but provides no requirements.txt, Dockerfile, or detailed dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Appendices E and F describe the experimental methodology in detail, but the paper does not provide step-by-step reproduction instructions (no README-style commands or scripts to run)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports 95% Wilson confidence intervals for all output vector coordinates and ℓ1 distances. Figure 3 shows shaded confidence sets. Appendix F describes the confidence interval methodology."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 95% confidence intervals to assess whether ℓ1 distances between aggregated and best single-model outputs are non-zero. For feasibility expansion, they note the CI [0.03, 0.15] 'notably does not contain 0' (Section 5.1), which is functionally equivalent to a significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2d reports ℓ1 distances with confidence intervals: support expansion [0.58, 0.94], feasibility expansion [0.03, 0.15], binding set contraction [0.21, 0.35]. These provide the magnitude of the elicitability-expansion effect."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 30 trials were used (Appendix F) but provides no justification for this choice and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for all quantities in Table 2, encoding variance information. The paper explicitly describes averaging over 30 trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares aggregated outputs x(A) against the best achievable single-model output x∗_P(x(A)), found via brute-force search over all prompt specifications (Section 5). This is the natural baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is GPT-4o-mini without aggregation — the same contemporary model used for the aggregated system."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system being studied (aggregation of prompts) has essentially one component — the aggregation rule. The paper tests different aggregation operations separately, but there are no multi-component systems to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary evaluation metric is the ℓ1 distance between x(A) and x∗_P(x(A)). Coordinate-wise output vectors are reported but serve as descriptive measurements, not a separate evaluation metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The paper relies entirely on LLM-as-a-judge (GPT-4o-mini) to classify paper topics. Human validation of topic classifications could have strengthened the empirical claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No train/test paradigm exists. The paper evaluates LLM prompting behavior, not supervised model performance. The brute-force prompt search is part of the evaluation methodology, not tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by all three mechanisms (support expansion, binding set contraction, feasibility expansion) in Table 2 and Figure 3, with per-dimension output vectors for each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that feasibility expansion evidence is 'slightly weaker than for the other two mechanisms' with a very small gap (Section 5.1). Table 1 explicitly shows which mechanisms each aggregation rule cannot implement."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows fundamental limitations: intersection aggregation cannot implement support expansion (Proposition A.1), addition cannot implement feasibility expansion (Proposition A.2). Propositions A.4 and A.5 show that the natural mechanisms are not sufficient for elicitability-expansion."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: three mechanisms are formalized (Section 3), necessity is proven (Theorem 3.7), full characterization via strengthened mechanisms is proven (Theorems 4.3, 4.4), and empirical illustration is provided (Section 5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The theoretical claims (aggregation 'expands' elicitability, 'overcomes' limitations) are formally proven within the principal-agent model. The empirical claims are hedged as 'results suggest' and 'empirical illustration' (Section 5.1)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper repeatedly describes the framework as 'stylized' (Section 1). Section 6 explicitly lists what the model does NOT capture: stochasticity, nonlinear constraints, multi-turn interactions, tool use, fine-tuning. The abstract says 'takes a step towards characterizing' rather than claiming full characterization of real systems."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses model limitations and extensions including randomness, nonlinear limitations, and multi-turn interactions. For the empirical results, the paper shows that using finer-grained prompt topics (output dimension topics vs. prompt topics) can achieve similar effects without aggregation (Section 5.1)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between its theoretical notion of elicitability-expansion (Definition 2.4) and the empirical operationalization based on ℓ1 distance closeness. Section 5 states 'we relax these concepts to allow for closeness in output vectors rather than necessarily requiring an exact match.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The exact model version 'gpt-4o-mini-2024-07-18' is specified in Appendix E and F. The embedding model 'all-mpnet-base-v2' is also specified (Appendix E)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendices E and F: five perspective prompts for reference generation, intersection-style and addition-style aggregation instructions, and the LLM-as-a-judge classification prompt. The Section 5 experiment prompts and prompt template are also given in Appendix F."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is reported: 1.0 for the Section 2.4 embedding experiments (Appendix E), 0 for all Section 5 experiments (Appendix F)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLM is directly prompted with single-turn queries."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F documents preprocessing: title reformatting (lowercase, remove punctuation, remove whitespace), deduplication via substring matching, LLM-as-judge classification procedure, and Wilson confidence interval computation. Appendix E describes the embedding shift to the nonnegative orthant."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) contains a substantive 'Model limitations and extensions' subsection discussing multiple specific limitations of the framework."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific limitations: deterministic agent assumption (real LLMs are stochastic), linear constraints and features (real systems may have nonlinear limitations), single-round interaction (debate protocols use multi-turn), and agent reward independence (real interactions have interdependencies)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what the results do NOT show: 'we assume for simplicity that the agent chooses an output deterministically,' 'we restrict the output constraints and the feature map to linear functional forms,' 'we also assume each agent's reward depends only on its own outputs.' Extensions needed for nonlinear limitations and multi-turn interactions are acknowledged."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The code to regenerate experiments is released on GitHub, but the actual raw LLM outputs from the 30 trials are not explicitly stated to be available. LLM outputs may differ over time even with the same code."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is well-described: GPT-4o-mini is queried with specific prompts, outputs are classified by LLM-as-judge, output vectors are computed as topic fractions, averaged over 30 trials (Sections 5, E, F)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is generated from LLM queries using a standard public dataset (AlpacaEval) for embedding calibration."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Appendices E and F: prompting → LLM output → title reformatting/deduplication → aggregation → LLM-as-judge classification → output vector computation → confidence interval calculation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Acknowledgments) states: 'This work was partially supported by a Stanford AI Lab postdoctoral fellowship.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Nivasini Ananthakrishnan at UC Berkeley, Meena Jagadeesan at Stanford University. Neither author is affiliated with OpenAI (the evaluated model provider)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Stanford AI Lab postdoctoral fellowship has no financial interest in the outcomes of this study. The paper evaluates OpenAI's GPT-4o-mini, and the funder (Stanford) is independent."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It studies the structural properties of prompting and aggregation in a toy illustration task — not measuring model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model on a benchmark. The reference-generation task studies prompting/aggregation behavior, not whether the model can solve benchmark problems."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. The empirical part is a toy illustration of theoretical mechanisms, not a measurement of model capability on a contamination-sensitive task."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Primarily a theoretical paper. The empirical illustration is a toy task; cost reporting is not relevant to the theoretical contributions."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Primarily a theoretical paper. The empirical component is a small-scale illustration, not a computationally intensive experiment."
       }
     }
   },
   "claims": [
     {
       "claim": "Any aggregation operation must implement at least one of three mechanisms (feasibility expansion, support expansion, or binding set contraction) to expand the set of elicitable outputs.",
       "evidence": "Theorem 3.7 (Section 3.2) with formal proof in Section C.1, building on Theorem 4.4.",
       "supported": "strong"
     },
     {
       "claim": "Strengthened versions of the three mechanisms provide necessary and sufficient conditions that fully characterize elicitability-expansion.",
       "evidence": "Theorem 4.3 (sufficiency, proof in Section D.6) and Theorem 4.4 (necessity, proof in Section D.5) establish the full characterization via the power-characterizing condition (Definition 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Intersection aggregation cannot implement support expansion for any problem instance, and addition aggregation cannot implement feasibility expansion for any problem instance.",
       "evidence": "Proposition A.1 and Proposition A.2 (Section A.1, Table 1) provide formal proofs of these impossibility results.",
       "supported": "strong"
     },
     {
       "claim": "LLMs (GPT-4o-mini) in a toy reference-generation task empirically demonstrate all three mechanisms, and each mechanism expands elicitability.",
       "evidence": "Section 5, Figure 3, Table 2. Support expansion: ℓ1 distance [0.58, 0.94]. Binding set contraction: ℓ1 distance [0.21, 0.35]. Feasibility expansion: ℓ1 distance [0.03, 0.15]. All 95% CIs exclude 0.",
       "supported": "moderate"
     },
     {
       "claim": "The three natural mechanisms (non-strengthened) are not sufficient for elicitability-expansion.",
       "evidence": "Propositions A.4 and A.5 (Section A.2) construct counterexamples where support expansion and binding set contraction are implemented but aggregation is not elicitability-expanding.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Toy empirical illustration",
       "detail": "The empirical section uses a single model (GPT-4o-mini) on a contrived reference-generation task with hand-picked topics designed to demonstrate each mechanism. The gap between this stylized illustration and real compound AI systems is substantial. The paper acknowledges this but the empirical evidence for the theoretical framework's practical relevance remains thin."
     },
     {
       "flag": "LLM-as-judge self-evaluation",
       "detail": "GPT-4o-mini classifies the topics of reference lists that it generated itself. This introduces potential systematic bias in the output vector measurements, since the model's own biases in generation and classification may be correlated."
     },
     {
       "flag": "Weak feasibility expansion evidence",
       "detail": "The feasibility expansion experiment has a very small ℓ1 gap ([0.03, 0.15]), which the paper acknowledges is 'slightly weaker' evidence. The confidence interval barely excludes 0, raising questions about whether this mechanism is practically meaningful for LLMs."
     }
   ],
   "cited_papers": [
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba", "Joshua B. Tenenbaum", "Igor Mordatch"],
       "year": 2024,
       "relevance": "Multi-agent LLM debate protocol for improving factuality, directly motivates the aggregation framework studied in this paper."
     },
     {
       "title": "How we built our multi-agent research system",
       "authors": ["Anthropic"],
       "year": 2025,
       "relevance": "Multi-agent research system where a lead LLM delegates to specialized agents and aggregates outputs — a key motivating application."
     },
     {
       "title": "Ask me anything: A simple strategy for prompting language models",
       "authors": ["Simran Arora", "Avanika Narayan", "Mayee F. Chen"],
       "year": 2022,
       "arxiv_id": "2210.02441",
       "relevance": "Prompt ensembling approach that replaces complex single-model prompting with diverse multi-model strategies, directly relevant to aggregation benefits."
     },
     {
       "title": "Debating with more persuasive LLMs leads to more truthful answers",
       "authors": ["Akbir Khan", "John Hughes", "Dan Valentine"],
       "year": 2024,
       "relevance": "LLM debate where different agents argue for different answers, a form of aggregation studied in this paper."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Self-consistency as a form of output aggregation across multiple reasoning traces from the same model."
     },
     {
       "title": "Adversaries can misuse combinations of safe models",
       "authors": ["Erik Jones", "Anca Dragan", "Jacob Steinhardt"],
       "year": 2025,
       "relevance": "Adversarial model combination to generate unsafe outcomes — explores when aggregation enables capabilities not available from individual models."
     },
     {
       "title": "Deep reinforcement learning from human preferences",
       "authors": ["Paul F. Christiano", "Jan Leike", "Tom Brown"],
       "year": 2017,
       "relevance": "RLHF framework for training models with human feedback, related to reward function specification in the principal-agent model."
     },
     {
       "title": "The consensus game: Language model generation via equilibrium search",
       "authors": ["Athul Paul Jacob", "Yikang Shen", "Gabriele Farina", "Jacob Andreas"],
       "year": 2023,
       "arxiv_id": "2310.09139",
       "relevance": "Consensus between generators and discriminators as a form of multi-agent LLM aggregation."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT and RLHF alignment, cited for empirical observation of alignment-hedging side effects relevant to feasibility expansion mechanism."
     },
     {
       "title": "Is best-of-n the best of them? Coverage, scaling, and optimality in inference-time alignment",
       "authors": ["Audrey Huang", "Adam Block", "Qinghua Liu"],
       "year": 2025,
       "relevance": "Inference-time alignment via best-of-n sampling and coverage properties, directly related to aggregation at test time."
     },
     {
       "title": "CoT-based synthesizer: Enhancing LLM performance through answer synthesis",
       "authors": ["Bohan Zhang", "Xiaokang Zhang", "Jing Zhang"],
       "year": 2025,
       "relevance": "Answer synthesis as a form of output aggregation across multiple LLM reasoning traces."
     },
     {
       "title": "How do classifiers induce agents to invest effort strategically?",
       "authors": ["Jon Kleinberg", "Manish Raghavan"],
       "year": 2020,
       "relevance": "The principal-agent framework that this paper extends to multiple agents and aggregation, foundational to the theoretical model."
     }
   ]
 }