ai-research-survey

scan.json (26841B)

---

 {
   "paper": {
     "title": "Institutional AI: Governing LLM Collusion in Multi-Agent Cournot Markets via Public Governance Graphs",
     "authors": ["Marcantonio Bracale", "Federico Pierucci", "Marcello Galisai", "Matteo Prandi", "Piercosma Bisconti", "F. Giarrusso", "O. Sorokoletova", "V. Suriani", "D. Nardi"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.11369",
     "doi": "10.48550/arXiv.2601.11369"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Institutional governance (governance-graph-based enforcement) substantially reduces LLM collusion in Cournot markets: mean collusion tier drops from 3.1 to 1.8 (Cohen's d=1.28), and severe-collusion incidence drops from 50% to 5.6% across 6 model configurations (N=90 runs/condition). Prompt-only Constitutional baselines yield no reliable improvement over ungoverned conditions, suggesting declarative prohibitions do not bind under optimization pressure. The effect holds across homogeneous and heterogeneous model pairs including cross-provider configurations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'results artifacts' containing emitted manifests and logs but provides no download link or public repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or library version listing is provided. Models are named but no environment setup details given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. The methodology section describes the pipeline conceptually but lacks executable reproduction details."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports mean ± SD for all metrics across 90 runs per condition."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Two-sided Welch t-tests with p-values reported in Table 6; two-proportion z-tests for tier shares; paired sign-flip permutation tests (p=0.0312) for cross-configuration inference (Section 5.5, Section 7)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d reported for all continuous endpoints in Table 6 (d=1.28, 1.05, 1.51, etc.), with explicit magnitude interpretation ('large by conventional standards')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "N=90 runs/condition (5 runs/label/batch/condition × 6 labels × 3 batches) is stated but no power analysis or justification for why 5 runs per cell is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations reported across runs in Table 6 (e.g., '3.10 ± 1.06'). IQR also reported for median tier (Section 7)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines: Ungoverned (replication of Lin et al. 2024) and Constitutional (prompt-only prohibition). Both clearly defined in Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Constitutional baseline draws on Palla et al. (2025) and Hua et al. (2024) policy-as-prompt approaches. Ungoverned replicates Lin et al. (2024). These are contemporary references."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.2.3 mentions a 'small factorial ablation matrix around the selected institution to isolate the marginal contribution of core governance levers (fine salience, credit timing, tier persistence, and credit budget)' but results of this ablation are not reported in the paper."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: collusion tier, HHI excess, CV excess (max and mean), tier share percentages (≥3, ≥4), profit excess, total profit. All reported in Table 6."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a computational experiment with LLM agents in a simulated market. Human evaluation of outputs is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a prediction task with train/test splits. It is a simulation experiment comparing governance regimes."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 and Figure 5 provide per-model-configuration breakdowns of mean collusion tier across all 6 configurations and 3 conditions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 notes Constitutional baseline sometimes increases collusion (GPT-5 Mini: 2.93→3.60). Section 9 discusses governance brittleness and Goodhart risk. 5.6% severe collusion remains under Institutional regime."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Constitutional baseline yields no reliable improvement — this is a key negative finding. GPT-5 Mini Constitutional tier exceeding Ungoverned is reported (Table 7). 244 suspension requests denied is also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: mean tier 3.1→1.8 (d=1.28), severe collusion 50%→5.6%, Constitutional yields no reliable improvement. All supported by Table 6 and Section 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about Institutional governance suppressing collusion. The experimental design (3 conditions × 6 configurations × 3 batches, randomized) with controlled single-variable manipulation (governance regime) supports causal inference within the simulated setting."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 explicitly states limitations: 'two-firm Cournot abstracts away contracts, asymmetric information, richer strategic instruments, and endogenous entry/exit.' The abstract uses 'may benefit from' and 'these results suggest' language. Title specifies 'Cournot Markets.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 discusses governance brittleness (Goodhart risk), that fixed proxy thresholds may be gamed. Section 8 considers whether compliance is incentive-based vs norm-following. The paper also addresses whether model heterogeneity drives results (it doesn't; Section 7)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines collusion through market-structure proxies (CV excess, HHI excess) in Section 5.3, derives them from Cournot-Nash benchmarks, and maps them to a discrete tier system (Table 1). The proxy relationship between these metrics and actual collusion is discussed, with normalization relative to Nash equilibrium."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'GPT-5 Mini', 'Grok-4 Fast', 'Gemini 2.5 Flash' — marketing names without snapshot dates or API versions. Lin et al. (2024) specified 'GPT-4o-2024-08-06' but the present paper does not follow this practice for its own models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the full agent decision prompt template with actual structure. Appendix B provides the full Constitutional text. Appendix C provides verbatim Institutional notice examples. Appendix D references the manifest. The prompts are substantially complete."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Lin et al. (2024) used temperature=1, but this paper does not state temperature or sampling parameters for its own model configurations. Market parameters (α, β, κ) are stated but LLM API hyperparameters are not."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes the agent architecture in detail: rolling 30-round market trace, PLANS/INSIGHTS memory, structured JSON output with retries, feasibility enforcement. Section 6.2 details the Oracle/Controller governance engine. Figure 1 shows the pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 describes the full execution flow: environment reset, manifest emission, per-round cycle (notice → proposal → Oracle/Controller → market clearing → memory update). Post-horizon processing (Nash computation via SLSQP, excess ratio derivation, tier assignment) is specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Limitations and Further Research' provides substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 identifies specific threats: two-firm Cournot is narrow (no contracts, asymmetric information, entry/exit); fixed proxy thresholds may be Goodharted; no inter-firm communication tested; limited model zoo."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 explicitly states what was not tested: n≥3 firms, explicit inter-firm communication, multilingual models, broader coordination domains. Section 1 states the paper focuses on 'implementation, operational definitions, and empirical results' not conceptual foundations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'results artifacts' and 'full emitted manifests and logs are included in the results artifacts' (Appendix D) but provides no download link or public access."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and 6.3 describe the experimental setup: 6 model configs × 3 conditions × 3 batches × 5 runs, with fixed market parameters and the execution pipeline fully specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from LLM agent simulations using standard public API models."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 documents the full pipeline: environment initialization → per-round execution (governance notice → agent proposal → Oracle/Controller → market clearing → memory update) → post-horizon computation of reference solutions and metrics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement, acknowledgments section, or grant disclosures found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: DEXAI–Icaro Lab, Sapienza University of Rome, Sant'Anna School of Advanced Studies, VU Amsterdam."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed. Authors are affiliated with DEXAI–Icaro Lab which may have interests in the governance framework being proposed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate model capability on a benchmark. It tests governance regimes' effects on LLM agent behavior in a simulated market. Training cutoff is irrelevant — the agents are making strategic decisions, not recalling memorized answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model knowledge. The Cournot game is a novel interactive setting, not a static test set."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no static benchmark is used. The experimental setting is a dynamic multi-round game."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Experiment is a computational simulation with LLM agents."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or per-run cost reported despite running 270 runs (90/condition × 3 conditions) each with 50 rounds of LLM calls across 6 different models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, compute hours, or hardware specifications reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are aggregated over 3 independent batches of 5 runs each per label/condition (N=90/condition), with SD reported. This effectively tests sensitivity across runs, though 'seed' is not explicitly used since LLM APIs handle randomness internally."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: '5 runs/label/batch/condition; N = 90 runs/condition' across 'three independent batches' (Section 5.5)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.2.3 mentions 'a low-replication screening sweep over a small candidate set' and 'a small factorial ablation matrix' for parameter selection, but the number of configurations tried and compute spent on search are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2.3 describes a staged procedure: screening sweep → robustness check on second model family → factorial ablation → manifest locked by semantic digest before final runs. Selection criterion (eliminate configurations failing to reduce severe outcomes) is stated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple statistical tests are reported (Welch t-tests on 4 continuous metrics, z-tests on 2 tier shares, permutation tests) without mention of Bonferroni, Holm, or other family-wise error rate corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the Institutional governance system and evaluate it against baselines. No discussion of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All three regimes use the same underlying LLM calls per round. The Institutional regime adds deterministic Oracle/Controller computation which is negligible. Compute differences are negligible across conditions."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 explicitly defines collusion through market-structure metrics (HHI, CV) with theoretical grounding in Cournot-Nash equilibrium benchmarks. The proxy nature of the tier system is discussed. Section 5.2 justifies Cournot competition as a testbed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models use the same agent architecture (Section 6.1): same prompt structure, same memory system, same output schema. The governance regime modifies only the MARKET GOVERNANCE prompt block. Cross-model comparisons use identical scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model knowledge. The Cournot game is a novel interactive simulation without pre-existing solutions to memorize."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a prediction task. Agents make strategic decisions in a simulated market; there is no answer to leak."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no train/test split. Each run is an independent simulation."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation with leakage risk. The experimental setting is a dynamic game, not a static test set."
       }
     }
   },
   "claims": [
     {
       "claim": "Institutional governance reduces mean collusion tier from 3.1 to 1.8 (Cohen's d = 1.28) relative to Ungoverned baseline.",
       "evidence": "Table 6: mean tier U=3.10±1.06 vs Inst=1.82±0.93, Welch p=4.67e-15, d=1.28. N=90 runs/condition across 6 model configs and 3 batches.",
       "supported": "strong"
     },
     {
       "claim": "Severe-collusion incidence drops from 50% to 5.6% under Institutional governance.",
       "evidence": "Table 6: Tier≥4 rate U=50.0% vs Inst=5.6%, two-proportion z-test p=2.81e-11.",
       "supported": "strong"
     },
     {
       "claim": "Prompt-only Constitutional baseline yields no reliable improvement over Ungoverned.",
       "evidence": "Table 6: Constitutional tier=3.02±1.05 vs Ungoverned=3.10±1.06. Table 7 shows Constitutional sometimes increases collusion (GPT-5 Mini: 2.93→3.60).",
       "supported": "strong"
     },
     {
       "claim": "The institutional effect holds across all 6 model configurations including heterogeneous cross-provider pairs.",
       "evidence": "Table 7: Institutional reduces tier in every configuration. Permutation test on 6 labels: p=0.0312.",
       "supported": "strong"
     },
     {
       "claim": "Model heterogeneity is not a first-order driver of collusion in this setting.",
       "evidence": "Table 7: heterogeneous pairs show broadly similar tiers to homogeneous duopolies. Section 7 states no evidence that cross-provider heterogeneity disrupts collusion.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Authors evaluate their own system",
       "detail": "The authors designed the Institutional AI framework and governance graph, then evaluate it against baselines. No independent replication or third-party evaluation. The parameter selection procedure (Section 6.2.3) involves tuning the institution to work well before final evaluation, creating researcher degrees of freedom."
     },
     {
       "flag": "No code or data release",
       "detail": "Despite 270 runs producing detailed governance logs and market traces, no code repository or data archive is provided. The paper mentions 'results artifacts' but gives no public access."
     },
     {
       "flag": "Missing API hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters for the 6 model configurations are not reported. LLM behavior in strategic settings is highly sensitive to temperature, making this a significant omission."
     },
     {
       "flag": "No cost reporting",
       "detail": "270 runs × 50 rounds × 2 agents = 27,000 LLM calls minimum across 6 different commercial models. Total API cost is unreported."
     },
     {
       "flag": "Ablation results not shown",
       "detail": "Section 6.2.3 mentions 'a small factorial ablation matrix' was run to isolate governance lever contributions, but the results are not reported. The reader cannot assess which components of the institution drive the effect."
     }
   ],
   "cited_papers": [
     {
       "title": "Strategic collusion of LLM agents: Market division in multi-commodity competitions",
       "authors": ["R. Y. Lin", "S. M. Ojha", "K. Cai", "M. F. Chen"],
       "year": 2024,
       "arxiv_id": "2410.00031",
       "relevance": "Primary replication target; demonstrates LLM agents collude in Cournot markets without explicit instructions."
     },
     {
       "title": "Alignment faking in large language models",
       "authors": ["R. Greenblatt", "C. Denison", "B. Wright"],
       "year": 2024,
       "arxiv_id": "2412.14093",
       "relevance": "Shows LLMs can fake alignment, motivating external governance over internal alignment."
     },
     {
       "title": "AI control: Improving safety despite intentional subversion",
       "authors": ["R. Greenblatt", "B. Shlegeris", "K. Sachan", "F. Roger"],
       "year": 2024,
       "arxiv_id": "2312.06942",
       "relevance": "Control-oriented approach to AI safety; separates detection from enforcement, related to Oracle/Controller design."
     },
     {
       "title": "Multi-agent risks from advanced AI",
       "authors": ["L. Hammond", "A. Chan", "J. Clifton"],
       "year": 2025,
       "arxiv_id": "2502.14143",
       "relevance": "Risk taxonomy for multi-agent AI including collusion, miscoordination, and conflict failure modes."
     },
     {
       "title": "Beyond single-agent safety: A taxonomy of risks in LLM-to-LLM interactions",
       "authors": ["P. Bisconti", "M. Galisai", "F. Pierucci", "M. Bracale", "M. Prandi"],
       "year": 2025,
       "arxiv_id": "2512.02682",
       "relevance": "Taxonomy of systemic risks in multi-agent LLM systems from the same research group."
     },
     {
       "title": "Sycophancy to subterfuge: Investigating reward-tampering in large language models",
       "authors": ["C. Denison", "M. MacDiarmid", "F. Barez"],
       "year": 2024,
       "arxiv_id": "2406.10162",
       "relevance": "Demonstrates reward tampering in LLMs, motivating external governance over reward-based alignment."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Shows deceptive policies can survive safety training, motivating runtime governance."
     },
     {
       "title": "Hidden in plain text: Emergence and mitigation of steganographic collusion in LLMs",
       "authors": ["Y. Mathew", "O. Matthews", "R. McCarthy"],
       "year": 2024,
       "arxiv_id": "2410.03768",
       "relevance": "Demonstrates steganographic collusion channels between LLM agents."
     },
     {
       "title": "Policy-as-prompt: Rethinking content moderation in the age of large language models",
       "authors": ["K. Palla", "J. L. R. García", "C. Hauff"],
       "year": 2025,
       "relevance": "Evaluates prompt-based policy enforcement for LLMs; Constitutional baseline in this paper draws on this approach."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "MAST taxonomy of 14 failure modes in multi-agent LLM systems with 1600+ annotated traces."
     },
     {
       "title": "Artificial intelligence, algorithmic pricing, and collusion",
       "authors": ["E. Calvano", "G. Calzolari", "V. Denicolò", "S. Pastorello"],
       "year": 2020,
       "relevance": "Foundational work showing Q-learning agents converge to supra-competitive prices without explicit communication."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": ["Y. Bai", "S. Kadavath", "S. Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Constitutional AI training technique; this paper's RLINF proposal is positioned as an institutional analogue."
     }
   ]
 }