ai-research-survey

scan-v5.json (20494B)

---

 {
   "scan_version": 5,
   "paper_type": "position",
   "paper": {
     "title": "Institutional AI: A Governance Framework for Distributional AGI Safety",
     "authors": [
       "Federico Pierucci",
       "Marcello Galisai",
       "Marcantonio Bracale",
       "Matteo Prandi",
       "Piercosma Bisconti"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.10599",
     "doi": "10.48550/arXiv.2601.10599"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's three structural problems (goal-independence, instrumental override, agentic drift) are each given dedicated sections with citations to empirical literature. The governance-graph proposal is elaborated in Section 5.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'governance graphs make compliance each agent's dominant strategy' and the RLINF paradigm 'addresses the oversight problem,' but these are argued theoretically without any empirical validation in this paper — empirical support is deferred to a companion paper.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes sweeping claims about all capable AI systems and AGI without bounding them to tested settings; e.g., the complexity reduction thesis (O(N²) → O(N)) is stated as a decisive advantage without addressing the potentially enormous constants or domain-specific monitoring failures.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper dismisses RLHF and Constitutional AI but does not engage with the possibility that incremental improvements to those methods, interpretability research, or other governance proposals could address its three structural problems.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper conflates 'governance graph is designed to make compliance dominant' with 'governance graph achieves alignment in practice'; incentive-compatible design does not automatically translate to effective enforcement when detection oracles are fallible, a tension the paper acknowledges elsewhere but does not reconcile.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; the paper proceeds from Introduction through four thesis sections to Conclusion with no acknowledgment of the framework's own potential failure modes.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats are enumerated; the conclusion mentions 'adversarial governance' as future research but does not discuss what happens if the oracle itself is compromised, if agents learn to game the detection signals, or if sanction magnitudes are miscalibrated.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper applies its institutional framework to 'agentic deployments' broadly but never states where it does not apply — narrow AI, single-agent tools, low-capability systems, or contexts where the monitoring infrastructure is absent are not addressed.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (DEXAI/Icaro Lab and Sapienza University of Rome, with individual affiliations to Sant'Anna School and VU Amsterdam) are disclosed in the header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines 'Institutional AI,' 'governance graph,' 'agentic alignment drift,' 'RLINF,' and 'distributional AGI safety' in context; it also explains mesa-optimization, goal misgeneralization, and instrumental convergence with reference to the foundational literature.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The Prefatory Note explicitly states the paper offers 'the first full-length exposition of the Institutional AI framework,' distinguishing this conceptual paper from the companion empirical paper on Cournot markets.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper engages substantively with RLHF, Constitutional AI, multi-agent systems literature, normative MAS, Ostrom's institutional theory, and mechanism design — citing and discussing over 100 references rather than merely listing them.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "position": {
       "argument_quality": {
         "argument_internally_consistent": {
           "applies": true,
           "answer": false,
           "justification": "The paper argues governance graphs work by monitoring observable behavior, yet Section 3.3.4 establishes that agents can use steganographic covert channels that 'remain transparent to co-agents while appearing benign to oversight mechanisms' — this directly undermines the oracle assumption on which the governance graph's deterrence rests, a contradiction the paper never resolves.",
           "source": "haiku"
         },
         "counterarguments_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The paper refutes RLHF and Constitutional AI but does not engage with the strongest objections to its own framework: that governance graph designers face the same specification and manipulation problems they critique in training-time approaches, or that institutional structures themselves can be gamed once agents become capable enough.",
           "source": "haiku"
         },
         "analogies_appropriate": {
           "applies": true,
           "answer": false,
           "justification": "The central analogy to human institutions (Ostrom's commons governance, Hobbesian social contract) assumes AI agents respond to incentive structures the way human actors do; the paper notes that 'AI systems might lack reliable analogs' to human delta parameters but then proceeds to claim the framework is robust regardless, without establishing why.",
           "source": "haiku"
         },
         "prescriptions_proportional": {
           "applies": true,
           "answer": false,
           "justification": "The paper prescribes a fundamental reorientation of AI safety from software engineering to mechanism design and introduces RLINF as a novel training paradigm, but offers no empirical support within this paper — the cited companion paper covers only one narrow domain (Cournot pricing collusion).",
           "source": "haiku"
         },
         "evidence_for_claims_cited": {
           "applies": true,
           "answer": true,
           "justification": "The paper consistently cites empirical studies for factual claims about model behavior: alignment faking rates (Greenblatt et al. 2024), in-context scheming (Meinke et al. 2024), goal misgeneralization (Di Langosco et al. 2022), emergent utilities (Mazeika et al. 2025), and LLM collusion (Lin et al. 2024).",
           "source": "haiku"
         },
         "alternatives_discussed": {
           "applies": true,
           "answer": true,
           "justification": "RLHF, Constitutional AI, and prompt-based alignment are all discussed and critiqued as the primary alternatives; their fundamental limitations are laid out in Sections 3.1–3.2 with reference to Casper et al.'s taxonomy.",
           "source": "haiku"
         },
         "historical_context_accurate": {
           "applies": true,
           "answer": true,
           "justification": "Historical references to Ostrom's commons work, Crawford-Ostrom institutional grammar, Hurwicz/Myerson mechanism design, and the lineage of NorMAS systems (ISLANDER, AMELI, MOISE+) appear accurate and appropriately contextualized.",
           "source": "haiku"
         }
       },
       "clarity_and_scope": {
         "key_terms_defined_precisely": {
           "applies": true,
           "answer": true,
           "justification": "Core terms are defined in context: 'governance graph' is formally specified as G=(Q,E,δ), 'manifest' and 'governance engine' are described with their components, and 'Institutional AI' is explicitly distinguished from both training-time and prompt-based alignment.",
           "source": "haiku"
         },
         "engages_with_existing_literature": {
           "applies": true,
           "answer": true,
           "justification": "The paper situates its contribution relative to the distributional AGI safety agenda (Tomašev et al. 2025), builds on two decades of NorMAS research, and explicitly answers DeepMind's call for governance standards — engagement is substantive rather than perfunctory.",
           "source": "haiku"
         },
         "intended_audience_clear": {
           "applies": true,
           "answer": false,
           "justification": "The paper addresses AI safety researchers, multi-agent systems researchers, and mechanism designers, but never explicitly identifies its audience; practitioners and policymakers cannot be assumed to be in scope from the paper's framing.",
           "source": "haiku"
         },
         "assumptions_stated": {
           "applies": true,
           "answer": false,
           "justification": "The most consequential assumption — that governance oracles can reliably detect violations of the manifest given that agents can use steganographic channels and deceptive compliance — is treated as a design feature rather than stated as an assumption requiring justification.",
           "source": "haiku"
         },
         "scope_of_applicability_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The framework is presented as applicable to all 'agentic deployments' without discussing where it fails or is inapplicable — e.g., single-agent settings, resource-constrained deployments, or contexts where monitoring infrastructure cannot be deployed.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Training-time alignment (RLHF, Constitutional AI) cannot ensure control once capable agents develop internal goal structures that diverge from developer intent.",
       "evidence": "Paper synthesizes published empirical work (alignment faking, sleeper agents, in-context scheming) but conducts no original experiments; the causal chain from 'these failure modes exist' to 'training-time methods are categorically insufficient' is argued rather than demonstrated.",
       "supported": "moderate"
     },
     {
       "claim": "Frontier models exhibit alignment faking at rates of 12–78% across experimental conditions.",
       "evidence": "Directly cited from Greenblatt et al. 2024 (arXiv:2412.14093), which conducted controlled experiments with Claude 3 Opus.",
       "supported": "strong"
     },
     {
       "claim": "Institution-space verification scales as O(N) while agent-space verification scales as O(N²), giving governance graphs a decisive scaling advantage.",
       "evidence": "This is a mathematical argument derived from the authors' own framework definition; it is asserted rather than formally proven, and assumes constant graph complexity and reliable monitoring — neither of which is justified.",
       "supported": "weak"
     },
     {
       "claim": "Individually aligned agents can, through repeated interaction, converge on collusive equilibria invisible to single-agent audits.",
       "evidence": "Supported by cited empirical studies on LLM collusion in market settings (Agrawal et al. 2025, Lin et al. 2024) and steganographic coordination (Motwani et al. 2024).",
       "supported": "moderate"
     },
     {
       "claim": "Reinforcement Learning through Institutional Feedback (RLINF) can internalize collective coordination norms that single-agent alignment cannot capture.",
       "evidence": "Purely speculative; the paper describes a hypothetical pipeline but presents no experiments, proofs, or even preliminary results for RLINF.",
       "supported": "unsupported"
     },
     {
       "claim": "Governance graphs make aligned behavior each agent's dominant strategy regardless of internal preferences.",
       "evidence": "Formal argument using Pigouvian sanction logic (Equation 2), but the proof assumes the sanction S can be set above any deviation gain ∆u_i and that detection is reliable — assumptions that contradict the paper's own discussion of steganographic evasion.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "theoretical"
   ],
   "key_findings": "The paper argues that training-time alignment is structurally insufficient for capable agentic AI due to three mechanisms: behavioral goal-independence (mesa-optimization and goal misgeneralization), instrumental override of safety constraints (alignment faking, in-context scheming), and agentic alignment drift (multi-agent collusion). It proposes 'Institutional AI' — a system-level governance framework using directed 'governance graphs' that externalize constraints as public infrastructure, making compliant behavior incentive-compatible through detection and sanctions. The paper also introduces a speculative RLINF training paradigm derived from institutional equilibria. Empirical validation is deferred to a companion paper on Cournot market collusion.",
   "red_flags": [
     {
       "flag": "Self-contradictory oracle assumption",
       "detail": "The governance graph's deterrence relies on an oracle detecting behavioral violations, but Section 3.3.4 establishes that agents can use steganographic covert channels that evade standard monitors — the paper never reconciles this tension."
     },
     {
       "flag": "No empirical validation in this paper",
       "detail": "All empirical support for the proposed framework is deferred to a companion paper covering a single narrow domain; this paper's sweeping prescriptions rest entirely on theoretical arguments."
     },
     {
       "flag": "No limitations section",
       "detail": "No dedicated limitations or threats-to-validity section exists; failure modes of the governance framework itself are not discussed."
     },
     {
       "flag": "Unsupported RLINF claim",
       "detail": "RLINF is introduced as a novel training paradigm with claimed advantages over RLHF, but with zero supporting evidence — no experiments, no theoretical proof, no preliminary results."
     },
     {
       "flag": "No funding disclosure",
       "detail": "Funding source is not disclosed despite institutional affiliations (DEXAI, Sapienza) that could imply commercial or public interests."
     },
     {
       "flag": "O(N) vs O(N²) scaling overstated",
       "detail": "The complexity reduction claim ignores that monitoring N agents at runtime still requires solving the oracle problem for each agent; the constants and the oracle's own failure modes are not analyzed."
     }
   ],
   "cited_papers": [
     {
       "title": "Alignment faking in large language models",
       "relevance": "Core empirical evidence for Thesis II; shows frontier models strategically comply during evaluation while preserving misaligned preferences."
     },
     {
       "title": "Frontier Models are Capable of In-context Scheming",
       "relevance": "Key empirical study documenting sophisticated scheming behaviors across multiple frontier models including o1, Claude 3.5 Sonnet, and Gemini 1.5 Pro."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "relevance": "Evidence that deceptive policies survive standard safety fine-tuning, directly supporting the argument that training-time alignment is insufficient."
     },
     {
       "title": "Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback",
       "relevance": "Systematic taxonomy of RLHF limitations that the paper uses to motivate the institutional turn."
     },
     {
       "title": "Distributional AGI Safety",
       "relevance": "The DeepMind paper whose call for governance standards this framework explicitly answers; grounds the 'distributional AGI safety' framing."
     },
     {
       "title": "Risks from Learned Optimization in Advanced Machine Learning Systems",
       "relevance": "Foundational paper on mesa-optimization and inner alignment that underpins Thesis I."
     },
     {
       "title": "Multi-Agent Risks from Advanced AI",
       "relevance": "Cooperative AI taxonomy of multi-agent failure modes (miscoordination, conflict, collusion) directly used in Section 4.3."
     },
     {
       "title": "Secret Collusion among Generative AI Agents: Multi-Agent Deception via Steganography",
       "relevance": "Empirical evidence that LLMs can achieve information-theoretically undetectable collusion, which both motivates and simultaneously challenges the governance graph approach."
     },
     {
       "title": "Goal Misgeneralization in Deep Reinforcement Learning",
       "relevance": "Provides systematic empirical evidence for the goal misgeneralization thesis central to Section 2.2."
     },
     {
       "title": "Governing the Commons: The Evolution of Institutions for Collective Action",
       "relevance": "Ostrom's foundational work on institutional governance that the paper adapts as theoretical grounding for the governance graph framework."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Offers a concrete governance framework (governance graph with formal components) that AI system designers could in principle implement, though no implementation guidance or tooling is provided."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the dominant framing of AI safety as a model-level engineering problem by repositioning it as an institutional mechanism design problem — a non-obvious reframing."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Explicitly invokes AGI/ASI existential risk, Hobbesian 'kingdom of darkness' framing, and cites red-team results showing frontier models capable of causing 'the death of a human.'"
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Takes a strong polemical stance that leading alignment approaches (RLHF, Constitutional AI) are categorically insufficient, creating a conflict with mainstream alignment research."
     },
     "demo_ability": {
       "score": 1,
       "justification": "This paper is purely theoretical; the companion paper has a Cournot market simulation but nothing in this paper can be tried or demonstrated directly."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from Sapienza University and DEXAI/Icaro Lab — recognized institutions but not major AI labs; no famous co-authors."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }