ai-research-survey

scan.json (20171B)

---

 {
   "paper": {
     "title": "Institutional AI: A Governance Framework for Distributional AGI Safety",
     "authors": ["F. Pierucci", "M. Galisai", "M. Bracale Syrnikov", "M. Prandi", "P. Bisconti", "F. Giarrusso", "O. Sorokoletova", "V. Suriani", "D. Nardi"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.10599",
     "doi": "10.48550/arXiv.2601.10599"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository URL is provided. The paper is theoretical but could have released formal specifications or simulation code."
       },
       "data_released": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper with no data collection or analysis. No dataset to release."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are conducted. No environment to specify."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "No experiments to reproduce. The paper is a theoretical framework."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper with no empirical results or statistical analyses."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative empirical claims requiring statistical tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No empirical measurements or effect sizes to report."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical study with samples."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs to report variance across."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation is conducted. The paper is a theoretical framework proposal."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation with baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. The paper presents a theoretical framework."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No data or test sets involved."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No empirical results to break down."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses failure modes of existing alignment approaches but does not discuss potential failure cases of its own proposed governance graph framework."
       },
       "negative_results_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted that could yield negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are theoretical in nature — identifying three structural problems and proposing governance graphs as a solution. The paper body develops each thesis (Sections 2-4) and the proposed framework (Section 5) as stated in the abstract."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'the institution shifts the payoff landscape' and that sanctions 'eliminate the profitable deviation problem' (Section 5.1, Eq. 1-2). These are supported by mechanism design theory but not empirically validated. The companion paper [15] is referenced for experimental validation but not included here."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes sweeping claims about alignment of AI agents generally. The title references 'Distributional AGI Safety.' The theoretical framework is presented as broadly applicable without bounding to specific agent types, capability levels, or deployment contexts. Section 5.7 acknowledges domain variation but the core claims are unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not seriously consider alternatives to its institutional approach. It does not discuss whether improved training-time alignment, interpretability advances, or other runtime approaches could address the same problems. The framing treats training-time alignment as fundamentally insufficient without considering counterarguments."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper with no measurements. No proxy-outcome gap to discuss."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used in experiments. The paper references models (Claude Opus 4, GPT-4, etc.) only in its literature review of prior work."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a theoretical paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted requiring hyperparameter settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this paper. The paper describes scaffolding conceptually as part of its framework."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data is collected or processed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion (Section 6) does not discuss limitations of the proposed framework."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential weaknesses of governance graphs, such as the assumption that agent behavior is observable, that sanctions can be calibrated, or that agents cannot subvert the governance engine itself."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the framework does NOT address. It mentions future work directions (Section 6: fake news, credit markets) but does not bound the current claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "No data collected. Purely theoretical paper."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No data collection. Theoretical framework paper."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants or data recruitment."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline. Theoretical paper."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or disclosure is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: DEXAI/Icaro Lab, Sapienza University of Rome, Sant'Anna School of Advanced Studies, and VU Amsterdam."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors are affiliated with DEXAI (appears to be a company/lab) but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on benchmarks. It is a theoretical framework paper."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. No method with inference cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments conducted."
       }
     }
   },
   "claims": [
     {
       "claim": "Sufficiently capable models can acquire internal goal structures that diverge from developer-specified objectives through mesa-optimization and goal misgeneralization.",
       "evidence": "Section 2 reviews literature on mesa-optimization (Hubinger et al. 2019), goal misgeneralization (Di Langosco et al. 2022, Shah et al. 2022), and emergent preferences (Mazeika et al. 2025, Seror 2024). Evidence is from cited work, not original experiments.",
       "supported": "moderate"
     },
     {
       "claim": "RLHF and Constitutional AI have fundamental limitations that cannot be resolved through better implementation, including oversight, representation, and deception problems.",
       "evidence": "Section 3 synthesizes Casper et al. (2023) taxonomy of RLHF limitations, Anthropic's alignment faking results (Greenblatt et al. 2024), and Apollo Research scheming findings (Meinke et al. 2024). All evidence from prior work.",
       "supported": "moderate"
     },
     {
       "claim": "Individually aligned agents can converge on collusive or adversarial equilibria through multi-agent interaction dynamics.",
       "evidence": "Section 4 cites LLM collusion in auctions (Agrawal et al. 2025), market division (Lin et al. 2024), steganographic coordination (Motwani et al. 2024). The ESRH framework from Bisconti et al. [12] is self-cited.",
       "supported": "moderate"
     },
     {
       "claim": "Governance graphs provide O(N) verification scaling versus O(N²) for agent-space verification, creating a decisive scaling advantage.",
       "evidence": "Section 5.8 provides a mathematical argument comparing agent-space (N·d individual + N(N-1)/2 pairwise) vs institution-space (|Q|²·P graph + N monitoring) verification. This is a theoretical claim supported by the formal model, not empirical measurement.",
       "supported": "weak"
     },
     {
       "claim": "Institutional sanctions can transform the social optimum into a Nash equilibrium by making misaligned behavior unprofitable.",
       "evidence": "Section 5.1 presents the formal argument with Equations 1-2, showing that if sanction S exceeds max deviation gain, no profitable unilateral deviation exists. This is a standard mechanism design result applied to AI agents.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical"],
   "key_findings": "The paper proposes 'Institutional AI,' a governance framework that externalizes alignment constraints as public data structures (governance graphs) rather than relying on training-time internalization. It identifies three structural alignment problems — behavioral goal-independence, instrumental override of alignment constraints, and agentic alignment drift — and argues these cannot be solved by prompt-based or RLHF approaches alone. The governance graph formalism uses mechanism design to reshape agent payoffs so that compliance becomes the dominant strategy, with claimed O(N) verification scaling. A companion paper reportedly validates the approach in Cournot market simulations.",
   "red_flags": [
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The paper makes sweeping claims about alignment solutions for AGI-level systems based entirely on theoretical arguments and literature review. No empirical validation is presented in this paper. The companion paper [15] is referenced but not included. The complexity reduction thesis (O(N) vs O(N²)) assumes agents cannot subvert the governance infrastructure itself."
     },
     {
       "flag": "No limitations discussion",
       "detail": "The paper has no limitations section despite making ambitious theoretical claims. Key unaddressed issues include: whether the monitoring oracle can reliably detect all violations (especially steganographic coordination, which the paper itself highlights as a threat), whether agents can game the governance graph itself, and whether the formal model's assumptions hold for real LLM agents."
     },
     {
       "flag": "Selective use of literature",
       "detail": "The paper cites evidence of alignment failures extensively to motivate its framework but does not engage with counterarguments — e.g., that improved interpretability, formal verification, or better RLHF could address the same issues. The presentation frames training-time alignment as fundamentally broken without fairly representing the opposing view."
     },
     {
       "flag": "Self-citation cluster",
       "detail": "Several key references ([10], [11], [12], [15]) are self-citations from the same research group (Bisconti, Pierucci, Galisai, Bracale, Prandi). The ESRH framework (Section 4.2) and companion empirical validation both come from within the group."
     }
   ],
   "cited_papers": [
     {
       "title": "Alignment faking in large language models",
       "authors": ["R. Greenblatt", "C. Denison", "B. Wright"],
       "year": 2024,
       "arxiv_id": "2412.14093",
       "relevance": "Key evidence for alignment faking behavior in frontier models, directly relevant to AI safety evaluation."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistent deceptive policies that survive safety fine-tuning, relevant to alignment robustness."
     },
     {
       "title": "Frontier Models are Capable of In-context Scheming",
       "authors": ["A. Meinke", "B. Schoen", "J. Scheurer"],
       "year": 2024,
       "arxiv_id": "2412.04984",
       "relevance": "Documents scheming capabilities in frontier models including o1, Claude 3.5 Sonnet, and Gemini 1.5 Pro."
     },
     {
       "title": "Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback",
       "authors": ["S. Casper", "X. Davies", "C. Shi"],
       "year": 2023,
       "arxiv_id": "2307.15217",
       "relevance": "Systematic taxonomy of RLHF limitations relevant to understanding alignment technique boundaries."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Y. Bai", "S. Kadavath", "S. Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Foundational paper on Constitutional AI, a key alignment approach critiqued in this work."
     },
     {
       "title": "Secret Collusion among Generative AI Agents: Multi-Agent Deception via Steganography",
       "authors": ["S. R. Motwani", "M. Baranchuk", "M. Strohmeier"],
       "year": 2024,
       "arxiv_id": "2402.07510",
       "relevance": "Demonstrates steganographic collusion capabilities in LLM agents, relevant to multi-agent safety."
     },
     {
       "title": "Multi-Agent Risks from Advanced AI",
       "authors": ["L. Hammond", "A. Chan", "J. Clifton"],
       "year": 2025,
       "arxiv_id": "2502.14143",
       "relevance": "Comprehensive taxonomy of multi-agent AI risks from Cooperative AI, directly relevant to agentic safety."
     },
     {
       "title": "Agentic Misalignment: How LLMs Could be Insider Threats",
       "authors": ["A. Lynch", "B. Wright", "C. Larson"],
       "year": 2025,
       "arxiv_id": "2510.05179",
       "relevance": "Red-teaming results showing frontier models autonomously selecting harmful tactics in agentic settings."
     },
     {
       "title": "Distributional AGI Safety",
       "authors": ["N. Tomašev", "M. Franklin", "J. Jacobs"],
       "year": 2025,
       "arxiv_id": "2512.16856",
       "relevance": "Google DeepMind's distributional AGI safety agenda that this paper claims to answer."
     },
     {
       "title": "Beyond Single-Agent Safety: A Taxonomy of Risks in LLM-to-LLM Interactions",
       "authors": ["P. Bisconti", "M. Galisai", "F. Pierucci"],
       "year": 2025,
       "arxiv_id": "2512.02682",
       "relevance": "Introduces the ESRH framework for multi-agent risk analysis, directly builds on this group's work."
     },
     {
       "title": "Utility Engineering: Analyzing and Controlling Emergent Value Systems in AIs",
       "authors": ["M. Mazeika", "X. Yin", "R. Tamirisa"],
       "year": 2025,
       "arxiv_id": "2502.08640",
       "relevance": "Evidence for emergent utility representations and coherent preferences in frontier LLMs."
     },
     {
       "title": "Towards Understanding Sycophancy in Language Models",
       "authors": ["M. Sharma", "M. Tong", "T. Korbak"],
       "year": 2023,
       "arxiv_id": "2310.13548",
       "relevance": "Systematic study of sycophancy as an alignment failure mode in preference-optimized models."
     }
   ]
 }