ai-research-survey

scan.json (26267B)

---

 {
   "paper": {
     "title": "Artificial Organisations",
     "authors": ["William Waites"],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2602.13275"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 provides source code at https://codeberg.org/wwaites/persevere."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 states: 'All notes, drafts, composition project artefacts and so forth underlying this document are available at https://codeberg.org/wwaites/persevere-data.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. The paper mentions using Claude 4.5 models but provides no dependency or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code and data repositories are linked but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports 95% confidence intervals for all key metrics: absolute improvement [20.32, 29.64], relative improvement [57.04%, 102.73%], per-iteration improvement [11.75, 17.72], and iterations to convergence [3.75, 4.86]."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., quality improvement over iterations, fabrication detection rates) but does not report any statistical significance tests (no p-values, t-tests, or similar)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports effect sizes with baseline context: mean absolute improvement of 24.91 points, mean relative improvement of 78.85%, per-iteration improvement of 14.70 points. The absolute point values with CIs provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 474 projects (98 with complete review cycles) is described but never justified. No power analysis or reasoning for why this sample size is adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports confidence intervals for all key metrics, and Figure 2 shows the full distribution of score improvements and iteration counts. The wide CI for relative improvement [57.04%, 102.73%] explicitly reflects high variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only its own system's performance. No comparison against a single-agent baseline, policy-based compartmentalisation, or any alternative architecture. Section 8 explicitly identifies controlled architectural comparison as future work."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so the question of whether they are contemporary does not arise. The paper acknowledges this gap in Section 8."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. Section 8 explicitly proposes ablation studies as future work: 'controlled architectural comparison through ablation studies would determine whether compartmentalisation actually matters.'"
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports multiple metrics: fabrication detection rate (52%), project completion rate (69%), absolute quality improvement (24.91 points), relative improvement (78.85%), iterations to convergence (4.30), and cost per project ($0.29)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated through the system's own Corroborator and Critic agents. No independent human evaluation of output quality is reported. The paper makes claims about quality improvement but quality is assessed only by the system's own LLM-based Critic."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a benchmark evaluation paper. The system processes composition tasks; there is no train/test split or held-out evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-category breakdown of verdict distribution (passed/failed/mixed). Section 6 breaks down costs by outcome (completed $0.29, failed $0.20, aborted $0.59). Figure 2 shows distributions of improvement and iteration counts."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents six consecutive FABRICATED verdicts during self-documentation. Section 5 documents a case where the system was given an impossible task. Section 6 reports 18.1% failure rate and 12.7% abort rate with discussion of causes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports a 31% non-completion rate (18.1% failed + 12.7% aborted), three projects with negative improvement (Figure 2 caption), and the honest refusal case where the system could not complete the assigned task. Section 7 acknowledges the system cannot operate without human oversight."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (52% fabrication detection, 79% quality improvement, 4.3 mean iterations, progression toward honest refusal) are all supported by Section 6 results and Section 5 case study. The abstract uses hedged language ('patterns consistent with the institutional hypothesis', 'findings motivate controlled investigation')."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies causal claims throughout (e.g., 'architectural enforcement produces reliable collective behaviour', 'adversarial feedback loop drove this progression') but the study design is observational with no controlled comparisons. Section 7 acknowledges: 'we cannot definitively isolate architectural effects from instruction-following.' However, the language regularly implies causation without adequate causal identification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly bounds generalisability: 'All results were obtained using Claude 4.5 models during January-February 2026; generalisability to other large language model architectures remains unknown.' Section 5 states: 'It does not support claims about other models, architectures, domains, or systems lacking similar capacities.' The paper consistently qualifies scope throughout."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses the reflexivity confound: 'did honest refusal emerge from architectural constraints enabling detection and iteration, or from theoretical exposure to refusal frameworks?' Section 5 discusses confounded variables preventing causal isolation. The paper explicitly considers whether results reflect model-specific behaviour rather than architectural principles."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states 'Claude 4.5 models (Haiku, Sonnet, Opus)' but does not specify exact API versions, snapshot dates, or model IDs. 'Claude 4.5' is a marketing name without sufficient version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided in the paper. Agent roles are described in natural language (Section 3.2) but the actual system prompts, instructions, and tool definitions given to each agent are not included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported. Temperature, top-p, max tokens, and other LLM API settings are not mentioned. The convergence threshold tau=85 is specified (Section 3.4) but LLM inference parameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.5 provide detailed description of the agentic scaffolding: seven agent roles with specific information access profiles, three-layer architecture, workflow orchestrator (PerseveranceGraph), state management (GraphState), iteration logic, convergence threshold, and document visibility system with six levels."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the 474 projects were selected for analysis, what filtering was applied, or how the 98 projects with 'complete review cycles' were identified from the full corpus. The jump from 474 total to 98 with complete cycles to 167 with verdicts is not fully explained."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is a dedicated 'Limitations' section spanning approximately one full page with substantive discussion of multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: reflexivity confound ('the system documented the honest refusal case whilst simultaneously experiencing the phenomenon'), single model family limitation, unfunded research constraints, human-in-the-loop requirement preventing extrapolation to unsupervised deployment. These are specific to this study, not generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states scope boundaries: 'We cannot extrapolate these findings to unsupervised deployment, adversarial settings, or different domains.' Section 5 states: 'It does not support claims about other models, architectures, domains, or systems lacking similar capacities for articulated reasoning about constraints.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 provides a link to all composition project artefacts at https://codeberg.org/wwaites/persevere-data. Case studies reference specific project identifiers (e.g., 'pouncing-siamese-deluxe-reggae') enabling verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 describes the data source: 474 projects processed during development of the paper, with outcome categories (327 completed, 86 failed, 60 aborted, 1 active). The Corroborator evaluated 733 documents. The operational context is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The data comes from the system's own operational logs during composition tasks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how operational logs were transformed into the reported statistics. The filtering from 474 projects to 167 with verdicts to 98 with complete review cycles is not fully explained with criteria at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: 'This research was unfunded, conducted at personal expense with total operational cost of $149.22 across 474 projects.' The absence of funding is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation (University of Southampton) is listed on the first page. The paper evaluates the author's own system (PCE), and the Author Contributions section transparently describes this relationship."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "The research is explicitly unfunded, conducted at personal expense. NA for unfunded work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper. The author evaluates their own system, which could have commercial implications, but no formal declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It evaluates a multi-agent system architecture for document composition, not model knowledge on test sets."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The system processes novel composition tasks, not pre-existing test sets."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The operational corpus consists of unique composition tasks, not standardised benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper analyses automated system logs from multi-agent composition tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyses the operational behaviour of a software system."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The single user (the author) directed the system but was not a research subject."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions requiring randomisation."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The architectural compartmentalisation between agents is described but this is system design, not blinding of human subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Project completion/failure/abort rates are reported but these are system outcomes, not participant attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports: '$0.29 in operational costs' per completed project, total costs of $149.22 across 474 projects, and 222,079,538 total tokens consumed. Cost breakdowns by outcome category are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports total operational costs of $149.22 across 474 projects, with 222,079,538 tokens consumed (83% from cache). Footnote 3 notes a more realistic estimate excluding labour is $500. Section 8 estimates validation would require approximately two person-years plus substantial compute."
       }
     }
   },
   "claims": [
     {
       "claim": "The verification agent classified 52% of submitted drafts as fabricated, requiring iterative revision toward full substantiation.",
       "evidence": "Section 6: 'Of these, 352 documents (48.0%) received substantiated verdicts... while 381 documents (52.0%) received fabricated verdicts' across 733 documents evaluated.",
       "supported": "strong"
     },
     {
       "claim": "Quality scores improved by 78.85% on average from initial submission to final acceptance.",
       "evidence": "Section 6, Table 3: 'Mean relative improvement reached 78.85% [95% CI: 57.04%, 102.73%]' across 98 projects with complete review cycles.",
       "supported": "strong"
     },
     {
       "claim": "Projects required a mean of 4.3 iterations to reach convergence.",
       "evidence": "Section 6, Table 3: 'Mean iterations to convergence 4.30 [95% CI: 3.75, 4.86].'",
       "supported": "strong"
     },
     {
       "claim": "When assigned an impossible task, the system progressed from fabrication toward honest refusal with alternative proposals.",
       "evidence": "Section 5, Table 1: Five iterations showing progression from fabricated thematic categories (iter 1) to operationalised refusal framework scoring 92/100 (iter 5). Referenced by project identifier pouncing-siamese-deluxe-reggae.",
       "supported": "moderate"
     },
     {
       "claim": "Architectural enforcement of information compartmentalisation produces reliable collective behaviour from unreliable components.",
       "evidence": "This is the central thesis. Evidence includes the 474-project corpus, but no controlled comparison against alternative architectures is provided. Section 8 explicitly acknowledges this requires future ablation studies.",
       "supported": "weak"
     },
     {
       "claim": "Mean cost per completed project was $0.29, demonstrating economic feasibility of structured verification.",
       "evidence": "Section 6: 'Projects reaching successful completion averaged $0.29 in operational costs. Total operational costs across all 474 projects amounted to $149.22.'",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["case-study", "observational"],
   "key_findings": "The Perseverance Composition Engine, a multi-agent system with architecturally enforced information compartmentalisation, processed 474 document composition tasks with a 69% completion rate, achieving 52% fabrication detection and 79% quality improvement over an average of 4.3 iterations at $0.29 per completed project. Two case studies illustrate specific mechanisms: verification rigour during self-documentation (six consecutive fabrication rejections) and progression from fabrication toward honest refusal under impossible task constraints. The paper frames these observations as consistent with organisational theory applied to multi-agent AI safety, while explicitly acknowledging the absence of controlled experiments comparing alternative architectures.",
   "red_flags": [
     {
       "flag": "Author evaluating own system",
       "detail": "The single author designed, implemented, and evaluated the Perseverance Composition Engine. All quality assessments are produced by the system's own LLM-based agents (Corroborator and Critic), not independent evaluators. There is no external validation of output quality."
     },
     {
       "flag": "Self-referential evaluation",
       "detail": "The paper was composed by the system it describes (acknowledged in Author Contributions). The system analysed its own operational logs to produce the reported statistics. This reflexivity creates attribution confounds the paper acknowledges but cannot resolve."
     },
     {
       "flag": "No baselines or controlled comparisons",
       "detail": "All results report only the system's own performance. No comparison against single-agent baselines, policy-based compartmentalisation, or alternative architectures. The paper explicitly identifies ablation studies as future work, meaning the central claim about architectural enforcement is unsupported by comparative evidence."
     },
     {
       "flag": "Quality metrics are self-assessed",
       "detail": "The 78.85% quality improvement metric is based entirely on the system's own Critic agent scores (0-100). No independent human evaluation validates whether these scores reflect actual quality improvement. The Critic is an LLM with no demonstrated correlation to human quality judgments."
     },
     {
       "flag": "Causal language without causal design",
       "detail": "Despite careful hedging in some passages, the paper regularly uses causal language ('produces', 'drove this progression', 'enables') for observational findings. The study design cannot distinguish architectural effects from model capabilities, prompt engineering, or other confounds."
     }
   ],
   "cited_papers": [
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Y. Bai", "S. Kadavath", "S. Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Foundational work on AI alignment through training, the individual-level approach this paper contrasts with institutional design."
     },
     {
       "title": "AI Safety via Debate",
       "authors": ["G. Irving", "P. Christiano", "D. Amodei"],
       "year": 2018,
       "arxiv_id": "1805.00899",
       "relevance": "Proposes debate between AI agents as a scalable oversight mechanism, a key theoretical foundation for adversarial verification in multi-agent systems."
     },
     {
       "title": "Improving Factuality and Reasoning in Language Models through Multiagent Debate",
       "authors": ["Y. Du", "S. Li", "A. Torralba", "J. B. Tenenbaum", "I. Mordatch"],
       "year": 2023,
       "arxiv_id": "2305.14325",
       "relevance": "Demonstrates that multi-agent debate improves factual accuracy in LLMs, directly relevant to adversarial verification approaches."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist through Safety Training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates model organisms methodology for studying AI safety failures, which this paper extends to collective/institutional safety."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Q. Wu", "G. Bansal", "J. Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Major multi-agent LLM framework used as a reference point for contemporary approaches to agent coordination."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["S. Hong", "X. Zheng", "J. Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework assigning organisational roles to LLM agents, directly comparable to the organisational approach of this paper."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Language Model Society",
       "authors": ["G. Li", "H. A. A. K. Hammoud", "H. Itani"],
       "year": 2023,
       "arxiv_id": "2303.17760",
       "relevance": "Role-playing multi-agent framework relevant to how agent coordination and role assignment affect LLM outputs."
     },
     {
       "title": "Large Language Models Miss the Multi-Agent Mark",
       "authors": ["E. L. Malfa", "G. L. Malfa", "S. Marro"],
       "year": 2025,
       "relevance": "Identifies gaps in LLM multi-agent systems research including social agency and coordination protocols, which this paper claims to address."
     },
     {
       "title": "MultiAgentBench: Evaluating the Collaboration and Competition of LLM Agents",
       "authors": ["K. Zhu", "H. Du", "Z. Hong"],
       "year": 2025,
       "doi": "10.18653/v1/2025.acl-long.421",
       "relevance": "Standardised benchmark for evaluating multi-agent LLM systems, relevant to evaluation methodology in the multi-agent space."
     },
     {
       "title": "Debate, Deliberate, Decide (D3): A Cost-Aware Adversarial Framework for Reliable and Interpretable LLM Evaluation",
       "authors": ["A. Harrasse", "C. Bandi", "H. Bandi"],
       "year": 2026,
       "relevance": "Multi-agent evaluation framework using structured debate and role specialisation, directly comparable to the adversarial verification approach."
     },
     {
       "title": "Risks from Learned Optimization in Advanced Machine Learning Systems",
       "authors": ["E. Hubinger", "C. van Merwijk", "V. Mikulik"],
       "year": 2019,
       "arxiv_id": "1906.01820",
       "relevance": "Foundational work on deceptive alignment risks in individual AI systems, the individual-level safety problem that institutional design aims to complement."
     },
     {
       "title": "Scalable Agent Alignment via Reward Modeling: A Research Direction",
       "authors": ["J. Leike", "D. Krueger", "T. Everitt"],
       "year": 2018,
       "arxiv_id": "1811.07871",
       "relevance": "Research direction on scalable alignment through reward modeling, relevant to the broader alignment landscape this paper positions against."
     }
   ]
 }