ai-research-survey

scan.json (22963B)

---

 {
   "paper": {
     "title": "Cognitive Models and AI Algorithms Provide Templates for Designing Language Agents",
     "authors": ["Ryan Liu", "Dilip Arumugam", "Cedegao E. Zhang", "Sean Escola", "Xaq Pitkow", "Thomas L. Griffiths"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.22523"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository URL is provided anywhere in the paper. Although the paper is theoretical/position, it formalizes agent templates and surveys implementations — analysis scripts or template specifications could have been released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or structured data (e.g., a catalog of agent templates mapped to cognitive models) is released. The survey content exists only in prose form."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a position/survey paper that does not run any experiments, so environment specifications are structurally inapplicable."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted, so reproduction instructions are structurally inapplicable."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a position paper that does not report any new experimental results; no statistical analysis is performed."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No new experiments are conducted, so significance tests are not applicable."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No new experiments are conducted; effect sizes are not applicable."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No new experiments are conducted; sample size justification is not applicable."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No new experiments are conducted; variance reporting is not applicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a position paper, not an empirical evaluation. It does not compare against prior surveys or provide a systematic evaluation of templates."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation is conducted; baseline recency is not applicable."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or method is evaluated, so ablation is structurally inapplicable."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation is conducted, so multiple metrics are not applicable."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation is conducted; human evaluation is not applicable to a position paper making no empirical claims about system outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; held-out test sets are not applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper organizes its survey by category: communication (Section 4.1), reasoning and planning (Section 4.2), representation (Section 4.3), search (Section 5.1.1), divide and conquer (Section 5.1.2), policy iteration (Section 5.2.1), posterior sampling (Section 5.2.2), and information-directed sampling (Section 5.2.3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses alternative views and limitations of the agent template approach, including the No Free Lunch Theorem argument, the paradigm-shift argument, and the automated-discovery argument. Section 5.1.2 notes that divide-and-conquer approaches may be less necessary given modern reasoning models."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report any negative results from its own work. It references results from cited works but does not present cases where the proposed approach (cognitive/AI-inspired templates) failed or was inferior."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that cognitive models and AI algorithms provide templates for language agents, and the paper supports this through formal definitions (Section 3) and multiple detailed examples from communication (Section 4.1), reasoning (Section 4.2), representation (Section 4.3), search (Section 5.1.1), divide-and-conquer (Section 5.1.2), and RL algorithms (Section 5.2)."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a position paper that does not make causal claims. It argues for the value of an approach and surveys existing work rather than claiming that using templates causes improved performance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly addresses generalization limits: the No Free Lunch Theorem means no single template works for all tasks, and the paper acknowledges that 'if such a shift does exist, it has yet to make itself apparent in the agents of today.' The paper frames its claims as about the utility of templates as a design approach, not universal superiority."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 ('Alternative Views') discusses three specific alternatives: (1) finding a single best template instead of a general framework, (2) the possibility that LLMs represent a paradigm shift requiring novel templates, and (3) automatically discovering templates via LLMs rather than drawing from cognitive science/AI. Each is substantively addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not run any experiments with LLMs; it surveys existing work. Model version specification is not applicable."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in any experiments; it is a position/survey paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; hyperparameter reporting is not applicable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is implemented or evaluated; the paper describes templates at a conceptual level."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper surveys existing work but does not describe how papers were selected for inclusion. There is no systematic search methodology, inclusion/exclusion criteria, or filtering pipeline documented. Papers appear to be selected based on the authors' knowledge of the field."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 ('Alternative Views') functions as a limitations section, discussing three substantive reservations about the proposed approach. While not titled 'Limitations,' it provides substantive discussion of weaknesses."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses specific threats: the No Free Lunch Theorem implying no universal template, the possibility that LLMs require fundamentally novel architectures, and the potential for automated template discovery to supersede human-designed templates. These are specific to this work's claims."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT cover. It does not bound which cognitive models or AI algorithms are out of scope, nor does it explain why certain relevant areas (e.g., attention mechanisms, memory consolidation models) are excluded. Section 6.2 mentions some future directions (hypothesis generation, evolutionary algorithms, game theory) but frames these as calls to action rather than explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "This is a position/survey paper with no collected data. Raw data availability is structurally inapplicable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper surveys existing work but does not describe how the surveyed papers were identified or collected. No search methodology, databases queried, or time period is stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved and no data source is a standard benchmark. This is a position paper surveying the literature."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No pipeline from literature collection to analysis is documented. The paper does not describe how papers were found, filtered, or categorized for inclusion in the survey."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states: 'This work was supported by funds provided by the National Science Foundation and by DoD OUSD (R & E) under Cooperative Agreement PHY-2229929 (the NSF AI Institute for Artificial and Natural Intelligence) and by ONR MURI N00014-24-1-2748.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Princeton University (Departments of Computer Science and Psychology), MIT (Brain and Cognitive Sciences), Columbia University (Zuckerman Mind Brain Behavior Institute and Psychiatry), and Carnegie Mellon University (Neuroscience Institute and Machine Learning)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from NSF and DoD/ONR, which are government agencies with no direct financial stake in whether cognitive models or AI algorithms are good templates for language agents."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark. It is a position/survey paper."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model is evaluated on any benchmark; contamination concerns are structurally inapplicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted; contamination is structurally inapplicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this position paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a position/survey paper; no method with inference cost is proposed or evaluated."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a position/survey paper; no computation is performed."
       }
     }
   },
   "claims": [
     {
       "claim": "Cognitive models and AI algorithms provide effective templates for designing language agents, with agent templates formalized as directed acyclic graphs (DAGs) over LLMs and tools.",
       "evidence": "Section 3 provides a formal definition of agent templates as DAGs G = (V, E) where V ⊆ F are LLMs or tools and E are connections. Sections 4 and 5 give concrete examples of templates derived from cognitive models (RSA, reasoning/planning, Language of Thought) and AI algorithms (search, divide-and-conquer, policy iteration, posterior sampling, IDS).",
       "supported": "moderate"
     },
     {
       "claim": "RSA-inspired language agents outperform baselines and ablations for communication tasks according to human judgments.",
       "evidence": "Section 4.1 cites Liu et al. (2023) as demonstrating that an agent using episodic future thinking to simulate audience reactions 'outperforms baselines and ablations according to human judgments.' This is a claim from cited work, not new evidence.",
       "supported": "moderate"
     },
     {
       "claim": "The Modular Agentic Planner (MAP), inspired by prefrontal cortex models, achieves strong results on Tower of Hanoi and PlanBench.",
       "evidence": "Section 4.2 cites Webb et al. (2025) for MAP's results. No new evaluation is presented in this paper.",
       "supported": "moderate"
     },
     {
       "claim": "Tree of Thoughts and related search-based agent templates improve LLM reasoning by externalizing the heuristic search process.",
       "evidence": "Section 5.1.1 describes Tree of Thoughts (Yao et al., 2023a) and beam-search variants (Xie et al., 2023) as instantiations of search algorithms. Evidence comes from cited works.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-based PSRL agents retain the efficient exploration properties of the original PSRL algorithm, as confirmed by cumulative regret curves.",
       "evidence": "Section 5.2.2 cites Arumugam & Griffiths (2025) for 'cumulative regret curves confirming that this agent template can succeed' in natural-language tasks like Wordle. Evidence is from cited work.",
       "supported": "moderate"
     },
     {
       "claim": "IDS-based language agents approximate the Bayes-optimal policy more closely than PSRL-based agents in a numerical Wordle variant.",
       "evidence": "Section 5.2.3 cites Arumugam & Griffiths (2025) for this finding. The authors note 'the empirical study conducted... for IDS was preliminary and limited to a single domain.'",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["theoretical"],
   "key_findings": "This position paper argues that cognitive models (Rational Speech Acts, reasoning/planning models, Language of Thought) and AI algorithms (search algorithms, divide-and-conquer, policy iteration, posterior sampling, information-directed sampling) provide principled templates for designing modular language agents. The authors formalize agent templates as directed acyclic graphs (DAGs) over LLMs and tools and survey existing language agents that instantiate these templates. The paper calls for researchers to explore additional cognitive and algorithmic templates, including game-theoretic models and evolutionary algorithms.",
   "red_flags": [
     {
       "flag": "No systematic survey methodology",
       "detail": "Despite surveying existing language agents and their connections to cognitive models, the paper provides no systematic search methodology, inclusion/exclusion criteria, or explanation of how surveyed works were selected. This makes the coverage appear selective rather than comprehensive."
     },
     {
       "flag": "All cited empirical evidence comes from co-authored work",
       "detail": "Most of the concrete empirical claims (PSRL agents, IDS agents, communication agents) cite work where the paper's own authors are co-authors (Arumugam & Griffiths, 2025; Liu et al., 2023; Webb et al., 2025). While not inherently problematic for a position paper, this means the evidence base is largely self-referential."
     },
     {
       "flag": "No quality assessment of surveyed work",
       "detail": "The paper reports results from cited works at face value without assessing the methodological quality of those studies. For a survey-style paper arguing for an approach, some critical evaluation of the strength of evidence would be expected."
     }
   ],
   "cited_papers": [
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao", "Izhak Shafran", "Thomas Griffiths", "Yuan Cao", "Karthik Narasimhan"],
       "year": 2023,
       "relevance": "Key example of search-algorithm-inspired agent design; uses BFS/DFS with LLM-based node expansion and evaluation."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Foundational language agent design combining reasoning and acting, relevant to agentic workflow evaluation."
     },
     {
       "title": "Cognitive architectures for language agents",
       "authors": ["Theodore Sumers", "Shunyu Yao", "Karthik Narasimhan", "Thomas Griffiths"],
       "year": 2023,
       "relevance": "CoALA framework connecting language agent research with cognitive science, directly related to evaluating agent design methodology."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Major language agent for automated software engineering; relevant to evaluating agentic coding tool methodology."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "relevance": "Key agent architecture using verbal self-reflection for iterative improvement; relevant to agent design evaluation."
     },
     {
       "title": "Autogen: Enabling next-gen llm applications via multi-agent conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang", "Yiran Wu", "Beibin Li", "Erkang Zhu", "Li Jiang", "Xiaoyun Zhang", "Shaokun Zhang", "Jiale Liu"],
       "year": 2024,
       "relevance": "Multi-agent conversation framework; relevant to evaluating multi-agent system design."
     },
     {
       "title": "Openhands: An open platform for ai software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song", "Frank F. Xu", "Xiangru Tang", "Mingchen Zhuge"],
       "year": 2024,
       "relevance": "Open platform for AI coding agents; directly relevant to evaluating agentic software engineering tools."
     },
     {
       "title": "A brain-inspired agentic architecture to improve planning with llms",
       "authors": ["Taylor Webb", "Shanka Subhra Mondal", "Ida Momennejad"],
       "year": 2025,
       "relevance": "MAP architecture inspired by prefrontal cortex models for LLM planning; key example of cognitive-model-derived agent template."
     },
     {
       "title": "Large Language Models can Implement Policy Iteration",
       "authors": ["Ethan Brooks", "Logan Walls", "Richard L. Lewis", "Satinder Singh"],
       "year": 2023,
       "relevance": "Early work on LLM-based RL agent templates (ICPI); directly relevant to evaluating AI-algorithm-derived agent designs."
     },
     {
       "title": "Toward Efficient Exploration by Large Language Model Agents",
       "authors": ["Dilip Arumugam", "Thomas L. Griffiths"],
       "year": 2025,
       "arxiv_id": "2504.20997",
       "relevance": "Introduces PSRL and IDS agent templates for LLM exploration; key empirical backing for the position paper's claims."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Michael Z. Pan", "Shuyan Yang", "Lakshya A. Agrawal"],
       "year": 2025,
       "relevance": "Studies failure modes of multi-agent LLM systems; directly relevant to evaluating agent system reliability."
     },
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "authors": ["Cong Lu", "Chris Lu", "Robert T. Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Automated scientific discovery agent; relevant to evaluating agentic AI capabilities and methodology."
     }
   ]
 }