ai-research-survey

scan.json (24848B)

---

 {
   "paper": {
     "title": "Augmented Language Models: a Survey",
     "authors": [
       "Grégoire Mialon",
       "Roberto Dessì",
       "Maria Lomeli",
       "Christoforos Nalmpantis",
       "Ram Pasunuru",
       "Roberta Raileanu",
       "Baptiste Rozière",
       "Timo Schick",
       "Jane Dwivedi-Yu",
       "Asli Celikyilmaz",
       "Edouard Grave",
       "Yann LeCun",
       "Thomas Scialom"
     ],
     "year": 2023,
     "venue": "arXiv preprint",
     "arxiv_id": "2302.07842"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or analysis scripts are provided. The paper is a narrative survey with no released artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset, corpus of reviewed papers, or structured extraction of surveyed works is released. A survey can release its search corpus or extracted data tables, but this one does not."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments, so environment specifications are structurally inapplicable."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions are provided for reproducing the survey's paper selection, classification, or analysis. The paper does not describe a systematic search methodology that could be replicated."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a narrative survey that does not run experiments or perform statistical aggregation. No original quantitative results are reported."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or statistical comparisons are conducted by the authors. Tables (e.g., Table 1 on GSM8K) reproduce results from other papers."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments are conducted. The survey reproduces results from other works without performing meta-analytic aggregation."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or data collection are performed by the authors. This is a narrative review."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments are conducted. No statistical aggregation across reviewed papers is performed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare itself against prior surveys or provide a structured comparison framework showing how it differs from or improves upon existing surveys (e.g., Huang and Chang 2022 or Qiao et al. 2022, which are mentioned as related surveys on reasoning)."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experimental baselines are applicable to a narrative survey paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or method is proposed, so ablation is not applicable."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation of any system is conducted by the authors."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are produced that would require human evaluation. This is a literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted. Not applicable to a survey."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey organizes reviewed work into categories: reasoning (Section 2), tools and acting (Section 3), and learning methods (Section 4), with subcategories such as prompting approaches, recursive prompting, retrieval-augmented models, code interpreters, physical/virtual agents, supervision, and RL. Table 1 provides per-method breakdowns on GSM8K and Table 2 compares retrieval-augmented models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 discusses limitations of abstract reasoning, noting there is no guarantee intermediate steps are valid and reasoning may not actually be used by the model. Section 4.3 discusses limitations of current RL methods including instability. Section 5 discusses ethical concerns and limitations of ALMs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings from reviewed work: that the compositionality gap does not narrow with model size (Section 2.1, Press et al. 2022), that chain-of-thought can lead to correct predictions despite nonsensical intermediate reasoning (Section 5, Lewkowycz et al. 2022), and that there is little evidence retrieval-augmented LMs improve truthfulness (Section 5, Krishna et al. 2021)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the survey 'reviews works in which language models are augmented with reasoning skills and the ability to use tools' and concludes the direction 'has the potential to address common limitations.' The body comprehensively covers reasoning (Section 2), tools (Section 3), and learning approaches (Section 4), supporting both claims. The potential-to-address claim is appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The survey does not make original causal claims. It describes findings and mechanisms from reviewed papers but does not itself claim causal relationships based on its own analysis."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title and framing suggest broad coverage ('Augmented Language Models: a Survey') but the scope boundaries are not explicitly stated. The paper does not clearly specify what is excluded — for example, vision-language models are partially covered, robotics is partially covered, but the inclusion/exclusion criteria for the survey are never stated. Section 1.2 notes 'Other axes could naturally have been chosen' but does not specify what was excluded or why."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "As a survey paper presenting no original empirical results, alternative explanations for its own findings are not applicable. The survey does note alternative interpretations of reviewed work (e.g., whether LMs truly reason or just produce helpful context, Section 2.4)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used by the authors. This is a survey paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting experiments are conducted by the authors. Prompt examples shown (Figures 1-6) are reproduced from reviewed papers."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted by the authors."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used by the authors. This is a survey paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not describe how papers were selected for inclusion. There is no search methodology, database queries, inclusion/exclusion criteria, or filtering pipeline described. The reader cannot understand or reproduce the paper selection process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section for the survey itself. Section 4.3 discusses 'Limitations and future directions' of the reviewed methods (RL instability, data scarcity), and Section 5 includes a Discussion, but neither addresses limitations of the survey's own methodology or coverage."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity of the survey itself are discussed. There is no discussion of potential bias in paper selection, coverage gaps, or limitations of the narrative review approach."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The scope is loosely described: 'the survey focuses on works that combine reasoning or tools with LMs' (Section 1.2) and 'we focus on LLMs' but the paper does not clearly state what is excluded or what the results do NOT show. For instance, it is unclear whether multi-modal foundation models, pure RL agents without LM components, or closed-source commercial systems are in scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (list of all surveyed papers, search queries, extraction spreadsheets) is available. The survey cannot be independently verified for completeness or accuracy of coverage."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the surveyed papers were identified or collected. There is no mention of search databases, queries, time periods, or systematic search protocols."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper is a literature survey, not a study with recruited subjects. The data source (research papers) is not a standard benchmark either, so NA applies."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The survey does not describe how papers were found, screened, included, or categorized. The classification into reasoning/tools/learning appears to be the authors' taxonomy but the process of arriving at it is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The Acknowledgements section only thanks Marco Baroni for feedback. All authors are Meta AI employees, but no funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors clearly list their Meta AI affiliation and email addresses on the first page. Roberto Dessì's additional affiliation at Universitat Pompeu Fabra is also noted."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors work at Meta AI. Meta has commercial interests in the technologies surveyed (LLMs, tool-augmented AI). The survey reviews several Meta/FAIR papers (e.g., Toolformer, Atlas, PEER, Galactica) alongside other work. The funder (Meta) is not independent of the survey's conclusions about the promise of augmented language models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Given that all authors are Meta employees and the survey covers Meta's research extensively, a disclosure would be expected."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any model on benchmarks. Contamination concerns are not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is conducted. Not applicable to a survey paper."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is conducted. Not applicable to a survey paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a literature survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no method of its own. Cost reporting is not applicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments. Not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "Augmented Language Models (ALMs) that combine reasoning and tool use can address common limitations of traditional LMs such as interpretability, consistency, and scalability issues.",
       "evidence": "Abstract and Section 5 ('Discussion - Augmented Language Models benefits') list potential advantages: truthfulness through grounding, uncertainty estimation, interpretability via intermediate reasoning steps, and enhanced capabilities. However, the paper acknowledges 'there is surprisingly little evidence' that retrieval-augmented LMs improve truthfulness (Section 5, citing Krishna et al. 2021).",
       "supported": "weak"
     },
     {
       "claim": "Chain-of-thought prompting enables LMs to be better reasoners, with the capability emerging at scale (100B+ parameters).",
       "evidence": "Section 2.1 cites Wei et al. (2022c) for few-shot CoT and Kojima et al. (2022) for zero-shot CoT. Table 1 shows CoT improving GSM8K accuracy from 15.6% to 46.9% (text-davinci-002) and 19.7% to 63.1% (code-davinci-002). Section 2.1 notes emergence requires 100B+ parameters per Wei et al. (2022b).",
       "supported": "strong"
     },
     {
       "claim": "Tool-augmented LMs can be seen as a generalization of the non-parametric framework, potentially yielding smaller models that retain capabilities of larger counterparts.",
       "evidence": "Section 5 ('Generalizing the non-parametric framework') discusses this claim conceptually, citing RETRO and Atlas as examples where retrieval reduces parameter requirements. Table 2 compares retrieval-augmented models. However, this is presented as a conjecture about future scaling laws rather than an empirically validated claim.",
       "supported": "weak"
     },
     {
       "claim": "Combining reasoning and tool use via reinforcement learning from human feedback (RLHF) yields models that outperform humans in question-answering tasks.",
       "evidence": "Section 4.2 describes WebGPT (Nakano et al. 2021) which, after fine-tuning with RLHF, produced answers preferred over human-generated ones on two QA datasets. This is cited evidence from a single paper, not independently verified in this survey.",
       "supported": "moderate"
     },
     {
       "claim": "The compositionality gap in language models does not narrow with increasing model scale.",
       "evidence": "Section 2.1 cites Press et al. (2022) who 'observe that this gap does not narrow when increasing the size of the model.' This is evidence from a single study on 2-hop questions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "meta-analysis"
   ],
   "key_findings": "This survey from Meta AI categorizes augmented language model (ALM) research into three axes: reasoning (chain-of-thought prompting, recursive prompting, fine-tuning), tool use (retrieval, search engines, code interpreters, robotic control), and learning methods (supervision, reinforcement learning). The paper argues that combining reasoning and tools allows LMs to depart from pure language modeling toward more capable agents. Key limitations identified include the lack of guarantee that intermediate reasoning steps are valid, instability of RL training methods, and the open question of whether LMs can achieve System 2 reasoning through current approaches.",
   "red_flags": [
     {
       "flag": "No systematic search methodology",
       "detail": "The survey does not describe any systematic search protocol, database queries, inclusion/exclusion criteria, or paper selection methodology. This makes the coverage impossible to evaluate for completeness or bias. The reader cannot know what was excluded or why."
     },
     {
       "flag": "Potential conflict of interest: Meta authors surveying Meta research",
       "detail": "All 13 authors are Meta AI employees. The survey prominently features Meta/FAIR papers (Toolformer, Atlas, PEER, Galactica, Socratic Models, Flamingo, BlenderBot, etc.) alongside work from other groups. No conflict of interest statement is provided. While the coverage of non-Meta work appears broad, the framing and conclusions may favor Meta's research directions."
     },
     {
       "flag": "No quality assessment of reviewed papers",
       "detail": "The survey describes what methods exist and how they work, but does not assess the methodological quality or strength of evidence in the reviewed papers. Results from different papers are presented side-by-side (e.g., Table 1) without noting differences in evaluation methodology, making the survey a potential laundering mechanism for weak results alongside strong ones."
     },
     {
       "flag": "Speculative conclusions presented as survey findings",
       "detail": "Several claims in Section 5 and the abstract are speculative (e.g., ALMs 'have the potential to address common limitations') rather than conclusions supported by structured analysis of the evidence. The claim about truthfulness is immediately qualified by noting 'surprisingly little evidence' supports it."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational work on chain-of-thought prompting for LLM reasoning, central to the survey's reasoning section."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì", "Roberta Raileanu", "Maria Lomeli", "Luke Zettlemoyer", "Nicola Cancedda", "Thomas Scialom"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Key paper on self-supervised tool learning for LMs, demonstrating LMs can learn when and how to call external tools."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Influential work on interleaving reasoning and acting in LMs, combining chain-of-thought with tool use for decision making."
     },
     {
       "title": "PAL: Program-aided language models",
       "authors": ["Luyu Gao", "Aman Madaan", "Shuyan Zhou", "Uri Alon", "Pengfei Liu", "Yiming Yang", "Jamie Callan", "Graham Neubig"],
       "year": 2022,
       "relevance": "Demonstrates offloading computation to a Python interpreter via CoT-style prompting, achieving strong results on reasoning benchmarks."
     },
     {
       "title": "WebGPT: Browser-assisted question-answering with human feedback",
       "authors": ["Reiichiro Nakano", "Jacob Hilton", "Suchir Balaji", "Jeff Wu", "Long Ouyang"],
       "year": 2021,
       "arxiv_id": "2112.09332",
       "relevance": "Early work on training LMs to use web browsers via RLHF, demonstrating tool-augmented QA that surpasses human performance."
     },
     {
       "title": "Improving language models by retrieving from trillions of tokens",
       "authors": ["Sebastian Borgeaud", "Arthur Mensch", "Jordan Hoffmann"],
       "year": 2022,
       "relevance": "RETRO: key retrieval-augmented LM showing non-parametric memory can match larger model capabilities with fewer parameters."
     },
     {
       "title": "Atlas: Few-shot learning with retrieval augmented language models",
       "authors": ["Gautier Izacard", "Patrick Lewis", "Maria Lomeli"],
       "year": 2022,
       "arxiv_id": "2208.03299",
       "relevance": "Jointly trains retriever and seq2seq model for few-shot learning, demonstrating strong performance despite being much smaller than other LLMs."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang", "Diogo Almeida"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "InstructGPT paper on RLHF for instruction following, foundational to the learning methods discussed in the survey."
     },
     {
       "title": "Do as I can, not as I say: Grounding language in robotic affordances",
       "authors": ["Michael Ahn", "Anthony Brohan", "Noah Brown"],
       "year": 2022,
       "arxiv_id": "2204.01691",
       "relevance": "SayCan: combines LM planning with robot affordances for physical task execution, key paper on LMs acting in the physical world."
     },
     {
       "title": "Galactica: A large language model for science",
       "authors": ["Ross Taylor", "Marcin Kardas", "Guillem Cucurull", "Thomas Scialom"],
       "year": 2022,
       "arxiv_id": "2211.09085",
       "relevance": "Demonstrates prompt pre-training with explicit reasoning tokens for scientific reasoning, a key approach to teaching LMs to reason."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Codex paper: foundational to code generation and tool-use capabilities of LMs, enabling code-based reasoning and tool invocation."
     },
     {
       "title": "STaR: Self-taught reasoner bootstrapping reasoning with reasoning",
       "authors": ["Eric Zelikman", "Jesse Mu", "Noah D. Goodman", "Yuhuai Tony Wu"],
       "year": 2022,
       "relevance": "Key bootstrapping approach for teaching LMs to reason without extensive human demonstrations, bridging supervision and RL."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot prompting as a paradigm for LM capability, foundational to both reasoning and tool-use research."
     }
   ]
 }