ai-research-survey

scan.json (24970B)

---

 {
   "paper": {
     "title": "Check Your Facts and Try Again: Improving Large Language Models with External Knowledge and Automated Feedback",
     "authors": [
       "Baolin Peng",
       "Michel Galley",
       "Pengcheng He",
       "Hao Cheng",
       "Yujia Xie",
       "Yu Hu",
       "Qiuyuan Huang",
       "Lars Liden",
       "Zhou Yu",
       "Weizhu Chen",
       "Jianfeng Gao"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2302.12813"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We make the source code and models publicly available' in the abstract, and provides a URL: https://aka.ms/llm-augmenter (footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: DSTC7 Track 2 (News Chat), DSTC11 Track 5 (Customer Service), and OTT-QA (Wiki QA). All are public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or detailed dependency listing is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code is released, but the paper itself does not include instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Tables 1, 2, and 5 report only point estimates. Figure 3 shows shaded regions for max/min over 5 runs for the RL learning curve, but the main evaluation results lack confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports 'All differences are significant (p < 0.05)' for the human evaluation. Significance testing is mentioned for human evaluation results."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context, e.g., 'improve ChatGPT by 32.3% in Usefulness' and 'absolute +10% in F1' and specific point improvements like '10 and 6 points' in KF1. Tables provide both baseline and improved scores."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the sample sizes used. The 948 examples for human evaluation and 1370 examples for News Chat evaluation are stated without justification for why these sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Figure 3 shows max/min over 5 runs for the RL policy training curve only. The main automatic evaluation results in Tables 1, 2, and 5 do not report variance, standard deviation, or any spread measure across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "ChatGPT without augmentation serves as the baseline across all experiments. For Wiki QA, DPR vs. CORE retrieval is also compared. The paper also mentions comparison with state-of-the-art fine-tuned models (Ma et al., 2022)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The primary baseline is ChatGPT alone (zero-shot). There are no comparisons against other contemporary retrieval-augmented approaches (e.g., REPLUG, WebGPT, or other RAG methods mentioned in related work). The only fine-tuned comparison is Ma et al. (2022)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 provides ablation studies: (1) different policies for knowledge consolidator usage (Figure 4), (2) different feedback types (Table 4), (3) utility function + feedback combinations (Figure 5). These ablate individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 8 automatic metrics for dialog tasks (KF1, BLEU, ROUGE, chrF, METEOR, BERTScore, BARTScore, BLEURT) and precision/recall/F1 for Wiki QA. Human evaluation also includes Usefulness and Humanness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 reports human evaluation using Amazon Mechanical Turk with 948 examples. Workers rated Usefulness and Humanness on a 5-point Likert-like scale (Table 3)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For News Chat, a dedicated evaluation set of 1370 examples is used. For Customer Service, the validation set is used (test set was unavailable). For Wiki QA, a test set is used. The RL policy training uses test data for evaluation (Figure 3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-category breakdown is provided. The Wiki QA dataset has single-hop (13%), two-hop (57%), and multi-hop (30%) categories, but results are not broken down by these categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 mentions that 'most error cases are hallucinated answers and ChatGPT abstains from answering for 17% cases.' Table 6 in the appendix shows a qualitative example of the feedback loop correcting an initial bad response."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that 'there still remains a noticeable gap in performance' compared to fine-tuned models (Section 4.3), that inter-annotator agreement is low (0.15 and 0.07 Krippendorff's alpha in Table 3), and acknowledges latency issues in Section 6."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims LLM-AUGMENTER 'significantly reduces ChatGPT's hallucinations without sacrificing the fluency and informativeness.' Tables 1, 2, 5 show improved KF1 while maintaining or improving fluency metrics, and Table 3 shows human evaluation support."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (e.g., 'incorporating feedback leads to substantial improvement'). The ablation studies (Section 3.5) systematically add/remove components to establish causal contribution of each module. This is adequate controlled manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Improving Large Language Models with External Knowledge and Automated Feedback' implies generality, but experiments are only on ChatGPT across three specific tasks. While the paper mentions applicability to 'other LLMs such as GPT-3 or PaLM,' this is not tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the improvements. For example, improvements could be partly due to longer or more verbose responses, or the added prompting context could be doing the work rather than the iterative feedback mechanism specifically. No confounds are analyzed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'ChatGPT' throughout without specifying a version, snapshot date, or API version. The T5-Base model is named but ChatGPT is unversioned. The paper mentions using ChatGPT's 'current limited bandwidth' suggesting early API access but no version is specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix Tables 7, 8, and 9 provide the actual prompt templates used for News Chat, Customer Service, and Wiki QA respectively. The templates include the exact instruction text and structure with placeholders for dynamic content."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other LLM API parameters are reported for ChatGPT. The paper does not mention sampling settings. For the RL policy, the T5-Base model is mentioned but training hyperparameters are not detailed."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides detailed descriptions of the agentic scaffolding: Working Memory (Section 2.1), Policy (Section 2.2), Action Executor with Knowledge Consolidator and Prompt Engine (Section 2.3), and Utility module (Section 2.4). The MDP formulation and feedback loop are well-documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the News Chat data crawling process: selecting Reddit threads with URLs from 2021-2022, restricting to curated news websites, selecting oracle passages via ROUGE-F1, filtering by F1 threshold, resulting in 1370 examples. DSTC11 and OTT-QA are standard benchmarks."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Future Directions' is a dedicated section discussing limitations including latency, bandwidth constraints, and absence of human evaluation for Wiki QA."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: (1) interactive feedback with ChatGPT significantly slows down user experience, (2) RL experiments used T5-Base instead of ChatGPT due to bandwidth limitations, (3) human evaluation was initially not included for Wiki QA. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound generalizations to specific LLMs, tasks, or domains tested. The limitations section discusses practical constraints but not the boundaries of the claims themselves."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying datasets (DSTC7, DSTC11, OTT-QA) are publicly available benchmarks. The source code and models are also publicly released at https://aka.ms/llm-augmenter, which would include generated outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the News Chat data collection: crawling Reddit threads from 2021-2022, filtering by news websites, extracting oracle passages via ROUGE-F1, and applying threshold filtering. Customer Service uses DSTC11 Track 5 with 14768 sessions."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "For human evaluation (Section 3.4), the paper describes: 'using Amazon Mechanical Turk, we hired master-level workers with lifetime HIT acceptance rate above 95%.' This describes the recruitment channel and qualification criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the pipeline: Reddit threads -> URL domain filtering -> oracle passage extraction via ROUGE-F1 -> threshold filtering -> 1370 examples. For Wiki QA, the OTT-QA pipeline including DPR retrieval, entity linking, and evidence chaining is described in Section 4.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section is present. The Acknowledgements section thanks individuals for discussions but does not disclose funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Microsoft Research and Columbia University. All but one author (Zhou Yu) are from Microsoft Research."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work is from Microsoft Research. Microsoft has a direct commercial interest in the success of LLM augmentation systems (Azure OpenAI, Bing Chat, etc.). The funder is not independent of the outcome. No funding disclosure is provided to assess this, but the affiliation itself creates a non-independent relationship."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Microsoft Research authors working on LLM augmentation tools have potential commercial interests that are not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses ChatGPT but does not state its training data cutoff date. This is relevant because the DSTC7 and DSTC11 datasets may have been in ChatGPT's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ChatGPT's training data includes the DSTC7, DSTC11, or OTT-QA datasets. These are publicly available benchmarks that could have been in the training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OTT-QA was published in 2020, DSTC7 in 2019, and DSTC9/11 in 2020/2023. ChatGPT was trained on data up to at least 2021. The paper does not address whether these benchmarks were in the training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite using Amazon Mechanical Turk workers for human evaluation."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographics are reported for the Mechanical Turk workers beyond 'master-level workers with lifetime HIT acceptance rate above 95%.' No information on experience, geographic distribution, or other characteristics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies inclusion criteria: 'master-level workers with lifetime HIT acceptance rate above 95%' (Section 3.4). This defines who was eligible to participate."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation is a rating study comparing outputs of two systems, not an experimental study with treatment/control groups requiring randomization."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described for the human evaluation. It is not stated whether evaluators knew which responses came from ChatGPT vs. LLM-AUGMENTER."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is provided. The paper states 948 examples were used but does not report how many workers participated, how many dropped out, or quality control filtering."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or latency numbers are reported. Section 6 qualitatively mentions that 'interactive feedback with a computationally expensive model such as ChatGPT can significantly slow down the user experience' and that 'ChatGPT is often queried twice for a single response,' but no quantitative cost data is given."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, GPU hours, or hardware specifications are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-AUGMENTER significantly reduces ChatGPT's hallucinations without sacrificing fluency and informativeness.",
       "evidence": "Tables 1, 2 show KF1 improvements of ~10 points (News Chat) and ~6 points (Customer Service) with BM25 retrieval, while maintaining or improving BLEU, ROUGE, chrF, METEOR, BERTScore, BARTScore, and BLEURT. Table 3 human evaluation shows 32.3% improvement in Usefulness and 12.9% in Humanness.",
       "supported": "strong"
     },
     {
       "claim": "Automated feedback further improves response quality beyond external knowledge alone.",
       "evidence": "Tables 1 and 2 show consistent improvements when feedback is added: +3.3 KF1 on News Chat and +7.2 on Customer Service with golden knowledge. Table 4 ablation shows w/ Rule-based Feedback achieves 37.41 vs. 34.07 without feedback.",
       "supported": "strong"
     },
     {
       "claim": "Knowledge consolidation (CORE) is superior to raw retrieval (DPR) for grounding LLM responses.",
       "evidence": "Table 5 shows CORE achieves 8.08 F1 vs. DPR's 2.38 F1 on Wiki QA, a substantial improvement from evidence consolidation.",
       "supported": "strong"
     },
     {
       "claim": "A trainable RL policy can learn to effectively select actions to maximize reward.",
       "evidence": "Figure 3 shows the learning curve with T5-Base policy improving from random baseline (~33 KF1) to ~37.5 KF1 over 1000 training episodes, averaged over 5 runs. However, this is with T5-Base, not ChatGPT, due to bandwidth limitations.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-AUGMENTER is applicable to other LLMs beyond ChatGPT.",
       "evidence": "The paper states 'It is straightforward to apply LLM-AUGMENTER to other LLMs, such as GPT-3 or PaLM' (Section 3.2), but no experiments with other LLMs are provided.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "LLM-AUGMENTER is a plug-and-play framework that augments black-box LLMs with external knowledge retrieval and automated feedback to reduce hallucinations. On information-seeking dialog tasks (DSTC7/DSTC11), it improves ChatGPT's Knowledge F1 by 6-10 points with BM25 retrieval and by 21-34 points with golden knowledge, while maintaining fluency metrics. Human evaluation shows 32.3% Usefulness improvement. On multi-hop Wiki QA (OTT-QA), knowledge consolidation via CORE with feedback achieves 11.80 F1 vs. 0.59 for closed-book ChatGPT, but still lags behind fine-tuned state-of-the-art models.",
   "red_flags": [
     {
       "flag": "Company evaluating own product-adjacent technology",
       "detail": "The work is primarily from Microsoft Research. Microsoft has commercial interests in LLM augmentation (Azure OpenAI, Bing Chat). While the system is open-sourced, the evaluation setting favors demonstrating the value of augmentation infrastructure Microsoft could commercialize."
     },
     {
       "flag": "Unversioned model",
       "detail": "ChatGPT is used without specifying a version or API snapshot date. ChatGPT behavior changed significantly during early 2023, making these results non-reproducible without knowing exactly which model version was used."
     },
     {
       "flag": "Very low inter-annotator agreement in human evaluation",
       "detail": "Table 3 reports Krippendorff's alpha of 0.15 for Usefulness and 0.07 for Humanness. An alpha of 0.07 indicates essentially no agreement beyond chance. The human evaluation results are therefore unreliable despite being reported as significant."
     },
     {
       "flag": "No contamination analysis",
       "detail": "All three evaluation benchmarks (DSTC7, DSTC11, OTT-QA) are publicly available and may have been in ChatGPT's training data. The baseline ChatGPT performance may be artificially inflated or deflated due to data contamination, which is not discussed."
     },
     {
       "flag": "No variance reported for main results",
       "detail": "The main automatic evaluation results in Tables 1, 2, and 5 appear to be single-run results with no variance reporting. Only the RL training curve (Figure 3) shows multi-run spread."
     }
   ],
   "cited_papers": [
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandara Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper proposing joint retriever-generator architectures for knowledge-grounded text generation."
     },
     {
       "title": "REALM: Retrieval-Augmented Language Model Pre-training",
       "authors": ["Kelvin Guu", "Kenton Lee", "Zora Tung"],
       "year": 2020,
       "relevance": "Pioneering work on retrieval-augmented language model pretraining, directly relevant to augmenting LLMs with external knowledge."
     },
     {
       "title": "Retrieval Augmentation Reduces Hallucination in Conversation",
       "authors": ["Kurt Shuster", "Spencer Poff", "Moya Chen"],
       "year": 2021,
       "arxiv_id": "2104.07567",
       "relevance": "Directly addresses hallucination reduction through retrieval augmentation, a core premise of this paper."
     },
     {
       "title": "REPLUG: Retrieval-Augmented Black-Box Language Models",
       "authors": ["Weijia Shi", "Sewon Min", "Michihiro Yasunaga"],
       "year": 2023,
       "relevance": "Contemporary approach to augmenting black-box LLMs with retrieval, directly comparable to LLM-AUGMENTER."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessi"],
       "year": 2023,
       "relevance": "Related approach to augmenting LLMs with external tool use, relevant to the agentic AI tool-use paradigm."
     },
     {
       "title": "WebGPT: Browser-assisted question-answering with human feedback",
       "authors": ["Reiichiro Nakano", "Jacob Hilton", "Suchir Balaji"],
       "year": 2021,
       "arxiv_id": "2112.09332",
       "relevance": "Combines web knowledge with LLMs for question answering, a closely related system to LLM-AUGMENTER."
     },
     {
       "title": "Rethinking with Retrieval: Faithful Large Language Model Inference",
       "authors": ["Hangfeng He", "Hongming Zhang", "Dan Roth"],
       "year": 2022,
       "relevance": "Proposes making LLMs more faithful through retrieval, directly relevant to the hallucination reduction goal."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper relevant to understanding the LLMs being augmented in this work."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT/RLHF paper relevant to understanding ChatGPT's instruction-following capabilities used in LLM-AUGMENTER."
     },
     {
       "title": "Improving Alignment of Dialogue Agents via Targeted Human Judgements",
       "authors": ["Amelia Glaese", "Nathan McAleese", "Maja Trkebacz"],
       "year": 2022,
       "relevance": "Inspired the utility function design in LLM-AUGMENTER for evaluating response alignment."
     },
     {
       "title": "GODEL: Large-Scale Pre-Training for Goal-Directed Dialog",
       "authors": ["Baolin Peng", "Michel Galley", "Pengcheng He"],
       "year": 2022,
       "relevance": "Prior work by the same authors on dialog systems, providing the evaluation protocol used in this paper."
     }
   ]
 }