ai-research-survey

scan.json (25938B)

---

 {
   "paper": {
     "title": "Lost in the Middle: How Language Models Use Long Contexts",
     "authors": [
       "Nelson F. Liu",
       "Kevin Lin",
       "John Hewitt",
       "Ashwin Paranjape",
       "Michele Bevilacqua",
       "Fabio Petroni",
       "Percy Liang"
     ],
     "year": 2023,
     "venue": "TACL",
     "arxiv_id": "2307.03172"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we release our code and evaluation data' and provides the URL nelsonliu.me/papers/lost-in-the-middle (Section 1, footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available NaturalQuestions-Open dataset and states they release their evaluation data at the project URL. The synthetic key-value retrieval task uses randomly generated UUIDs with a described generation process."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of environment specifications, requirements files, or dependency details found in the paper. The paper describes using APIs (OpenAI, Anthropic) and open models (MPT, LongChat, Llama-2) but does not provide a reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper itself does not contain step-by-step reproduction instructions, a README description with commands, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Figures 5, 7, 8, 9, etc.) report only point estimates of accuracy without confidence intervals or error bars. Token count statistics in Appendix F include standard deviations, but the main experimental results do not."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparative claims (e.g., 'performance degrades significantly', 'performance is nearly identical') but uses no statistical significance tests. Comparisons are based solely on visual inspection of accuracy numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports effect sizes in context. For example: 'GPT-3.5-Turbo's multi-document QA performance can drop by more than 20%' (Section 2.3), '1.5% for GPT-3.5-Turbo and ~1% for Claude-1.3' (Section 5), '10-point worst-case degradation' (Appendix E). These provide baseline context for interpreting magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 2655 queries from NaturalQuestions-Open and 500 examples for key-value retrieval, but does not justify why these sizes were chosen or discuss whether they provide sufficient statistical power for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Experiments appear to be single-run evaluations. No standard deviations across experimental runs are reported for the main accuracy results. The only variance reported is for token counts in Appendix F, not for experimental outcomes."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes closed-book and oracle baselines (Table 1) to contextualize performance. Multiple models are compared against each other, and extended-context variants are compared against their standard counterparts."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated (GPT-3.5-Turbo, Claude-1.3, MPT-30B-Instruct, LongChat-13B, GPT-4, Llama-2) were all state-of-the-art or near-state-of-the-art at the time of publication in 2023."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes several ablation-style experiments: Section 4.1 (decoder-only vs. encoder-decoder), Section 4.2 (query-aware contextualization), Section 4.3 (instruction fine-tuning vs. base models), Appendix B (random vs. retrieved distractors), and Appendix C (randomized distractor order)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only accuracy (exact match) as the evaluation metric across all experiments. No additional metrics such as F1 or partial match are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates language model capabilities on factual retrieval tasks with ground-truth answers. Human evaluation of outputs is not relevant since correctness is objectively determined by exact match against annotated answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses NaturalQuestions-Open, a standard benchmark with established test data. No tuning or model selection is performed on the evaluation data; all models are evaluated zero-shot with fixed prompts."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by position of relevant information (multiple positions per context length), by context length (10, 20, 30 documents; 75, 140, 300 key-value pairs), by model, and appendices provide per-model tabulations (Tables 5-7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific failure modes: models performing worse than closed-book when information is in the middle (Section 2.3), LongChat-13B generating code instead of answers (Section 3.2), and the general failure pattern of U-shaped performance curves."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: extended-context models not improving over standard models (Section 2.3), query-aware contextualization not helping multi-document QA despite helping key-value retrieval (Section 4.2), and additional retrieved documents not improving reader performance (Section 5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that performance degrades when relevant information position changes, that there is a U-shaped performance curve with primacy and recency bias, and that extended-context models are not necessarily better. All of these are supported by the results in Sections 2.3, 3.2, and the figures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's ablation studies (Sections 4.1-4.3) use controlled single-variable manipulation to investigate causal factors. The main experiments use controlled positioning of information to establish causal effects of position on performance. The study design is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about generalization. It specifies exact models tested, limits claims to 'current language models', and explicitly notes limitations such as the cost of evaluating GPT-4 on full experiments (Appendix D footnote). The title 'How Language Models Use Long Contexts' is broad but the paper tests multiple model families to support breadth."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically investigates alternative explanations: whether the U-shape is due to decoder-only architecture (Section 4.1), lack of query-aware contextualization (Section 4.2), instruction fine-tuning bias (Section 4.3), hard negatives vs. random distractors (Appendix B), and ranking prior in prompts (Appendix C)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies model versions: 'gpt-3.5-turbo-0613', 'gpt-3.5-turbo-16k-0613', 'gpt-4-0613' (Section 2.2, footnote 5, Figure 15), Claude-1.3, Claude-1.3 (100K), MPT-30B-Instruct, LongChat-13B (16K), specific Llama-2 variants with parameter counts."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The exact prompts are provided in Figures 2, 6 for the multi-document QA and key-value retrieval tasks respectively. The modified prompt for randomized ordering is quoted in Appendix C."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We use greedy decoding when generating outputs' (Section 2.2), which fully specifies the generation strategy (temperature=0, no sampling). For API models this is the key hyperparameter."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The experiments involve direct prompting of language models without any agent framework, tool use, or multi-step reasoning pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the data pipeline in detail: 2655 queries from NaturalQuestions-Open filtered to paragraph-type long answers, Contriever retrieval of distractor documents, filtering of distractors that contain annotated answers. The key-value retrieval task generation is described in Section 3.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The paper discusses some caveats inline (e.g., cost of GPT-4 evaluation, greedy decoding only) but lacks a substantive standalone limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address, for example, whether results might change with different prompts, different tasks, non-English inputs, or different evaluation metrics."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While it mentions leaving exploration of other decoding methods to future work (Section 2.2), it does not systematically bound the scope of its conclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and evaluation data are released at nelsonliu.me/papers/lost-in-the-middle, and the underlying NaturalQuestions-Open dataset is publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the data collection in detail: NaturalQuestions-Open queries, filtering criteria (paragraph-type long answers yielding 2655 queries), Contriever retrieval for distractors, and the synthetic key-value generation process (Section 3.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data comes from standard benchmarks (NaturalQuestions-Open) and synthetic generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is well documented: NaturalQuestions-Open queries → filter to paragraph answers (2655 queries) → Contriever retrieval for distractors → filter out distractors containing answers → controlled reordering for position experiments. Appendix A addresses ambiguity filtering."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was supported by the Stanford Center for Research on Foundation Models (CRFM), by OpenAI via an API credits grant to the Stanford CRFM, and by Anthropic via the Claude academic access program.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Stanford University, UC Berkeley, and Samaya AI. The footnote notes work was 'partially completed as an intern at Samaya AI.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI and Anthropic provided API credits/access, and their models (GPT-3.5-Turbo, GPT-4, Claude-1.3) are the primary subjects of evaluation. Both companies have a financial interest in the perceived capabilities of their long-context models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Some authors are affiliated with Samaya AI, a company that may have interests related to retrieval-augmented generation, but no financial interest disclosures are provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the models evaluated. While model versions are specified (e.g., gpt-3.5-turbo-0613), the training cutoff is not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether NaturalQuestions-Open data appeared in any model's training data. The benchmark was published in 2019, well before all models' training periods, making contamination plausible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NaturalQuestions was published in 2019 and is widely available online. Models trained after 2019 may have seen these questions. The paper does not address this contamination risk, though it is partially mitigated by the controlled nature of the experiments (same questions across conditions)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that 'evaluating GPT-4 on the full multi-document QA and key-value retrieval experiments would cost upwards of $6000' (Appendix D footnote), but does not report the actual costs incurred for the experiments that were run."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, GPU hours for open models, or wall-clock time for experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Language model performance exhibits a U-shaped curve as a function of the position of relevant information in the input context, with highest performance at the beginning and end.",
       "evidence": "Figures 5 and 7 show this pattern across multiple models and both multi-document QA and key-value retrieval tasks. For example, GPT-3.5-Turbo drops by >20% when relevant information is in the middle vs. beginning/end (Section 2.3).",
       "supported": "strong"
     },
     {
       "claim": "Extended-context models are not necessarily better at using input context than their standard-context counterparts.",
       "evidence": "Figure 5 shows GPT-3.5-Turbo and GPT-3.5-Turbo (16K) have 'nearly superimposed' performance curves on 10- and 20-document settings that fit within both context windows (Section 2.3). Same pattern for Claude-1.3 vs Claude-1.3 (100K).",
       "supported": "strong"
     },
     {
       "claim": "Encoder-decoder models are more robust to position of relevant information within their training-time context window.",
       "evidence": "Figure 8 shows Flan-UL2 has only 1.9% absolute difference between best and worst case within its 2048-token window, but exhibits U-shaped degradation on longer sequences (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Query-aware contextualization dramatically improves key-value retrieval but minimally affects multi-document QA.",
       "evidence": "Section 4.2 reports GPT-3.5-Turbo (16K) achieves perfect performance on 300 key-value pairs with query-aware contextualization vs. 45.6% worst-case without it. Figure 9 shows minimal change for QA.",
       "supported": "strong"
     },
     {
       "claim": "Model performance saturates long before retriever recall in open-domain QA, indicating models fail to use additional retrieved documents effectively.",
       "evidence": "Figure 11 shows that using 50 instead of 20 retrieved documents improves performance by only ~1.5% for GPT-3.5-Turbo and ~1% for Claude-1.3, while retriever recall continues to increase (Section 5).",
       "supported": "strong"
     },
     {
       "claim": "The U-shaped performance curve appears even in base language models before instruction fine-tuning.",
       "evidence": "Figure 10 shows both MPT-30B and MPT-30B-Instruct exhibit U-shaped curves, though instruction fine-tuning slightly reduces the performance disparity (Section 4.3).",
       "supported": "strong"
     },
     {
       "claim": "Only sufficiently large language models (13B+ parameters) exhibit primacy bias; smaller models (7B) are solely recency-biased.",
       "evidence": "Figure 16 and Appendix E show Llama-2-7b models are solely recency-biased while 13B and 70B models exhibit both primacy and recency bias.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Language models exhibit a U-shaped performance curve when accessing information at different positions in long input contexts, performing best when relevant information is at the beginning or end and worst when it is in the middle. Extended-context models (e.g., GPT-3.5-Turbo-16K, Claude-1.3-100K) do not improve context utilization over their standard counterparts. Encoder-decoder models are more robust to information position within their training-time context window. In practical open-domain QA, reader performance saturates far before retriever recall, suggesting current models cannot effectively leverage additional retrieved documents.",
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All main results are reported as single-run point estimates without confidence intervals, error bars, or variance across runs. Given the stochastic nature of some models and the claim-heavy analysis, this is a notable gap."
     },
     {
       "flag": "No significance tests for comparative claims",
       "detail": "The paper makes numerous claims about performance differences (e.g., 'nearly identical', 'significantly degrades') without any statistical tests. Whether observed differences are statistically significant or within noise is never tested."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "NaturalQuestions was published in 2019 and is widely available. All evaluated models were trained after 2019. The paper does not discuss whether models may have seen these questions during training, though the controlled within-question position manipulation partially mitigates this concern."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Key limitations such as the use of a single evaluation metric (accuracy), a single task domain (factual QA), English-only evaluation, and greedy decoding only are not systematically discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì", "Roberta Raileanu", "Maria Lomeli", "Luke Zettlemoyer", "Nicola Cancedda", "Thomas Scialom"],
       "year": 2023,
       "relevance": "Foundational work on LLM tool use and agentic capabilities, directly relevant to understanding how models augmented with external information use their context."
     },
     {
       "title": "REPLUG: Retrieval-Augmented Black-Box Language Models",
       "authors": ["Weijia Shi", "Sewon Min", "Michihiro Yasunaga", "Minjoon Seo", "Rich James", "Mike Lewis", "Luke Zettlemoyer", "Wen tau Yih"],
       "year": 2023,
       "arxiv_id": "2301.12652",
       "relevance": "Proposes retrieval-augmented generation for black-box LLMs, directly relevant to understanding how retrieved context is used by models."
     },
     {
       "title": "FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness",
       "authors": ["Tri Dao", "Daniel Y. Fu", "Stefano Ermon", "Atri Rudra", "Christopher Ré"],
       "year": 2022,
       "arxiv_id": "2205.14135",
       "relevance": "Key algorithmic contribution enabling longer context windows in transformers, foundational infrastructure for long-context LLM research."
     },
     {
       "title": "Hyena Hierarchy: Towards Larger Convolutional Language Models",
       "authors": ["Michael Poli", "Stefano Massaroli", "Eric Nguyen", "Daniel Y. Fu", "Tri Dao", "Stephen Baccus", "Yoshua Bengio", "Stefano Ermon", "Christopher Ré"],
       "year": 2023,
       "relevance": "Alternative to attention-based architectures for long-context processing, relevant to understanding architectural approaches to long-context modeling."
     },
     {
       "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Major open-source LLM evaluated in this study; relevant to understanding the scaling and fine-tuning effects on context utilization."
     },
     {
       "title": "Scaling Instruction-Finetuned Language Models",
       "authors": ["Hyung Won Chung", "Le Hou", "Shayne Longpre"],
       "year": 2022,
       "arxiv_id": "2210.11416",
       "relevance": "Describes instruction fine-tuning at scale (Flan models), which this paper investigates as a potential factor in context utilization patterns."
     },
     {
       "title": "In-Context Retrieval-Augmented Language Models",
       "authors": ["Ori Ram", "Yoav Levine", "Itay Dalmedigos", "Dor Muhlgay", "Amnon Shashua", "Kevin Leyton-Brown", "Yoav Shoham"],
       "year": 2023,
       "arxiv_id": "2302.00083",
       "relevance": "Explores retrieval-augmented generation with in-context learning, directly relevant to the practical implications of context position effects."
     },
     {
       "title": "Large Language Models Struggle to Learn Long-Tail Knowledge",
       "authors": ["Nikhil Kandpal", "Haikang Deng", "Adam Roberts", "Eric Wallace", "Colin Raffel"],
       "year": 2022,
       "arxiv_id": "2211.08411",
       "relevance": "Studies LLM knowledge limitations, providing context for why retrieval augmentation matters and how models use provided information."
     },
     {
       "title": "Natural Questions: A Benchmark for Question Answering Research",
       "authors": ["Tom Kwiatkowski", "Jennimaria Palomaki", "Olivia Redfield"],
       "year": 2019,
       "relevance": "The primary benchmark dataset used in this study; foundational QA evaluation resource for LLM research."
     },
     {
       "title": "LLaMA: Open and Efficient Foundation Language Models",
       "authors": ["Hugo Touvron", "Thibaut Lavril", "Gautier Izacard"],
       "year": 2023,
       "arxiv_id": "2302.13971",
       "relevance": "Foundation model that LongChat-13B extends, relevant to understanding architectural choices for long-context modeling."
     }
   ]
 }