ai-research-survey

scan.json (24442B)

---

 {
   "paper": {
     "title": "BAMBOO: A Comprehensive Benchmark for Evaluating Long Text Modeling Capacities of Large Language Models",
     "authors": [
       "Zican Dong",
       "Tianyi Tang",
       "Junyi Li",
       "Wayne Xin Zhao",
       "Ji-Rong Wen"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2309.13345"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states in the abstract: 'We release our data, prompts, and code at https://github.com/RUCAIBox/BAMBOO.' A concrete GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark datasets are released via the same GitHub repository. The abstract states 'We release our data, prompts, and code.' Table 2 also lists the public source URLs for each dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or specific library versions in the paper. The paper does not describe the environment setup needed to reproduce the experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that users can 'utilize the BAMBOO benchmark with instructions provided in our project repository' (Section 3.5), but the paper itself does not contain step-by-step reproduction instructions. It defers to the repository, and the paper itself lacks sufficient detail."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-7 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'ChatGPT-16k consistently demonstrates optimal performance') but uses no statistical significance tests. Comparisons are based solely on raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw percentages without formal effect sizes (no Cohen's d, odds ratios, etc.). While absolute performance numbers are given, there is no systematic reporting of magnitude of differences with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 100-200 examples per dataset (Table 2) but no justification is given for why these sizes are adequate for the claims being made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs with no standard deviation, variance, or spread measure reported across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes random baselines for each task (Section 4.1) and compares five long-context LLMs against each other. Table 3 shows random baseline performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The five models tested (gpt-3.5-turbo-16k, Claude2-100k, ChatGLM2-6b-32k, Vicuna-7b-v1.5-16k, Longchat-7b-v1.5-32k) were all contemporary long-context models at the time of the study (2023)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes multiple analytical experiments that function as ablations: RQ1 (extension tax comparison), RQ2 (evidence vs. complete text), RQ3 (instruction positions), RQ4 (context compression methods), and RQ5 (evidence position effects). These systematically vary individual factors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used across different tasks: accuracy, concordance index, pass@1, and precision/recall/F1 (Table 2, Section 3.4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly avoids human evaluation, relying entirely on automatic metrics. The paper argues for 'accurate automatic evaluation' as a design principle (Section 3.1). While human annotators helped construct the benchmark, no human evaluation of model outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no mention of a dev/test split. The benchmark appears to be used as a single evaluation set without any distinction between tuning and evaluation data. The paper does not describe holding out any portion of the data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-dataset breakdowns across all 10 datasets and both length levels (4k and 16k). Performance is shown individually for each task rather than as a single aggregate."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 'Discussions' identifies specific failure modes: instruction forgetting, format errors, poor reasoning beyond long texts, and poor performance on uncommon tasks. Qualitative error patterns are described."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: models performing below random baselines on some tasks (Table 3, e.g., Longchat on ShowsPred), the 'extension tax' hurting short-text performance (RQ1), and summarization/truncation methods often performing poorly (RQ4, Table 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims about comprehensive evaluation, data contamination avoidance, and key findings (extending context is a double-edged sword, evidence/compression helps, instruction position sensitivity) are supported by results in Tables 3-7 and the RQ analyses in Section 4.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-style claims such as 'extending the context window...is beneficial for medium-length texts but harmful for short texts' and 'limited diversity of training data types plays a role in the subpar performance.' These are causal claims (X causes Y) but the study design (comparing different models, not controlled manipulation of training data) does not adequately support causal inference. Confounds between models are not addressed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Comprehensive Benchmark for Evaluating Long Text Modeling Capacities of Large Language Models' broadly, but results are from only 5 models (2 closed-source, 3 open-source 7B models). The paper does not bound its generalizations to these specific models. Claims like 'LLMs perform poorly on uncommon tasks' generalize beyond the tested set."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents interpretations for its findings (e.g., attributing poor performance to 'limited diversity of training data') without discussing alternative explanations. For example, the 'extension tax' could be due to model capacity, not just position interpolation, but alternatives are not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'gpt-3.5-turbo-16k' and 'Claude2-100K' without specifying exact API snapshots or version dates. For open models, 'Vicuna-7b-v1.5-16k', 'Longchat-7b-v1.5-32k', and 'ChatGLM2-6b-32k' are more specific but no checkpoint hashes or exact dates are provided. The closed-source models lack snapshot identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We release our data, prompts, and code.' The prompts are included as part of the released artifacts. The paper describes the task formats in detail (Section 3.3) and prompts are part of the released repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other generation hyperparameters are reported in the paper. For API-based models (GPT-3.5-turbo, Claude2), no sampling settings are specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The benchmark evaluates LLMs directly on tasks via single-turn prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes data collection: sources are specified, text extraction excludes images/tables/footnotes, texts under 1000 tokens are excluded, long texts are truncated. Section 3.3 details how each dataset was constructed. The process from raw sources to final datasets is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. Section 4.4 'Discussions' discusses problems of LLMs but not limitations of the benchmark or study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity of the benchmark or evaluation methodology are discussed. The paper acknowledges that 'we cannot guarantee the absence of any pre-training data used in our benchmark' (Section 3.1) but does not address other threats such as small sample sizes per task or limited model diversity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There are no explicit boundary statements about the generalizability of findings to other models, languages, or task types beyond those tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The data is released via GitHub (https://github.com/RUCAIBox/BAMBOO) as stated in the abstract. Table 2 also lists original source URLs for each dataset, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes data collection: four source types (NLP papers, government reports, TV show transcripts, committee meeting transcripts), all from 2023, with specific URLs listed in Table 2. Section 3.3 details construction of each of the 10 datasets."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'With the assistance of human labelers, we manually construct two multi-choice question answering datasets' (Section 3.3) but does not describe who these annotators were, how they were recruited, or what instructions they received beyond rephrasing expressions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw sources to final datasets is documented: collection from specific websites (Section 3.2), parsing and text extraction, filtering by length (>1000 tokens), truncation of long texts, and division into BAMBOO-4k and BAMBOO-16k subsets. Table 2 provides final counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Acknowledgments' discloses funding: 'National Natural Science Foundation of China under Grant No. 62222215, Beijing Natural Science Foundation under Grant No. 4222027 and L233008.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page: Renmin University of China (Gaoling School of AI, School of Information) and Université de Montréal (DIRO). The authors are academic researchers not affiliated with the companies whose models they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (National Natural Science Foundation of China, Beijing Natural Science Foundation) are government research agencies with no financial stake in the outcome of the benchmark evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states: 'training data cutoff for gpt-3.5-turbo and Claude2 are up to September 2021 and the early 2023, respectively.' Training cutoffs are explicitly discussed as part of the contamination avoidance design."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Data contamination is a core design principle. Section 3.1 extensively discusses the issue: data sources are retained only from 2023, answer modifications prevent memorization, and the paper acknowledges 'we cannot guarantee the absence of any pre-training data used in our benchmark.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark is designed to address contamination: using 2023 data sources, modifying answers in AltQA to prevent memorization, and constructing novel datasets (SenHallu, AbsHallu, PaperQA, MeetingQA). This is discussed in Sections 3.1 and 3.2."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Human annotators helped construct the benchmark but were not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. This is a benchmark evaluation of LLMs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time are reported for running the benchmark evaluations across the five models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, API spend, or hardware details are reported for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "ChatGPT-16k consistently demonstrates optimal performance across most datasets, surpassing other LLMs in nearly all tasks.",
       "evidence": "Table 3 shows ChatGPT-16k achieving the highest or near-highest scores on 8 out of 10 datasets across both length levels, with only a minor gap in hallucination detection where Claude2 slightly outperforms it on F1.",
       "supported": "strong"
     },
     {
       "claim": "Extending the context window of LLMs is a double-edged sword: beneficial for medium-length texts but harmful for short texts.",
       "evidence": "Figure 1 (Section 4.3, RQ1) compares short and long-context variants of ChatGPT and Vicuna. Vicuna-16k shows improved BAMBOO-4k performance but decreased MMLU performance compared to Vicuna.",
       "supported": "moderate"
     },
     {
       "claim": "Using evidence and properly compressing long texts can generally enhance LLM performance.",
       "evidence": "Table 4 (RQ2) shows evidence-only input generally improves performance. Table 6 (RQ4) shows retrieval-augmented methods achieve comparable or superior performance to long-context LLMs on some tasks.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs' performances on different datasets are sensitive to positions of instructions, with evidence at both ends being beneficial due to larger attention scores.",
       "evidence": "Table 5 (RQ3) shows varying performance across Pre-Ins, Post-Ins, and Both-Ins configurations. Figure 2 and Table 7 (RQ5) demonstrate U-shaped attention patterns and improved performance with evidence at text endpoints.",
       "supported": "moderate"
     },
     {
       "claim": "BAMBOO avoids data contamination by using data sources released in 2023.",
       "evidence": "Section 3.1 describes the contamination avoidance strategy and Section 3.2 confirms data sources are from 2023. The paper acknowledges this is a mitigation, not a guarantee.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BAMBOO evaluates five long-context LLMs on 10 datasets across 5 task types at two length levels (4k and 16k tokens), with data contamination avoidance as a key design principle. ChatGPT-16k consistently outperforms other models, while open-source models often fail to surpass random baselines on uncommon tasks like text sorting and code completion. The paper identifies that extending context windows can hurt short-text performance ('extension tax'), context compression via retrieval can match long-context models, and instruction position significantly affects performance due to U-shaped attention patterns.",
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All results are point estimates from apparently single runs with no confidence intervals, error bars, significance tests, or variance reporting. Comparative claims like 'consistently demonstrates optimal performance' are based solely on raw number comparisons."
     },
     {
       "flag": "Small sample sizes per dataset",
       "detail": "Individual datasets contain only 100-200 examples (Table 2), which combined with single-run evaluation and no uncertainty quantification makes the reliability of numerical differences unclear."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations or threats-to-validity section. The discussion section only addresses problems of LLMs, not limitations of the benchmark itself (e.g., limited model diversity, potential remaining contamination, small dataset sizes)."
     },
     {
       "flag": "Overgeneralized claims from limited model set",
       "detail": "Claims about 'LLMs' in general are based on only 5 models: 2 closed-source and 3 open-source 7B-parameter models. The open-source models are all relatively small, making claims about LLM capabilities overly broad."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "No generation hyperparameters (temperature, top-p, max tokens) are reported for any model, making results impossible to reproduce exactly and potentially sensitive to undisclosed settings."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and Codex, foundational work on LLM code generation evaluation."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2023,
       "arxiv_id": "2307.03172",
       "relevance": "Key study on positional biases in LLM long-context processing, directly validated by this paper's RQ5."
     },
     {
       "title": "L-eval: Instituting standardized evaluation for long context language models",
       "authors": ["Chenxin An", "Shansan Gong", "Ming Zhong"],
       "year": 2023,
       "arxiv_id": "2307.11088",
       "relevance": "Competing long-context benchmark that BAMBOO compares against in Table 1."
     },
     {
       "title": "Longbench: A bilingual, multitask benchmark for long context understanding",
       "authors": ["Yushi Bai", "Xin Lv", "Jiajie Zhang"],
       "year": 2023,
       "arxiv_id": "2308.14508",
       "relevance": "Competing long-context benchmark discussed as lacking contamination avoidance."
     },
     {
       "title": "Zeroscrolls: A zero-shot benchmark for long text understanding",
       "authors": ["Uri Shaham", "Maor Ivgi", "Avia Efrat"],
       "year": 2023,
       "arxiv_id": "2305.14196",
       "relevance": "Competing long-context benchmark compared in Table 1, criticized for contamination risk."
     },
     {
       "title": "Time travel in llms: Tracing data contamination in large language models",
       "authors": ["Shahriar Golchin", "Mihai Surdeanu"],
       "year": 2023,
       "arxiv_id": "2308.08493",
       "relevance": "Foundational work on data contamination detection that motivates BAMBOO's contamination avoidance design."
     },
     {
       "title": "A survey of large language models",
       "authors": ["Wayne Xin Zhao", "Kun Zhou", "Junyi Li"],
       "year": 2023,
       "arxiv_id": "2303.18223",
       "relevance": "Comprehensive LLM survey that inspired BAMBOO's capacity evaluation framework."
     },
     {
       "title": "HaluEval: A large-scale hallucination evaluation benchmark for large language models",
       "authors": ["Junyi Li", "Xiaoxue Cheng", "Wayne Xin Zhao"],
       "year": 2023,
       "arxiv_id": "2305.11747",
       "relevance": "Hallucination evaluation benchmark used as basis for BAMBOO's hallucination detection task construction."
     },
     {
       "title": "Retrieval meets long context large language models",
       "authors": ["Peng Xu", "Wei Ping", "Xianchao Wu"],
       "year": 2023,
       "arxiv_id": "2310.03025",
       "relevance": "Study on retrieval augmentation vs. long context, directly related to BAMBOO's RQ4 findings."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "LLM evaluation methodology paper; Vicuna model used as a baseline in BAMBOO."
     },
     {
       "title": "Instruction position matters in sequence generation with large language models",
       "authors": ["Yijin Liu", "Xianfeng Zeng", "Fandong Meng"],
       "year": 2023,
       "arxiv_id": "2308.12097",
       "relevance": "Study on instruction positioning that directly motivates BAMBOO's RQ3 experiments on instruction position effects."
     }
   ]
 }