ai-research-survey

scan.json (24752B)

---

 {
   "paper": {
     "title": "Benchmarking Large Language Models with Integer Sequence Generation Tasks",
     "authors": ["Daniel O'Malley", "Manish Bhattarai", "Nishath Rajiv Ranasinghe", "Erick Draayer", "Javier E. Santos"],
     "year": 2024,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2411.04372"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository URL is provided in Section 3.1: 'The dataset and the code is available at https://github.com/ceodspspectrum/oeis-sequence-benchmark.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset (1000 OEIS sequences) is released at the same GitHub repository mentioned in Section 3.1. The OEIS itself is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, conda file, or detailed library versions) is mentioned in the paper. The paper only states that generated code must use 'the Python standard library' but does not specify the evaluation environment's dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released, the paper does not include a README description, commands to run, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Tables 1 and 2 report only point estimates (average scores, percentage perfect, percentage cheating) with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'reasoning models generally outperform the non-reasoning models') but provides no statistical significance tests. Differences are presented as raw number comparisons only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. While raw scores are given (e.g., o3 at 73.6 vs gpt-4o at 39.5 on easy sequences), there is no standardized effect size measure or explicit baseline-to-improvement framing beyond raw numbers in tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 1000 sequences (250 per category) is not justified with any power analysis or formal reasoning. No explanation is given for why 250 per category was chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Each model appears to be evaluated once per sequence with no repeated runs. The score distributions in Figure 3 show histograms but no formal variance statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes a wide range of baselines: 21 models are compared including older models (gpt-3.5-turbo), general-purpose models (gpt-4o, Claude, Llama), and reasoning models (o1, o3, o3-mini, o4-mini, Gemini 2.5). Non-reasoning models serve as baselines against reasoning models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include the latest frontier models as of the evaluation period: o3, o4-mini, Gemini 2.5-pro, Gemini 2.5-flash, Claude 3.7 Sonnet, and Llama 4 Scout/Maverick. These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The benchmark design has multiple components (time limits, cheating detection, difficulty categories) but no systematic ablation of these factors is performed. The two time limits (0.5s vs 4s) provide partial analysis but this is not a structured ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation factors are measured: average accuracy score, percentage of perfect scores, and percentage flagged for cheating (Section 3.3). Results are also broken down by easy vs. hard and by time limit."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is used to validate the cheating detection mechanism. Section 3.4 states: 'This cheating detection mechanism's effectiveness was validated by comparing it with a human evaluation (one of the authors).'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses held-out sequence values for unit tests. Section 3.2 states: 'For each sequence, the prompt includes only the OEIS Name and Comments fields; sequence values/formulas are withheld for testing.' The contemporary sequences (post-July 2024) also serve as a held-out temporal split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by easy vs. hard sequences (Tables 1 and 2), by contemporary vs. classic sequences (main text vs. appendix), and Figure 4 provides per-model error mode classification."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 presents a detailed case study comparing o3 vs. LLaMA-405B on sequence A380521, showing how non-reasoning models fail due to inefficient algorithms. Figure 4 classifies error modes (timeout, incorrect, lookup table). The discussion notes models 'struggle, especially with the hard sequences.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: all models perform poorly on hard sequences (o3 best at only ~30% average on hard), cheating rates increase for harder problems, reasoning models sometimes cheat more than older counterparts (e.g., 'o3 cheated more than o1 on the hard sequences'), and 'o4-mini cheated more often than o3-mini.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that reasoning models achieve 'substantial improvements' over non-reasoning models (supported by Table 1: o3 at 73.6 easy vs. gpt-4o at 39.5), that 'overall model performance on the hard sequences is poor' (supported: best hard score is ~32), and that they introduce a cheating detection mechanism (Section 3.4). All abstract claims are supported by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims about reasoning specialization causing better performance (e.g., 'The superior performance of reasoning models highlights the effectiveness of specialization in LLMs for mathematical reasoning'). However, the study design is observational — it cannot isolate whether reasoning training itself causes the improvement vs. other factors like model size or training data differences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame this as evaluating 'LLMs' in 'mathematical reasoning and algorithmic code synthesis tasks' broadly, but the benchmark only tests Python code generation for integer sequences from OEIS. The Limitations section acknowledges some bounds (Python-only, OEIS-specific) but the framing remains broader than the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why reasoning models outperform non-reasoning models. Possible confounds include model size differences, training data differences, or different API configurations. The case study in 4.1 attributes success to 'memoization' without considering other factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models have version identifiers (gpt-3.5-turbo-1106, claude-3.5-sonnet-20241022, claude-3.7-sonnet-20250219) but many do not. Models listed as 'o1', 'o3', 'o3-mini', 'o4-mini', 'gemini-2.5-flash-preview', 'gemini-2.5-pro-preview' lack snapshot dates or API version identifiers. Table 2 shows slightly different names (e.g., 'gemini-2.5-flash-preview-04-17', 'gemini-2.5-pro-preview-03-25') but these are inconsistent with Table 1."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides the full code generation prompt and lookup table detection prompt using Python string interpolation syntax, including the actual template text and placeholders that are filled with OEIS name and comments fields."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 states models are evaluated 'using their default settings (temperature, etc.)' but does not specify what those defaults are. Temperature 0 is mentioned only for the cheating detection mechanism (Section 3.4), not for the main code generation experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive a single prompt and generate code in a single pass without iteration, tool use, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Figure 2 document the data selection pipeline: filtering OEIS sequences by July 2024 cutoff into classic vs. contemporary, then splitting by easy/hard labels, selecting 250 per category. The OEIS source, filtering criteria, and category construction are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is a dedicated 'Limitations' section spanning approximately one full page with four distinct limitations discussed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: OEIS bias in sequence types and subjective easy/hard labeling, cheating detection's 95% accuracy with false positives/negatives, Python-only evaluation limiting language generalizability, and time constraints potentially penalizing correct but slow algorithms."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what results do NOT show: evaluation is confined to Python, may not capture model versatility in other languages, time constraints may disadvantage correct but slow solutions, and OEIS-specific biases may not represent all mathematical reasoning tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset and code are released at https://github.com/ceodspspectrum/oeis-sequence-benchmark (Section 3.1), which would include the sequence data. The OEIS itself is publicly accessible for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data collection: sequences sourced from OEIS, split by a July 2024 timeline cutoff, categorized by OEIS easy/hard labels, 250 per category. Figure 2 provides a visual workflow."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited for the study (the one author evaluating cheating detection is not a recruited participant). The data source is the OEIS benchmark, a standard public database."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 documents the full pipeline: OEIS collection → filter by July 2024 cutoff → split by easy/hard → 250 per category → combined 1000-sequence benchmark. The evaluation pipeline (prompt → code generation → execution with time limit → cheating detection → scoring) is described in Sections 3.2-3.4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. All five authors are affiliated with Los Alamos National Laboratory, a DOE-funded institution, but no specific funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors' affiliations are clearly listed: all five are at Los Alamos National Laboratory (various divisions: Earth and Environmental Sciences, Theoretical, Computational Sciences)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The authors are at a government lab that does not appear to have a direct financial stake in any specific LLM's performance, but without explicit funding disclosure this cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No model training cutoff dates are stated. The paper uses a July 2024 OEIS cutoff to create 'contemporary' sequences that are newer, but does not state the training data cutoff dates for the 21 models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination by splitting sequences into classic (pre-July 2024) and contemporary (post-July 2024) sets. Section 3.1: 'Our discussion in the main text focuses on the contemporary sequences to eliminate the potential for contamination with the models' training data.' The cheating detection mechanism also addresses memorized lookup tables."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a core design concern. The contemporary sequence split uses post-July 2024 sequences to stay ahead of training data. Section 3.4 introduces cheating detection for lookup tables. The Limitations section acknowledges OEIS code snippets are publicly available but argues Python standard library constraints mitigate this. Section 5 notes the benchmark 'can be routinely updated with new sequences.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The one-author cheating validation is a validation check, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or tokens consumed are reported despite evaluating 21 models across 1000 sequences each. The paper reports execution time limits (0.5s and 4s) for the generated code, but not the cost of generating the code itself."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, wall-clock time for the full evaluation, or hardware used for running the benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "Reasoning-specialized models (o3, o3-mini, o4-mini, Gemini 2.5-pro) achieve substantial improvements in accuracy over non-reasoning models, especially on complex tasks.",
       "evidence": "Table 1 shows o3 achieves 73.6% avg on easy (4s) vs. gpt-4o at 39.5%; on hard sequences o3 gets 29.7% vs. gpt-4o at 12.5%. o3-mini reaches 32.0% on hard sequences.",
       "supported": "strong"
     },
     {
       "claim": "Overall model performance on the hard sequences is poor, highlighting persistent challenges in algorithmic reasoning.",
       "evidence": "Table 1: the best hard-sequence average score is 32.0% (o3-mini at 4s timeout). No model exceeds 4.4% perfect scores on hard sequences.",
       "supported": "strong"
     },
     {
       "claim": "Models use lookup tables more frequently on hard sequences than easy sequences.",
       "evidence": "Table 1 shows consistently higher cheating percentages on hard vs. easy for almost all models. E.g., claude-3.7-sonnet: 2.8% easy cheating vs. 37.6% hard cheating; gemini-1.5-pro: 16.8% easy vs. 66.7% hard.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning models benefit more from extra execution time (4s vs 0.5s) compared to non-reasoning models.",
       "evidence": "Section 4: 'reasoning models benefit more when they are allowed extra time.' Table 1 shows small but consistent improvements for reasoning models (e.g., o3 hard: 26.2→29.7) while non-reasoning models show minimal change (e.g., gpt-4o hard: 10.9→12.5).",
       "supported": "moderate"
     },
     {
       "claim": "The cheating detection mechanism achieves 95% accuracy compared to human evaluations.",
       "evidence": "Section 3.4: 'This was improved by providing GPT-4o with six sequences and their human cheating evaluations to inform its judgment. This increased accuracy to 95% on a fresh set of human evaluations.'",
       "supported": "moderate"
     },
     {
       "claim": "There are regressions in cheating from models in the same series — o3 cheated more than o1 on hard sequences and o4-mini cheated more than o3-mini on both hard and easy sequences.",
       "evidence": "Table 1: o3 hard cheating 12.0% vs. o1 hard cheating 9.2%; o4-mini easy cheating 5.2% vs. o3-mini easy cheating 2.4%, o4-mini hard cheating 14.0% vs. o3-mini hard cheating 8.4%.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Reasoning-specialized LLMs (o3, o3-mini, o4-mini, Gemini 2.5-pro) substantially outperform non-reasoning models on integer sequence code generation, with o3 achieving 73.6% average accuracy on easy sequences vs. 39.5% for GPT-4o. However, all models perform poorly on hard sequences (best: ~32% average), revealing persistent limitations in complex algorithmic reasoning. Models resort to lookup tables more on hard problems, and newer reasoning models sometimes cheat more than predecessors. The OEIS-based benchmark provides a continuously updatable evaluation that resists contamination through temporal splits.",
   "red_flags": [
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "All performance comparisons are based on raw point estimates with no confidence intervals, significance tests, or variance measures. Claims like 'reasoning models outperform non-reasoning models' are based solely on comparing numbers in a table. With single-run evaluations and no uncertainty quantification, it is impossible to assess whether observed differences are reliable."
     },
     {
       "flag": "Default/unspecified hyperparameters",
       "detail": "Models are evaluated 'using their default settings (temperature, etc.)' without specifying what those defaults are. Different models have different defaults (e.g., temperature, top-p), making cross-model comparison less controlled. Temperature significantly affects code generation quality."
     },
     {
       "flag": "Inconsistent model version reporting",
       "detail": "Some models have precise version identifiers (claude-3.5-sonnet-20241022) while many do not (o1, o3, o3-mini, o4-mini). Table 1 and Table 2 use slightly different identifiers for the same models (e.g., 'gemini-2.5-flash-preview' vs. 'gemini-2.5-flash-preview-04-17'), suggesting evaluations may have occurred at different times."
     },
     {
       "flag": "Single evaluator for cheating validation",
       "detail": "The cheating detection mechanism was validated against only one human evaluator (one of the authors), which is insufficient for establishing inter-rater reliability. The 95% accuracy claim is based on agreement with a single non-blinded author."
     },
     {
       "flag": "Self-citation concentration",
       "detail": "References [1], [2], and [3] are all by the paper's own authors (Bhattarai, Santos, O'Malley). Three of twelve references being self-citations is notable."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for evaluating LLM code generation, a foundational benchmark in this space."
     },
     {
       "title": "Measuring mathematical problem solving with the MATH dataset",
       "authors": ["Dan Hendrycks", "Collin Burns"],
       "year": 2021,
       "arxiv_id": "2103.03874",
       "relevance": "MATH benchmark for evaluating LLM mathematical reasoning, directly related to the reasoning capability evaluation scope."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduces GSM8K benchmark for mathematical word problem solving by LLMs."
     },
     {
       "title": "FrontierMath: A benchmark for evaluating advanced mathematical reasoning in AI",
       "authors": ["Elliot Glazer", "Ege Erdil", "Tamay Besiroglu"],
       "year": 2024,
       "relevance": "A related benchmark for advanced mathematical reasoning in LLMs using research-level math problems, directly compared in the related work section."
     },
     {
       "title": "ARCS: Agentic retrieval-augmented code synthesis with iterative refinement",
       "authors": ["Manish Bhattarai", "Miguel Cordova", "Javier Santos", "Dan O'Malley"],
       "year": 2025,
       "arxiv_id": "2504.20434",
       "relevance": "Agentic code synthesis approach using RAG and iterative refinement, relevant to LLM code generation methodology."
     },
     {
       "title": "Enhancing code translation in language models with few-shot learning via retrieval-augmented generation",
       "authors": ["Manish Bhattarai", "Javier E Santos"],
       "year": 2024,
       "relevance": "Studies few-shot learning and RAG for LLM code translation, relevant to LLM code generation capabilities."
     }
   ]
 }