ai-research-survey

scan.json (28948B)

---

 {
   "paper": {
     "title": "Is In-Context Learning Learning?",
     "authors": ["Adrian de Wynter"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2509.10414",
     "doi": "10.48550/arXiv.2509.10414"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "Large-scale empirical study (1.89M predictions per LLM) finds that ICL mathematically constitutes learning but is empirically weak. In the limit (50-100 exemplars), accuracy gaps between LLMs and prompting strategies narrow, and lexical features become less relevant than data features. However, ICL is brittle to out-of-distribution inputs (especially CoT and APO), shows inconsistent performance across formally similar tasks (up to 31% accuracy gaps), and overfocuses on spurious features from the observed distribution rather than learning feature relations within data.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository explicitly provided: 'Code and data is in https://github.com/adewynter/is-icl-learning' (Section 1). Section 10 states it will be open-sourced under MIT licence."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Synthetic data is generated by code in the repository. Section 10: 'All code is included in the repository.' The data generation procedures are fully specified in Section 4.5 and Appendix E."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F specifies: 'Standard ND40rs v2 instance in Azure, equipped with eight NVIDIA Tesla V100 GPUs with 32 Gb of memory each.' API details and model versions in Table 4. Baselines use scikit-learn (Pedregosa et al., 2011)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 10: 'All code is included in the repository.' Detailed methodology in Appendix F, full prompts in Appendix H and repository. Temperature set to zero for reproducibility."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports ±σ (standard deviation) throughout tables, but these represent variation across models or tasks, not confidence intervals. No formal CIs (e.g., 95% CI [x, y]) are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., 'CoT was most sensitive to OOD') based on comparing raw numbers and OLS slopes. No statistical significance tests (p-values, t-tests, etc.) are used to support these claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported throughout as absolute accuracy differences with context: e.g., '31% accuracy difference' between related tasks (Table 1), OLS slopes for trend magnitudes (Table 2), and specific accuracy ranges like '80±3%' vs '16±1%'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Test sets are 2000 entries (1000 evaluated due to cost), training sets 2000 entries. No power analysis or formal justification for these sizes. Cost is mentioned as a constraint but no formal sample size justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to zero, making each run deterministic. The σ values reported (e.g., Tables 1-2) represent variation across models, tasks, or prompts—not across repeated experimental runs. No multiple-seed or multiple-run variance is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Traditional ML baselines included: decision trees (DT), k-nearest neighbours (kNN), and multilayer perceptron (MLP). Section 4.4: 'We tested decision trees, k-nearest neighbours, and a multilayer perceptron in succession, and reported the best.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines (DT, kNN, MLP) are appropriate for the research question of whether ICL constitutes learning compared to traditional learning algorithms. The paper tests 4 contemporary LLMs (GPT-4 Turbo, GPT-4o, Mixtral, Phi-3.5) as the main comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation studies in Section 6: impact of lexical features (word salad, SoT), positionality of exemplars (shuffled vs unshuffled), alternate distributions (imbalanced, random labels), and compliance vs learning separation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary metric is accuracy (1 - error). While OLS slopes are used to measure trends, this is an analysis technique on the same metric rather than an independent metric. No additional metrics like F1, precision/recall, or calibration are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant—tasks are formal language recognition with objectively verifiable correct answers (membership in formal languages)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Clear separation: training set (2000 entries from P) used only for APO and baselines; five separate test sets (1000 entries each) from distributions Q with varying δ. Section 4.5."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns: per-task (9 tasks), per-model (4 LLMs), per-prompt (7 strategies), per-δ (5 distributions). Tables 1-2, Figure 3, and Appendix G provide comprehensive per-category results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases discussed extensively: Vending Machine (Sum) had near-zero slope indicating complete inability to learn (Section 7); SoT had lowest average performance (23±4%); CoT brittleness to OOD; inconsistent performance on related tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Many negative results: CoT and APO brittle to OOD (Section 5.2), SoT near-random performance (Section 6.1), ICL underperforms traditional baselines in half the tasks on average (Section 5.1), related tasks show 31% accuracy gaps (Table 1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 'accuracy is insensitive to exemplar distribution, model, prompt style' supported by Table 2 convergence; 'distributional sensitivity, especially in CoT' supported by δ slopes in Table 2; 'limited all-purpose generalisability' supported by task inconsistency in Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'ICL leverages statistical features from the prompt, as opposed to feature relations within the data') are supported by controlled ablation studies: word salad vs description, shuffled vs unshuffled exemplars, alternate distributions. The experimental design with controlled variables supports these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Generalization explicitly bounded: 'Our findings are constrained to easily-verifiable tasks in a single call' (Section 1.2). Appendix A discusses limitations. Appendix D explicitly discusses impact on conclusions when extended to natural language. Section 4.2 excludes reasoning models and multi-call strategies."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses two alternative explanations: (1) contamination (addressed with synthetic data and complex alphabets), and (2) tokenization effects (BPE impact on arithmetic). Appendix D discusses natural language priors as a factor."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on formal language recognition tasks and claims about ICL as a learning paradigm. The theoretical framework (PAC learning, Section 3) explicitly connects the measurement (empirical error) to the claim (learning ability). The gap between synthetic tasks and real-world ICL use is explicitly acknowledged in Section 1.2 and Appendix D."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 specifies versions: GPT-4 Turbo (GPT-4-0125), GPT-4o (GPT-4-0125), Phi-3.5-MoE-Instruct, Mixtral-8x7B instruct v01. Context windows and parameter counts given for open models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts provided in Appendix H (Prompts 1-5) with actual examples for multiple tasks and strategies. Section 10: prompts also in the repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F: temperature=0, max return tokens=3 (except CoT: 1024, APO: 512). APO: batch size 1024, beam width 4, search depth 6. Baselines: default scikit-learn params, random seed 13213."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. All evaluations are single-call LLM predictions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Synthetic data generation fully documented: automata with transition probabilities (Section 4.5, Figure 1), per-task specifications (Appendix E, Table 3), deduplication, 5% mislabelling rate, alphabet specifications. Baseline data preprocessing described in Appendix F.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A is titled 'Limitations' with substantive discussion of model updates, cost, limited testing paradigms, and baseline interpretation nuances."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: LLMs continuously updated making reproducibility difficult (Appendix A); synthetic data may not translate to natural language (Appendix D); input representation length affects baseline performance (Appendix A); BPE tokenization impact on arithmetic (Section 7)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'constrained to non-natural language tasks' (Section 1.2); 'single call' only (Section 4.2); reasoning models excluded; multi-step strategies excluded; 'results are limited to the ability of ICL to draw conclusions from the data's features alone' (Appendix D)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code to generate all data and run all experiments is in the public repository (Section 1, Section 10). Synthetic data is fully reproducible from the code with temperature=0."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data generation described in detail: Section 4.5 describes automata with transition probabilities, Figure 1 shows PARITY automaton, Appendix E provides full task descriptions with alphabets, ID/OOD characterizations, and string lengths (Table 3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data is synthetically generated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: automata generate strings → 2000 training entries from P → 5 test sets of 2000 from Q at different δ → 1000 evaluated per test set → 5% mislabelling → up to 5 retries per call → compliance/learning separation. Section 4.5 and Appendix F."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. No acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation clearly stated: 'Microsoft and the University of York.' This is relevant as Microsoft's Phi-3.5 is one of the four evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. The author works at Microsoft, which produces one of the evaluated models (Phi-3.5), creating a potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the four models. While the synthetic data mitigates contamination risk, the cutoff dates are not provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses contamination as an alternative explanation. The entire experimental design uses synthetic data with custom alphabets (e.g., '¯\\(ツ)/¯') specifically to prevent overlap with training data. Section 4.5: 'synthetic to account for the task's G and Σ.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a central design consideration. Section 7: 'Contamination could explain the accuracy in Pattern Matching, perhaps due to the (easy) Σ, {a, b, c}. Other tasks, like Reversal, used more complex Σ and had lower scores.' Custom alphabets and synthetic data generation are the primary contamination mitigation strategy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments use synthetic data evaluated by LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Ethics section (Section 9) discusses carbon footprint and potential misuse, not human subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No per-example or total inference cost reported. Section 9 mentions 'very high carbon footprint' qualitatively. Appendix A mentions cost constraints affected decisions (e.g., evaluating 1000 of 2000 test entries) but no dollar amounts or token counts."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware specified (Appendix F: 'Standard ND40rs v2 instance in Azure, eight NVIDIA Tesla V100 GPUs') but no total GPU hours, wall-clock time, or API spend reported. Appendix A notes 'running synchronously a single task per LLM could and has taken months.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature set to zero for deterministic output. No seed sensitivity analysis across different random seeds. While determinism is a strength for reproducibility, it means results may be specific to this one configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F: 'Every model was called up to five times to account for any potential parsing errors or rate limitations from APIs.' Temperature=0 means deterministic output, so one effective run per configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "APO parameters (batch 1024, beam 4, depth 6) are stated but no search budget for selecting them. Baseline parameters left as defaults. No discussion of how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations are reported (all 7 prompting strategies × 7 shot counts × 5 δ values × 4 models), not just the best. Tables 1-2 and Appendix G show comprehensive results across all settings."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable. The paper uses OLS slopes and descriptive statistics rather than hypothesis testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The author is at Microsoft, and Phi-3.5 (a Microsoft model) is one of four evaluated models. This potential bias is not acknowledged. While the study is not proposing a new system, the author's employer's model could receive favorable treatment in analysis."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute. GPT-4 Turbo and GPT-4o (128k context, unknown parameters) are compared with Mixtral (12.9B active) and Phi-3.5 (6.6B active) without discussing compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Extensive discussion of what synthetic formal language tasks measure vs ICL claims. Section 3 provides formal framework linking tasks to learning theory. Appendix D explicitly discusses limitations when extending to natural language. Section 1.2: 'our findings seek to characterise ICL as a learning paradigm, and not as an evaluation of prompt-based problem-solving capabilities.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All evaluations are single-call LLM predictions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Addressed by experimental design: all data is synthetically generated at experiment time, so it cannot have appeared in training data. Section 7 discusses contamination. Custom alphabets further mitigate leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of whether evaluation setup leaks answer information through context. While the synthetic design mitigates many leakage types, the paper does not discuss whether prompt structure or formatting provides hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Training and test sets explicitly drawn from different distributions (P vs Q with controlled δ). Section 4.5: test sets are 'balanced, deduplicated.' Independence is structural by design."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No formal leakage detection method (canary strings, membership inference, n-gram overlap). The approach relies on contamination avoidance through synthetic data generation rather than detection."
       }
     }
   },
   "claims": [
     {
       "claim": "In the limit (50-100 exemplars), the average accuracy gap narrows between LLMs and all prompting strategies steadily improve accuracy.",
       "evidence": "Table 2 shows positive shot slopes for all prompts (3.3-8.3 average). Section 5.3: per-shot σ OLS fit of -2.6±0.5 indicates narrowing model gap. Figure 2 and Figure 4 visualize convergence.",
       "supported": "strong"
     },
     {
       "claim": "ICL is brittle to altered test distributions (OOD), especially in CoT and APO.",
       "evidence": "Table 2: CoT has largest negative δ slope (-1.4), APO at -0.5. Section 5.2: all δ slopes were negative. GPT-4o most sensitive at -1.2 average. Figure 2 shows CoT brittleness visually.",
       "supported": "strong"
     },
     {
       "claim": "Closely-related tasks do not necessarily have closely-related performances, with accuracy differences as large as 31%.",
       "evidence": "Table 1: Pattern Matching (94±1%) vs Maze Solve (63±5%) = 31% gap, both FSA-class. Reversal vs Stack = 12% gap, both PDA-class.",
       "supported": "strong"
     },
     {
       "claim": "Traditional baselines (DT, kNN, MLP) outperform ICL average performance in half of the tasks evaluated.",
       "evidence": "Table 1 rightmost column: MLP beats ICL average in PARITY (95 vs 80), kNN in Pattern Matching (87 vs 94 best-of but context unclear), kNN in Reversal (72 vs 61), kNN in Stack (72 vs 73), DT in V.M. Verification (84 vs 81).",
       "supported": "moderate"
     },
     {
       "claim": "Word salad prompts eventually reach equivalence with description-based prompts in the limit.",
       "evidence": "Table 6 and Section 6.1: word salad matched non-salad best-of to within σ or σ/2 in most tasks. Slopes of 11±4.6 for word salad vs 4.4±2.2 for description. Figure 5 shows convergence.",
       "supported": "strong"
     },
     {
       "claim": "Autoregression can distinguish data features from lexical relations, but cannot fully identify feature relations within the data.",
       "evidence": "Section 7: word salad reaches equivalence (data features learned despite random words); SoT still achieves above-random scores (data features distinguishable); but task inconsistency and OOD brittleness show inability to learn feature relations.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('CoT is most sensitive to OOD', 'Mixtral improved the most') are based on comparing raw numbers and OLS slopes without formal significance testing. Given the large scale (1.89M predictions), tests would be easy to compute and would strengthen the claims."
     },
     {
       "flag": "Single deterministic run per configuration",
       "detail": "Temperature=0 gives determinism but means all results are from a single stochastic realization. No analysis of whether results change with different random states (e.g., different exemplar orderings beyond the one ablation, API non-determinism despite temp=0)."
     },
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Author is affiliated with Microsoft while evaluating Microsoft's Phi-3.5 model alongside competitors. This conflict is not acknowledged in the paper."
     },
     {
       "flag": "Possible version typo",
       "detail": "Table 4 lists both GPT-4 Turbo and GPT-4o with version 'GPT-4-0125', which appears to be the same version string. This may be a typo that could affect reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational paper on in-context learning capabilities of LLMs, central reference for the claims being evaluated."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Key prompting technique evaluated in this study; CoT shown to be brittle to OOD."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski"],
       "year": 2023,
       "relevance": "Data contamination in LLMs—relevant to the survey's contamination assessment category."
     },
     {
       "title": "Awes, laws, and flaws from today's LLM research",
       "authors": ["Adrian de Wynter"],
       "year": 2025,
       "relevance": "Meta-research on LLM evaluation methodology quality—directly relevant to the survey's scope."
     },
     {
       "title": "Quantifying language models' sensitivity to spurious features in prompt design",
       "authors": ["Melanie Sclar", "Yejin Choi", "Yulia Tsvetkov", "Alane Suhr"],
       "year": 2024,
       "relevance": "Prompt sensitivity evaluation for LLMs—key related work on evaluation methodology."
     },
     {
       "title": "Many-shot in-context learning",
       "authors": ["Rishabh Agarwal", "Avi Singh", "Lei M Zhang"],
       "year": 2024,
       "relevance": "Expanding shots improves ICL performance; independently confirms findings of this paper."
     },
     {
       "title": "Faith and fate: Limits of transformers on compositionality",
       "authors": ["Nouha Dziri", "Ximing Lu", "Melanie Sclar"],
       "year": 2023,
       "relevance": "LLM accuracy decay with task complexity—key finding expanded by this paper."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Challenges emergent ability claims in LLMs—relevant to evaluation methodology and overclaiming."
     },
     {
       "title": "Neural networks and the Chomsky Hierarchy",
       "authors": ["Grégoire Délétang", "Anian Ruoss", "Jordi Grau-Moya"],
       "year": 2023,
       "relevance": "Evaluates neural networks on formal language tasks from Chomsky hierarchy—directly comparable methodology."
     },
     {
       "title": "The expressive power of transformers with chain of thought",
       "authors": ["William Merrill", "Ashish Sabharwal"],
       "year": 2024,
       "relevance": "Theoretical analysis of transformer capabilities with CoT—relevant to understanding ICL limitations."
     },
     {
       "title": "Position: Stop making unscientific AGI performance claims",
       "authors": ["Patrick Altmeyer", "Andrew M. Demetriou", "Antony Bartlett", "Cynthia C. S. Liem"],
       "year": 2024,
       "relevance": "Meta-research critique of LLM evaluation claims—directly relevant to survey's methodology assessment."
     },
     {
       "title": "GSM-symbolic: Understanding the limitations of mathematical reasoning in large language models",
       "authors": ["Seyed Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi"],
       "year": 2025,
       "relevance": "Benchmark evaluating LLM reasoning limitations—relevant to understanding ICL capability boundaries."
     }
   ]
 }