ai-research-survey

scan.json (31643B)

---

 {
   "paper": {
     "title": "Formula-One Prompting: Adaptive Reasoning Through Equations For Applied Mathematics",
     "authors": [
       "Natapong Nitarach",
       "Pittawat Taveekitworachai",
       "Kunat Pipatanakul"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.19302",
     "doi": "10.48550/arXiv.2601.19302"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Formula-One Prompting (F-1) uses equation formalization as an intermediate representation before adaptive solving strategy selection, all within a single LLM call. Across 5 models and 4 math benchmarks (2,116 problems), F-1 outperforms Chain-of-Thought by +5.76% and Program-of-Thought by +8.42% on average, with largest gains on applied domains (+13.30% on FinanceMath, +7.24% on AICrypto). Ablation on GPT-5 shows equation formalization contributes roughly twice the improvement of adaptive selection, and strategy selection achieves 73% accuracy on applied domains, reaching 81–84% of the theoretical upper bound.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to implementation is provided anywhere in the paper. Only prompt templates are given in Appendix A."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses four publicly available benchmarks: IMO-Bench (He et al. 2024b), OlympiadBench (He et al. 2024a), FinanceMath (Zhao et al. 2024), and AICrypto (Wang et al. 2025), all referenced with citations."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup details are provided. The paper mentions model names and API-based inference but does not specify library versions, Python version, or dependency information."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The prompt templates in Appendix A and evaluation details in Appendix E provide partial information, but not enough to reproduce the full pipeline without significant effort."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4–8 and throughout the paper are reported as point estimates only. No confidence intervals, error bars, or ± notation appear anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'F-1 outperforms CoT by +5.76%' and similar comparative claims throughout, but no statistical significance tests (p-values, t-tests, bootstrap tests, etc.) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: e.g., '+5.76% over CoT', '+13.30% on FinanceMath over CoT', with full baseline numbers in Table 4 (CoT 43.00% → F-1 56.30% on FinanceMath). The reader can assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or formal justification for benchmark sizes. The Limitations section acknowledges small sizes ('AICrypto contains only 18 problems... OlympiadBench TP_physics includes 25 problems') but provides no justification beyond data scarcity."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Section 4.2 states 'we use a sampling temperature of 0' and 'each question is evaluated using a single generated output' — single-run deterministic results only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three single-call prompting baselines are compared: Zero-Shot (Wei et al. 2022a), Chain-of-Thought (Kojima et al. 2022), and Program-of-Thought (Chen et al. 2023b). Results in Table 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines (CoT, PoT, Zero-Shot) are the standard contemporary single-call prompting methods. The paper explicitly scopes its comparison to single-call methods (Table 1) and justifies excluding multi-call methods due to different compute budgets."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 presents an ablation study removing individual F-1 components: adaptive selection, equation formulation, and givens/targets identification, each tested on GPT-5 across three benchmarks."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy (Table 4), efficiency ratio (accuracy/tokens, Table 9), tokens per correct answer (Table 15), and strategy selection accuracy (Section 6.2). Multiple evaluation dimensions are covered."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. Proof-based problems use LLM-as-Judge (Appendix E) with Gemini-3-Pro and GPT-5.1 as judges. All evaluation is automated — either regex-based or LLM-based."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard benchmark test sets (IMO-Bench, OlympiadBench, FinanceMath, AICrypto). The prompts are fixed templates (Appendix A), not tuned on test data. No dev set is needed or used since the prompting approach is not learned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns in Appendix C: IMO-Bench by AnswerBench/ProofBench (Table 16), OlympiadBench by OE/TP math/physics (Table 17), FinanceMath by 7 financial categories (Table 18), AICrypto by 4 categories (Table 19)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 analyzes strategy selection failures ('Adapt×' category in Table 7). Section 6.3 provides qualitative examples including baseline failure modes. The paper notes IMO-Bench's 90.7% failure rate for all methods and discusses where F-1 doesn't help."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "F-1 underperforms on several specific results: Risk Management in FinanceMath (Table 18: F-1 gets 11.11% vs CoT 22.22% for GPT-5), minimal gains on IMO-Bench (+0.78% over CoT overall), and gains on competition math are near-zero (+0.44% OE_maths). F-1 loses to Zero-Shot on IMO-Bench for GPT-5 (55.58 vs 56.26)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported by Table 4: '+5.76% over CoT' (55.30→61.06), '+8.42% over PoT' (52.64→61.06), '+13.30% on FinanceMath over CoT' (43.00→56.30), physics gains larger than math in Table 5. All numbers match."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — 'equation formalization is the key component' — is supported by the ablation study (Table 6) using controlled single-variable removal. Each ablation variant removes one component while keeping others, constituting adequate causal identification for this type of claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract concludes 'F-1 is more effective than CoT in applied mathematics problems,' bounded to applied math. The title specifies 'For Applied Mathematics.' Limitations section explicitly bounds scope: 'Our study focuses on mathematical reasoning in physics, finance, and cryptography.' Models below 30B are flagged as untested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Training Data Composition' discusses an alternative explanation: F-1's gains may reflect training data composition (applied math content naturally presents equations) rather than the prompting technique itself. The Limitations section discusses model capability dependence."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims stay close to measurement granularity. The paper frames results as benchmark accuracy improvements on specific benchmarks and specific domains, not as improvements to general 'mathematical reasoning.' The abstract concludes with the bounded claim about 'applied mathematics problems.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-5', 'Gemini 2.5 Pro', 'DeepSeek-V3.1', 'Qwen3-235B', 'Qwen3-30B' — marketing names without API versions, snapshot dates, or specific model identifiers. No version pinning is described."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Complete prompt templates for all four benchmarks and all four methods (Zero-Shot, CoT, PoT, F-1) are provided in Appendix A (Sections A.1–A.4), including both system and user prompts with the actual text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'we use a sampling temperature of 0 and leave all other inference hyperparameters at their default values. Since we use greedy decoding, each question is evaluated using a single generated output.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. F-1 is a single-call prompting method — the LLM generates the full response in one generation with no tools, retry logic, or multi-step orchestration."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the evaluation protocol: temperature=0, greedy decoding, regex-based answer extraction with numerical tolerance ε=10⁻⁶, and LLM-as-Judge for proof problems. Appendix E documents evaluation prompts and methodology for each benchmark."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after Section 8, spanning approximately one full page with multiple substantive paragraphs covering model sizes, benchmark limitations, domain scope, and method constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: (1) model size floor not tested below 30B — 'equation formalization phase requires reliable symbolic abstraction,' (2) AICrypto n=18 and TP_physics n=25 are small, (3) limited model families, (4) F-1 'does not explicitly backtrack' when strategy is suboptimal."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'Our study focuses on mathematical reasoning in physics, finance, and cryptography'; 'We also do not evaluate F-1 on multiple-choice or simple arithmetic benchmarks (e.g., GSM8K)'; 'we do not include smaller models (e.g., 7B or 13B)'; 'Generalizing F-1 beyond equation-centric settings...would likely require different forms of formalization.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (individual model outputs, per-problem results, judge scores) is made available. Only aggregate results in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the four benchmarks with citations, domain descriptions, sizes (Table 3), and evaluation types. Section 4.2 describes inference protocol. The benchmarks are well-established public datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (IMO-Bench, OlympiadBench, FinanceMath, AICrypto)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from input to evaluation is documented: problem text → prompt construction (Appendix A) → model inference at temp=0 → answer extraction (regex or LLM judge, Section 4.2 and Appendix E) → scoring with specified tolerance. Code execution for PoT is described (30s timeout, sandboxed, Appendix E.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: 'SCB 10X, SCBX Group' — a corporate research entity. The affiliation is clearly stated under the author names."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed at all. Since the authors are from SCB 10X (a financial technology company) and one benchmark is FinanceMath, there is a potential interest in demonstrating improved financial mathematics capabilities, but this is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, financial disclosure, or conflict-of-interest declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five models used (GPT-5, Gemini 2.5 Pro, DeepSeek-V3.1, Qwen3-235B, Qwen3-30B). This is critical since the benchmarks are publicly available."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. OlympiadBench problems come from past olympiads, IMO-Bench from IMO problems — both long public and likely in training data of frontier models. No analysis or acknowledgment."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of benchmark contamination. OlympiadBench (2024), IMO-Bench (2024), and FinanceMath (2023) were all published before the likely training cutoffs of the frontier models used. No canary strings, temporal analysis, or contamination assessment."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates prompting methods on automated benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Benchmark evaluation study only."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides detailed token efficiency analysis: average tokens per method/model/benchmark (Tables 11–14), efficiency ratios (Table 9), tokens per correct answer (Table 15), and prompt overhead comparison (Table 10: F-1 adds +68 tokens over Zero-Shot)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated — no total API spend, total tokens consumed across all experiments, GPU hours, or wall-clock time for the full evaluation. Only per-problem averages are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from single deterministic runs (temperature=0, greedy decoding). No sensitivity analysis across different temperatures, sampling strategies, or repeated runs is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly states: 'Since we use greedy decoding, each question is evaluated using a single generated output.' The number of runs (1) is clearly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of how the F-1 prompt design was developed — how many prompt variations were tried, what alternatives were considered, or how the final prompt was selected. The prompt appears hand-designed but no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents a single prompt design (Section 3.3) without explaining how it was selected among alternatives. No validation set is mentioned for prompt development."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 4 methods × 5 models × 4 benchmarks (80 cells in Table 4) plus subcategory breakdowns, without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (Zero-Shot, CoT, PoT) themselves and compare against their own F-1 method. No acknowledgment of self-comparison bias (Lucic et al. 2018) or independent evaluation is discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Tables 9–10 and Appendix B compare performance relative to computational cost across all methods. Efficiency ratio (accuracy/tokens) and prompt overhead are reported, showing F-1 achieves comparable or better accuracy with similar token costs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 discusses why selected benchmarks test equation formalization and why GSM8K/MATH are excluded: 'they test sequential calculation rather than equation identification' and frontier models are at ceiling (96.8%). The paper justifies benchmark selection relative to what F-1 claims to improve."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — all methods are single-call prompting with no tools or orchestration."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. OlympiadBench problems are from past olympiads, IMO-Bench from past IMO competitions — both available online well before frontier model training cutoffs. This is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The paper does not consider whether evaluation setup (e.g., problem formatting, answer type hints in OlympiadBench prompts) leaks information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities with model training data or with each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied — no canary strings, membership inference, temporal splits, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "F-1 outperforms CoT by +5.76% and PoT by +8.42% on average across four benchmarks and five models",
       "evidence": "Table 4 shows overall macro-averaged accuracy: F-1 61.06% vs CoT 55.30% (+5.76%) and PoT 52.64% (+8.42%). F-1 is best in 18 of 20 benchmark-model combinations.",
       "supported": "moderate"
     },
     {
       "claim": "Gains are largest on applied domains: +13.30% on FinanceMath over CoT and +7.24% on AICrypto",
       "evidence": "Table 4: FinanceMath average F-1 56.30% vs CoT 43.00% (+13.30%), AICrypto F-1 87.54% vs CoT 80.30% (+7.24%). Consistent pattern across models.",
       "supported": "moderate"
     },
     {
       "claim": "Within OlympiadBench, F-1 gains are larger on physics than pure math, supporting the hypothesis that equation formalization benefits applied domains most",
       "evidence": "Table 5 (GPT-5 only): physics OE +2.55%, math OE +0.44%; physics TP +4.00% (n=25), math TP +3.77%.",
       "supported": "weak"
     },
     {
       "claim": "Equation formalization is the most critical component, contributing roughly twice the improvement of adaptive selection on applied domains",
       "evidence": "Table 6 ablation on GPT-5: removing equation formulation drops FinanceMath by 8.5pp (64.0→55.5), while removing adaptive selection drops it by 6.0pp (64.0→58.0). Pattern consistent across three benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "F-1 achieves 73% strategy selection accuracy on applied domains and reaches 81–84% of the theoretical upper bound",
       "evidence": "Section 6.2 and Tables 7–8: FinanceMath selection accuracy 73.0%, reaching 80.9% of upper bound. OlympiadBench 69.9% accuracy, 84.1% of upper bound.",
       "supported": "moderate"
     },
     {
       "claim": "F-1 maintains single-call efficiency with only +68 additional input tokens over Zero-Shot",
       "evidence": "Table 10: F-1 averages 465 input tokens vs Zero-Shot 397 (+68). Single API call confirmed. Efficiency ratio in Table 9 shows F-1 achieves 1.51 vs CoT 1.26 and PoT 1.24.",
       "supported": "strong"
     },
     {
       "claim": "The smallest model (Qwen3-30B) benefits comparably to frontier models from F-1, with +5.6% improvement over CoT",
       "evidence": "Table 4: Qwen3-30B overall F-1 63.33% vs CoT 57.72% (+5.61%). Gains are comparable to frontier models (GPT-5 +6.24%, Gemini +2.33%).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny benchmark with outsized claims",
       "detail": "AICrypto has only 18 problems. Results vary enormously (e.g., GPT-5 scores 98.50% vs Qwen3-235B 76.50%). The paper reports +7.24% average gains on AICrypto in the abstract without prominently noting this is based on 18 items. Table 3 footnotes this, and Limitations acknowledges it, but it still appears as a headline claim."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('outperforms by +5.76%') are based on raw accuracy differences across 80 benchmark-model cells with no significance tests, confidence intervals, or multiple comparison corrections. Given single-run deterministic decoding, there is no measure of result stability."
     },
     {
       "flag": "Ablation only on one model",
       "detail": "The ablation study (Table 6) showing equation formalization as the key component is conducted only on GPT-5. It is unclear whether the component contribution pattern holds for other models, especially the open-source ones that show different absolute performance profiles."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Frontier models (GPT-5, Gemini 2.5 Pro) may have been trained on OlympiadBench and IMO-Bench problems, which are publicly available competition math/physics problems. If models memorized solutions, prompting method differences could reflect different ways of triggering memorized answers rather than genuine reasoning improvements."
     },
     {
       "flag": "LLM-as-Judge without human validation",
       "detail": "Proof-based evaluation (IMO-ProofBench, AICrypto, OlympiadBench TP) relies on LLM judges (Gemini-3-Pro, GPT-5.1) with no human validation of judge accuracy. The qualitative examples in Appendix D show cases where judges give 7/10 vs 8/10 — small differences that determine correctness thresholds. Judge reliability is assumed, not demonstrated."
     },
     {
       "flag": "Corporate affiliation without COI disclosure",
       "detail": "Authors are from SCB 10X (a financial technology company under SCBX Group). One of the four benchmarks is FinanceMath, where F-1 shows its largest gains (+13.30%). No conflict-of-interest statement or funding disclosure is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Brian Ichter", "Fei Xia", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "Foundational prompting technique for LLM reasoning; the primary baseline that F-1 improves upon."
     },
     {
       "title": "Program of thoughts prompting: Disentangling computation from reasoning for numerical reasoning tasks",
       "authors": ["Wenhu Chen", "Xueguang Ma", "Xinyi Wang", "William W Cohen"],
       "year": 2023,
       "relevance": "Code-based reasoning alternative to CoT; baseline method showing different intermediate representation for math reasoning."
     },
     {
       "title": "Plan-and-solve prompting: Improving zero-shot chain-of-thought reasoning by large language models",
       "authors": ["Lei Wang", "Wanyu Xu", "Yihuai Lan"],
       "year": 2023,
       "relevance": "Single-call two-phase prompting method closest to F-1's architecture; comparison point for planning vs formalization."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao", "Izhak Shafran", "Thomas L Griffiths", "Yuan Cao", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Multi-call reasoning method exploring multiple paths; represents the compute-heavy alternative to F-1's single-call approach."
     },
     {
       "title": "Graph of thoughts: Solving elaborate problems with large language models",
       "authors": ["Maciej Besta", "Nils Blach", "Ales Kubicek"],
       "year": 2024,
       "relevance": "Graph-based multi-call reasoning framework; another high-compute reasoning approach contrasted with F-1's efficiency."
     },
     {
       "title": "Adaptive-solver framework for dynamic strategy selection in large language model reasoning",
       "authors": ["Jianpeng Zhou", "Wanjun Zhong", "Yanlin Wang", "Jiahai Wang"],
       "year": 2024,
       "arxiv_id": "2310.01446",
       "relevance": "Classifier-based routing across multiple LLM calls for math; closest prior work to F-1's adaptive selection but multi-call."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Multiple reasoning path sampling technique; upper bound analysis methodology referenced by F-1."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "relevance": "Cost-efficient LLM usage framework; F-1 references its token efficiency metrics."
     },
     {
       "title": "OlympiadBench: A challenging benchmark for promoting AGI with olympiad-level bilingual multimodal scientific problems",
       "authors": ["Chaoqun He", "Renjie Luo", "Yuzhuo Bai"],
       "year": 2024,
       "arxiv_id": "2402.14008",
       "relevance": "Primary evaluation benchmark containing both math and physics olympiad problems; enables controlled applied vs pure math comparison."
     },
     {
       "title": "FinanceMath: Knowledge-intensive math reasoning in finance domains",
       "authors": ["Yilun Zhao", "Hongjun Liu", "Yitao Long", "Rui Zhang", "Chen Zhao", "Arman Cohan"],
       "year": 2024,
       "arxiv_id": "2311.09797",
       "relevance": "Applied mathematics benchmark in finance domain; shows largest F-1 improvements and tests equation-based reasoning in real-world domain."
     },
     {
       "title": "AICrypto: A comprehensive benchmark for evaluating cryptography capabilities of large language models",
       "authors": ["Yu Wang", "Yijian Liu", "Liheng Ji"],
       "year": 2025,
       "arxiv_id": "2507.09580",
       "relevance": "Cryptography capabilities benchmark for LLMs; tests formal proof reasoning where F-1 shows strong gains."
     },
     {
       "title": "LLM-SR: Scientific equation discovery via programming with large language models",
       "authors": ["Parshin Shojaee", "Kazem Meidani", "Amir Barati Farimani", "Chandan K Reddy"],
       "year": 2024,
       "relevance": "Scientific equation discovery using LLMs; supports the broader idea that structured equation representations improve LLM reasoning."
     },
     {
       "title": "Large language models for mathematical reasoning: Progresses and challenges",
       "authors": ["Janice Ahn", "Rishu Verma", "Renze Lou", "Di Liu", "Rui Zhang", "Wenpeng Yin"],
       "year": 2024,
       "arxiv_id": "2402.00157",
       "relevance": "Survey of LLM mathematical reasoning techniques; provides context for the landscape of prompting approaches F-1 contributes to."
     }
   ]
 }