ai-research-survey

scan-v5.json (26009B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Formula-One Prompting: Adaptive Reasoning Through Equations For Applied Mathematics",
     "authors": [
       "Natapong Nitarach",
       "Pittawat Taveekitworachai",
       "Kunat Pipatanakul"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.19302",
     "doi": "10.48550/arXiv.2601.19302"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (+5.76% over CoT, +8.42% over PoT, +13.30% on FinanceMath) are directly supported by Table 4 macro-averaged results across five models.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claim that equation formalization drives gains is supported by ablation study (Table 6) removing components one at a time; ablation is the appropriate design for prompting research.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Paper explicitly excludes arithmetic benchmarks (GSM8K), acknowledges results apply only to equation-centric applied mathematics, and limitations section explicitly states scope boundaries.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The verification step ('Verify your solution') included in F-1 but absent from all baselines is an uncontrolled confound that is never isolated or discussed as an alternative explanation for observed gains.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims are about benchmark accuracy on mathematical problems, which is exactly what is measured; the paper does not conflate benchmark performance with broader real-world mathematical ability.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated Limitations section appears before the Ethics Statement, covering model scale, domain scope, small benchmark sizes (AICrypto n=18), and the deliberate single-call design constraint.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: AICrypto n=18 and OlympiadBench TP_physics n=25 are flagged as too small; testing no models below 30B is explicitly noted as a limitation.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Paper explicitly states F-1 is not evaluated on GSM8K or simple arithmetic, not tested on models below 30B, and generalization beyond equation-centric domains is explicitly out of scope.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper; there is no acknowledgments section or funding statement despite authors being employed at a commercial institution.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with SCB 10X, SCBX Group (a financial technology arm of a major bank) are listed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are employed by SCB 10X (banking/fintech), and the largest claimed gain (+13.30%) is on FinanceMath; this potential interest alignment is not acknowledged or discussed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent declarations, or financial interests declaration appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are operationally defined: 'governing equations,' 'equation formalization,' 'adaptive solving,' CoT, PoT, and Direct strategies are all explained with examples in Section 3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are bulleted in the introduction: the F-1 method itself, ablation showing formalization is the key component, and strategy selection accuracy analysis.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Related work section systematically positions F-1 against single-call and multi-call methods with Table 1 showing design-level differences across seven prior approaches.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository, GitHub link, or code release is mentioned anywhere in the paper.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All four evaluation benchmarks (IMO-Bench, OlympiadBench, FinanceMath, AICrypto) are standard public benchmarks with citations to their original papers.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or environment specification is provided; the sandboxed code execution environment is only described as '30s timeout, standard libraries' without specifics.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Complete prompts are in Appendix A and temperature=0 is stated, but no code, API wrappers, or step-by-step pipeline instructions are provided to actually run experiments.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported for any result; greedy decoding eliminates run-to-run variance but LLM judge variance and benchmark sampling variance are not quantified.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are conducted for any comparative claim; performance differences are reported as raw percentages only.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Percentage improvements over baselines are reported consistently (+5.76% over CoT, +8.42% over PoT, +13.30% on FinanceMath) with per-benchmark and per-model breakdowns.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No power analysis or prospective sample size justification is provided; AICrypto's n=18 is flagged after the fact as a limitation rather than addressed in study design.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance or standard deviation is reported across any dimension; single-run greedy decoding is used, but judge variability and benchmark sampling variance remain unquantified.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Three baselines are evaluated across all benchmarks and models: Zero-Shot, CoT, and PoT, covering the relevant single-call prompting space.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "CoT and PoT are the established standards for single-call mathematical prompting; multi-call methods (ToT, GoT) are excluded with explicit justification of different compute assumptions.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.1 ablates three F-1 components (adaptive selection, equation formulation, givens/targets identification) across three benchmarks, though only using GPT-5.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both accuracy (primary metric) and token efficiency ratio (accuracy/tokens) are reported across all benchmarks and models in Tables 9-15.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Human evaluation of system outputs is not applicable for automated mathematical benchmark evaluation; LLM-as-Judge is used for proof-based problems.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All four benchmarks are fixed test sets used without any training; F-1 is a prompting technique with no learned components.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by OlympiadBench subtask (OE/TP, math/physics), FinanceMath category, and AICrypto category in Appendix C tables.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Table 7 quantifies strategy failures (Adapt× category where baselines succeed but F-1 fails), and Section 6.3 provides qualitative failure analysis for all baseline methods.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "F-1 underperforms in 2 of 20 benchmark-model combinations (explicitly noted); IMO-Bench shows near-zero average gain (+0.78%) with specific models performing worse than CoT.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "GPT-5 and Gemini 2.5 Pro are referenced without snapshot dates or API version identifiers, making exact reproduction impossible for two of five evaluated models.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Complete verbatim prompts for all methods across all four benchmarks are provided in Appendix A, including system prompts, user prompts, and benchmark-specific adaptations.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only temperature=0 is explicitly stated; all other hyperparameters (top-p, max tokens, presence penalty, etc.) are described only as 'default values' without specifying what those defaults are.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "This is a single-call prompting study with no agentic scaffolding or multi-step orchestration; the sandboxed code execution for PoT is briefly described.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation protocol is documented in Appendix E: regex extraction with tolerance ε=10⁻⁶ for numerical answers, LLM-as-Judge with specific judge models and full prompts for proof-based problems.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw model outputs, intermediate equation generations, and LLM judge responses are not released; only summary accuracy statistics are provided.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "All four benchmarks are publicly available with citations to original papers; benchmark sizes, domains, and evaluation formats are documented in Table 3.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "Standard publicly available benchmarks are used; no human participant recruitment is involved.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full evaluation pipeline is described: prompt construction → model inference (temp=0, greedy) → answer extraction (regex or LLM judge) → accuracy computation, with judge prompts in Appendix E.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for GPT-5, Gemini 2.5 Pro, DeepSeek-V3.1, and Qwen3 models are not stated anywhere in the paper.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of potential training data overlap with IMO-Bench (2024) or OlympiadBench (2024), which predate the paper by 1-2 years and may be in training corpora.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "IMO-Bench and OlympiadBench were published in 2024 and are likely in training data for frontier models; contamination is entirely unaddressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Token efficiency ratio (accuracy/tokens × 100) and tokens per correct answer are reported across all benchmarks and models in Tables 9-15.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Total API cost or compute budget for running all experiments (5 models × 4 benchmarks × ~2,116 problems × 4 methods) is not stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "F-1 outperforms CoT by +5.76% and PoT by +8.42% on average across four benchmarks and five models",
       "evidence": "Table 4 macro-averaged overall row: F-1 61.06% vs CoT 55.30% vs PoT 52.64%; 18 of 20 benchmark-model combinations favor F-1",
       "supported": "strong"
     },
     {
       "claim": "Gains are largest in applied domains: +13.30% over CoT on FinanceMath",
       "evidence": "Table 4 FinanceMath: F-1 56.30% vs CoT 43.00%; confirmed by OlympiadBench physics (+2.55%) outpacing math (+0.44%) in Table 5",
       "supported": "strong"
     },
     {
       "claim": "Equation formalization is the primary driver, contributing roughly twice the improvement of adaptive selection alone",
       "evidence": "Table 6 ablation (GPT-5 only): removing equation formulation drops FinanceMath 8.5pp vs removing adaptive selection drops 6.0pp",
       "supported": "moderate"
     },
     {
       "claim": "F-1 achieves 73% strategy selection accuracy on applied domains",
       "evidence": "Section 6.2: defined as (Adapt✓ + F-1 Only)/(Adapt✓ + Adapt× + F-1 Only); FinanceMath=73.0%, OlympiadBench=69.9%",
       "supported": "moderate"
     },
     {
       "claim": "F-1 reaches 81-84% of the theoretical upper bound on applied domains",
       "evidence": "Table 8: FinanceMath 80.9%, OlympiadBench 84.1%, AICrypto 82.2%; upper bound defined as 100% minus all-failed rate",
       "supported": "moderate"
     },
     {
       "claim": "F-1 adds only +68 prompt tokens overhead versus Zero-Shot while maintaining single-call efficiency",
       "evidence": "Table 10: Zero-Shot 397 avg input tokens vs F-1 465 avg input tokens across all benchmarks",
       "supported": "strong"
     },
     {
       "claim": "Even the smallest tested model (Qwen3-30B) benefits from F-1 comparably to frontier models (+5.6% over CoT)",
       "evidence": "Table 4 Qwen3-30B: F-1 63.33% vs CoT 57.72%; macro average across benchmarks including small n=18 AICrypto",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "F-1 prompting, which instructs LLMs to formalize governing equations before adaptively selecting a solving strategy (CoT, PoT, or direct calculation), outperforms CoT by +5.76% and PoT by +8.42% on average across four applied mathematics benchmarks and five models ranging from 30B to frontier scale. The gains are strongly domain-specific: largest in finance (+13.30% over CoT) and cryptography (+7.24%), near-zero in competition mathematics (IMO-Bench). Ablation analysis identifies equation formalization as the primary mechanism, contributing roughly twice the performance gain of adaptive strategy selection alone. The method is computationally efficient, adding only 68 prompt tokens overhead within a single LLM call.",
   "red_flags": [
     {
       "flag": "Uncontrolled verification confound",
       "detail": "F-1 includes an explicit verification step ('Verify your solution') absent from all three baselines (Zero-Shot, CoT, PoT); this is never isolated in ablation, making it impossible to attribute gains solely to equation formalization versus self-checking."
     },
     {
       "flag": "AICrypto micro-benchmark (n=18)",
       "detail": "The cryptography domain claim (+7.24% over CoT) rests on 18 problems; this benchmark receives equal macro-average weight as OlympiadBench (n=1,438), inflating overall averages with high-variance estimates."
     },
     {
       "flag": "Model versions underspecified",
       "detail": "GPT-5 and Gemini 2.5 Pro are referenced without snapshot dates or API version identifiers, making exact reproduction impossible and preventing contamination assessment for two of five models."
     },
     {
       "flag": "Ablation on single model only",
       "detail": "Component ablation (Table 6) uses only GPT-5 (highest-performing model); the claimed hierarchy of formalization > adaptive selection may not hold for weaker models and cannot be verified from reported data."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "IMO-Bench and OlympiadBench were published in 2024 and are likely in training data for frontier models; no training cutoffs are stated and overlap is never discussed."
     },
     {
       "flag": "Financial institution evaluating on finance benchmark",
       "detail": "Authors are from SCB 10X (banking/fintech arm of a major Thai bank), and the largest claimed gain (+13.30%) is on FinanceMath; no conflict of interest statement addresses this potential alignment."
     },
     {
       "flag": "LLM-as-Judge variance unquantified",
       "detail": "Proof benchmarks (OlympiadBench TP, IMO-ProofBench, AICrypto) are evaluated via LLM-as-Judge with no inter-rater reliability measurement, agreement rate, or uncertainty quantification on judge scores."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "relevance": "Primary baseline and motivating prior work; F-1 is directly benchmarked against CoT across all experiments"
     },
     {
       "title": "Program of Thoughts Prompting: Disentangling Computation from Reasoning for Numerical Reasoning Tasks",
       "relevance": "Primary baseline; F-1 incorporates PoT as one of its adaptive solving strategies and compares against it throughout"
     },
     {
       "title": "Plan-and-Solve Prompting: Improving Zero-Shot Chain-of-Thought Reasoning by Large Language Models",
       "relevance": "Most closely related single-call two-phase approach; explicitly differentiated from F-1 in introduction and Table 1"
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "relevance": "Multi-call upper bound excluded from main comparison with justification; represents the compute-intensive alternative to single-call methods"
     },
     {
       "title": "OlympiadBench: A Challenging Benchmark for Promoting AGI with Olympiad-Level Bilingual Multimodal Scientific Problems",
       "relevance": "Primary evaluation benchmark (n=1,438) enabling controlled math vs. physics domain comparison central to paper's hypothesis"
     },
     {
       "title": "FinanceMath: Knowledge-Intensive Math Reasoning in Finance Domains",
       "relevance": "Key evaluation benchmark (n=200) for applied finance domain where F-1 shows its largest gains"
     },
     {
       "title": "AICrypto: A Comprehensive Benchmark for Evaluating Cryptography Capabilities of Large Language Models",
       "relevance": "Evaluation benchmark for cryptographic reasoning; provides the equation-formalization-heavy domain beyond physics and finance"
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "relevance": "Methodological basis for the LLM-as-Judge evaluation paradigm used in proof-based benchmark assessment"
     },
     {
       "title": "Adaptive-Solver Framework for Dynamic Strategy Selection in Large Language Model Reasoning",
       "relevance": "Multi-call adaptive baseline; motivates F-1's single-call alternative to classifier-based routing"
     },
     {
       "title": "LLM-SR: Scientific Equation Discovery via Programming with Large Language Models",
       "relevance": "Prior work on structured equation representations supporting F-1's theoretical motivation that formalized representations improve reasoning"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners with access to any frontier LLM API can immediately apply the verbatim prompts from Appendix A, though gains are limited to equation-centric applied math domains."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Writing governing equations before solving is intuitive pedagogy; the finding that this helps LLMs is confirmatory rather than surprising or counter to expectations."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Paper improves mathematical reasoning with no AI safety, risk, or misuse implications beyond a brief ethics statement about educational cheating."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward prompting technique paper with cooperative framing relative to prior work."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Anyone with API access can immediately try the verbatim prompts from Appendix A on their own math problems; no code or setup required."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "SCB 10X is not a well-known AI research lab; no famous author affiliations or high-profile institutional backing."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }