ai-research-survey

scan.json (29347B)

---

 {
   "paper": {
     "title": "OR-LLM-Agent: Automating Modeling and Solving of Operations Research Optimization Problems with Reasoning LLM",
     "authors": [
       "Bowen Zhang",
       "Pengcheng Luo",
       "Genke Yang",
       "Boon-Hee Soong",
       "Chau Yuen"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.10009",
     "doi": "10.48550/arXiv.2503.10009"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OR-LLM-Agent, a three-stage framework (Math Agent, Code Agent, Debugging Agent) built on reasoning LLMs, achieves 82.93% accuracy on the authors' BWOR benchmark with DeepSeek-R1, outperforming standalone reasoning LLMs by at least 7%. The authors find that reasoning LLMs sometimes underperform non-reasoning counterparts on existing OR benchmarks (NL4OPT, MAMO, IndustryOR), but consistently outperform them on BWOR. An ablation study shows each stage (modeling, coding, debugging) contributes incrementally to performance, with the full pipeline reducing code error rates by 4.04% compared to standalone LLMs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided in the paper: https://github.com/bwz96sco/or_llm_agent."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "BWOR dataset released on HuggingFace: https://huggingface.co/datasets/SJTU/BWOR. Other benchmarks (NL4OPT, MAMO, IndustryOR) are publicly available from the ORLM repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed dependency listing is provided in the paper. Only mentions Python and Gurobi solver without version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The methodology is described algorithmically (Algorithm 1) but there are no specific commands or setup instructions to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '82.93% accuracy') with no confidence intervals or error bars in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'outperforms... by at least 7% in accuracy' are based solely on comparing raw numbers. No statistical significance tests (t-tests, bootstrap, etc.) are used anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context throughout, e.g., 'improves average accuracy by 4.06%, from 62.20% to 66.26%' (ablation study) and 'reduces the mean code error rate by 4.04%, dropping from 4.56% to 0.52%' (Table 2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "BWOR contains only 82 problems with no justification for this sample size. No power analysis or discussion of whether 82 problems is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run. No standard deviation, variance, or spread measures are reported across any experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines included: SOTA methods (tag-BART, Chain-of-Experts, OptiMUS, ORLM), reasoning LLMs (GPT-o3, GPT-o4-mini, Gemini 2.5 Pro, DeepSeek-R1), non-reasoning LLMs (GPT-4o, Gemini 2.0 Flash, DeepSeek-V3), and open-source models (LLAMA3-8B, DeepSeek-R1-Distill-32B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent models: GPT-o3, GPT-o4-mini, Gemini 2.5 Pro, and DeepSeek-R1, all from 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study on BWOR compares three configurations: Direct Code Generation, Math Agent + Code Agent, and full OR-LLM-Agent (Math + Code + Debugging). Results in Figure 11 show each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (absolute error from ground truth below 0.1) is used as the evaluation metric. No other metrics (e.g., partial credit, model quality, solve time) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs is performed. All evaluation is automated via numerical comparison against ground-truth answers. Human evaluation of mathematical model quality would have strengthened the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks (NL4OPT, MAMO, IndustryOR, BWOR) are used purely for evaluation. No fine-tuning or hyperparameter selection is performed on any of these datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across five datasets (NL4OPT, MAMO-Easy, MAMO-Complex, IndustryOR, BWOR) in Table 1, and separately by code error rate (Table 2) and mathematical model accuracy (Table 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Error analysis in Tables 2 and 3 separately analyzes code error rates and mathematical model accuracy. The paper discusses how the debugging agent reduces code errors and when math model self-repair is triggered."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that reasoning LLMs sometimes underperform non-reasoning counterparts on NL4OPT, MAMO, and IndustryOR (e.g., 'GPT-o4-mini achieves 5.00% lower accuracy than GPT-4o' on IndustryOR). Also reports missing results due to service interruption."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'outperforms advanced methods... by at least 7% in accuracy' on BWOR. Table 1 shows OR-LLM-Agent(DeepSeek-R1) at 82.93% vs GPT-o3 at 75.61%, a 7.32% gap. The claim is supported, though narrowly scoped to BWOR."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that 'task decomposition improves performance' is supported by the ablation study (Figure 11) which uses controlled single-variable manipulation: adding Math Agent improves by 4.06%, adding Debugging Agent improves by another 5.49%."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automating Modeling and Solving of Operations Research Optimization Problems' broadly, but the primary results are on BWOR (82 textbook problems). The paper demotes established benchmarks where results are less favorable, making the generalization claim rest heavily on a small, author-constructed dataset."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The improvement could stem from using 3+ LLM calls per problem instead of 1 (more compute), but this confound is not considered. The paper attributes all gains to task decomposition without considering alternatives."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy (absolute error < 0.1 from ground truth) and claims to measure accuracy. The measurement directly matches the claim with no proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT-o3', 'GPT-o4-mini', 'Gemini 2.5 Pro', 'DeepSeek-R1'. No snapshot dates, API versions, or specific model identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 3, 4, 6, and 7 show the actual prompt text used for the Math Agent, Code Agent, Code Self-repair, and Math Model Self-repair respectively. The code repository also contains the prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API hyperparameters (temperature, top-p, max tokens, etc.) are reported for any of the LLMs used. These settings significantly affect output quality."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-agent scaffolding is described in detail: Algorithm 1 provides pseudocode, Figures 1-2 show workflow diagrams, and the debugging loop (5 attempts, code repair for attempts 1-3, math model repair on attempt 4) is fully specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "BWOR is described as '82 problems collected from standard OR textbooks (Hu 2010, 2012)' that were 'manually translated into English,' but no inclusion/exclusion criteria, selection rationale, or translation verification process is documented. For other benchmarks, they simply state they were 'obtained from the ORLM repository' with no further detail."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The paper moves directly from experimental results to the conclusion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. Issues like small dataset size, scaffold confounding, lack of variance reporting, and benchmark selection bias are not acknowledged."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what types of OR problems are excluded, what model families might not work, or what the approach cannot handle."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "BWOR dataset is released on HuggingFace (https://huggingface.co/datasets/SJTU/BWOR) and code on GitHub, enabling independent verification of results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "BWOR is described as '82 problems collected from standard OR textbooks (Hu 2010, 2012)' but no selection criteria, chapter coverage, or systematic collection methodology is provided. It is unclear how these 82 problems were chosen from the textbooks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is OR textbooks and standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from textbook to final benchmark is undocumented. How problems were selected, how translation was validated, and how ground-truth answers were verified by 'domain experts' are not described with enough detail to reproduce."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Shanghai Jiao Tong University (Ningbo AI Institute and Dept. of Automation) and Nanyang Technological University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence since no funding information is disclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (GPT-o3, GPT-o4-mini, Gemini 2.5 Pro, DeepSeek-R1, etc.). This is important since NL4OPT and MAMO are publicly available and may be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. NL4OPT (2022), MAMO, and IndustryOR are public benchmarks that could be in the training data of the evaluated models. This is especially relevant given the unexpected finding that reasoning LLMs underperform non-reasoning ones on these benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NL4OPT was published in 2022 and MAMO in 2024; both are publicly available and could be in training data. The paper does not discuss contamination risk despite this being a plausible alternative explanation for the puzzling reasoning vs. non-reasoning performance inversion on these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The framework makes 3+ LLM calls per problem (Math Agent + Code Agent + up to 5 debugging attempts), but no API costs, token counts, or latency measurements are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (API spend, total tokens, wall-clock time) is stated despite using expensive reasoning LLMs (GPT-o3, DeepSeek-R1) across multiple datasets."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run. No seed sensitivity analysis or multi-seed results are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from a single run or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Design choices like the number of debugging attempts (5), when to trigger math model repair (attempt 4), and prompt wording appear tuned but no search budget or justification is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The debugging thresholds (code repair for attempts 1-3, math model repair on attempt 4, give up after 5) are not justified. No explanation for why these specific values were chosen."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many pairwise comparisons are made across models and datasets (Table 1 has 15+ model-dataset combinations) with no correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own framework against baselines without acknowledging author-evaluation bias. Open-source baseline results (LLAMA3-8B, DeepSeek-R1-Distill-32B all scoring 0.00%) were reproduced using ORLM prompts, potentially disadvantaging those baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "OR-LLM-Agent uses 3+ reasoning LLM calls per problem while baselines use a single call. This substantial compute difference is never discussed or controlled for. The 7%+ improvement could partly or fully reflect more compute, not better architecture."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper analyzes whether existing OR benchmarks (NL4OPT, MAMO, IndustryOR) properly differentiate reasoning from non-reasoning models, comparing against known math/code benchmarks (AIME 2024, LiveCodeBench). They argue BWOR has better construct validity because reasoning models consistently outperform non-reasoning ones on it."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OR-LLM-Agent uses a multi-agent scaffold with debugging loops while baselines are single-call LLMs. The improvement is attributed to 'task decomposition' and 'reasoning LLMs' without controlling for the scaffold effect. The ablation adds components (and LLM calls) simultaneously."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems existed before the models' training cutoffs. NL4OPT (2022) and MAMO (2024) solutions could be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real-world OR problem solving."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities or overlap across datasets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied. No canary strings, membership inference, decontamination, or temporal analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "OR-LLM-Agent with DeepSeek-R1 outperforms all compared methods by at least 7% accuracy on BWOR.",
       "evidence": "Table 1 shows OR-LLM-Agent(DeepSeek-R1) achieves 82.93% on BWOR, compared to 75.61% for GPT-o3 (best standalone reasoning LLM) — a 7.32% gap.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning LLMs sometimes underperform non-reasoning counterparts on existing OR benchmarks (NL4OPT, MAMO, IndustryOR).",
       "evidence": "Figure 9 and surrounding analysis show GPT-o4-mini achieves 5% lower than GPT-4o on IndustryOR, Gemini 2.5 Pro underperforms Gemini 2.0 Flash by 7.11% on MAMO-Complex, and on MAMO-Easy all reasoning models score lower (1.84%-17.79% decline).",
       "supported": "moderate"
     },
     {
       "claim": "BWOR more effectively differentiates reasoning vs. non-reasoning LLMs than existing OR benchmarks.",
       "evidence": "On BWOR, reasoning LLMs consistently outperform non-reasoning counterparts (10.98%-35.37% improvement), aligning with their known math/code superiority, while other benchmarks show inconsistent patterns.",
       "supported": "moderate"
     },
     {
       "claim": "Task decomposition (Math + Code + Debug) improves performance over direct code generation by ~9.5%.",
       "evidence": "Ablation study (Figure 11) shows Direct Code Generation at 62.20% average, Math+Code at 66.26% (+4.06%), and full framework at 71.75% (+5.49% more) on BWOR.",
       "supported": "moderate"
     },
     {
       "claim": "OR-LLM-Agent reduces code error rate by 4.04% compared to standalone LLMs.",
       "evidence": "Table 2 shows mean code error rate of 0.52% for OR-LLM-Agent vs. 4.56% for reasoning and non-reasoning LLMs across all datasets.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Author-constructed primary benchmark",
       "detail": "The primary benchmark BWOR (82 problems) is constructed by the authors. Established benchmarks (NL4OPT, MAMO, IndustryOR) where the framework shows less clear advantage are demoted to 'supplementary analysis' and 'excluded from core comparisons.' This creates a risk of benchmark selection bias."
     },
     {
       "flag": "No error bars on tiny dataset",
       "detail": "All results on BWOR (82 problems) are single-run point estimates with no variance, confidence intervals, or significance tests. On 82 binary-ish outcomes, stochastic variation in LLM outputs could easily account for several percentage points of difference."
     },
     {
       "flag": "Scaffold confound unaddressed",
       "detail": "OR-LLM-Agent makes 3+ LLM calls per problem (modeling + coding + up to 5 debugging attempts) while all baselines use a single LLM call. The 7% improvement could partly or wholly reflect more compute rather than better architecture. No matched-compute comparison is provided."
     },
     {
       "flag": "Contamination as alternative explanation ignored",
       "detail": "The finding that reasoning LLMs underperform on NL4OPT/MAMO but not BWOR could be explained by differential contamination (public benchmarks in training data help non-reasoning models' memorization while BWOR is novel). This alternative explanation is not discussed."
     },
     {
       "flag": "Zero-scoring baselines suggest prompt/setup issues",
       "detail": "LLAMA3-8B-Base, LLAMA3-8B-Instruct, and DeepSeek-R1-Distill-32B all score exactly 0.00% across every dataset. This suggests the ORLM prompt or execution setup may be incompatible with these models rather than reflecting their actual capability."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or scope boundary discussion despite multiple methodological concerns (tiny primary benchmark, missing variance, compute confound, potential contamination)."
     }
   ],
   "cited_papers": [
     {
       "title": "Autogen: Enabling next-gen llm applications via multi-agent conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent LLM framework for collaborative code generation and debugging, directly relevant to agentic AI workflows."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex and HumanEval benchmark — foundational work on LLM code generation capability evaluation."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Competitive programming code generation with large-scale sampling, relevant to LLM coding capability evaluation."
     },
     {
       "title": "Opencodeinterpreter: Integrating code generation with execution and refinement",
       "authors": ["Tianyu Zheng", "Ge Zhang", "Tianhao Shen"],
       "year": 2024,
       "arxiv_id": "2402.14658",
       "relevance": "Integrated code generation, execution, and optimization pipeline, relevant to automated programming agent design."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Core reasoning LLM used in the framework, relevant to understanding reasoning vs non-reasoning model capabilities."
     },
     {
       "title": "Gpt-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, a key baseline model in LLM capability evaluation."
     },
     {
       "title": "OptiMUS: Optimization Modeling Using mip Solvers and large language models",
       "authors": ["Ali AhmadiTeshnizi", "Wenzhi Gao", "Madeleine Udell"],
       "year": 2024,
       "relevance": "LLM-based modular agent for solving optimization problems from natural language, direct competitor and relevant baseline."
     },
     {
       "title": "Orlm: A customizable framework in training large models for automated optimization modeling",
       "authors": ["Chenghao Huang", "Zhitao Tang", "Shuang Hu"],
       "year": 2025,
       "relevance": "Fine-tuning framework for LLMs on OR tasks with synthetic data pipeline, key baseline and benchmark source."
     },
     {
       "title": "Chain-of-experts: When llms meet complex operations research problems",
       "authors": ["Ziyang Xiao", "Dongxiang Zhang", "Yangjun Wu"],
       "year": 2023,
       "relevance": "Multi-agent cooperative framework for linear programming, directly relevant to LLM-based OR problem solving."
     },
     {
       "title": "Livecodebench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark used to validate model coding capability claims."
     },
     {
       "title": "Deepseek-v3 technical report",
       "authors": ["Aixin Liu", "Bei Feng", "Bing Xue"],
       "year": 2024,
       "arxiv_id": "2412.19437",
       "relevance": "Technical report for DeepSeek-V3, a non-reasoning baseline model in the evaluation."
     },
     {
       "title": "Phi-4 technical report",
       "authors": ["Marah Abdin", "Jyoti Aneja", "Harkirat Behl"],
       "year": 2024,
       "arxiv_id": "2412.08905",
       "relevance": "Small language model technical report, relevant to understanding LLM capabilities in mathematical reasoning."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "OR practitioners could use the released framework to automate optimization problem solving, though it requires Gurobi licenses and expensive API access to reasoning LLMs."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that reasoning LLMs underperform non-reasoning ones on some OR benchmarks is mildly surprising, though the explanation is underdeveloped."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or conflicts with other work."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub and dataset on HuggingFace, though requires Gurobi solver setup and API keys to run."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses well-known models (GPT-o3, DeepSeek-R1) but authors are from SJTU/NTU, not top-of-mind AI labs."
     }
   }
 }