ai-research-survey

scan.json (25239B)

---

 {
   "paper": {
     "title": "CHORUS: Zero-shot Hierarchical Retrieval and Orchestration for Generating Linear Programming Code",
     "authors": ["Tasnim Ahmed", "Salimur Choudhury"],
     "year": 2025,
     "venue": "LION 19 (19th Learning and Intelligent Optimization Conference)",
     "arxiv_id": "2505.01485"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided in the paper. The paper describes the framework but does not release the source code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The authors curated the NL4Opt-Code dataset (extending NL4Opt with Gurobi code annotations) but do not provide a download link or release it. The original NL4Opt dataset is publicly available, but the code annotations they created are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (Intel Xeon Platinum 8358, NVIDIA H100 NVL GPU, 251 GB RAM) in Section 3.3, but provides no software environment details such as library versions, requirements.txt, or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a conceptual level but there are no scripts, commands, or README-style instructions to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (e.g., accuracy 0.6125) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CHORUS 'improves performance by a significant margin' and 'outperforms' baselines, but no statistical significance tests (p-values, t-tests, etc.) are provided. Comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'accuracy of Llama3.3 (70B) increases from 0.2289 to 0.5675' (Section 3.4) and '386.95% accuracy improvement' (Section 3.5). Absolute before/after numbers are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the exact number of test instances in the NL4Opt-Code benchmark, nor justify the sample size. No power analysis or justification for the dataset size is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or multi-run results are reported. All results appear to be single-run numbers with no indication of result stability across multiple executions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes baseline comparisons: the same LLMs without CHORUS, plus GPT-3.5 and GPT-4. Table 2 includes Traditional RAG as another baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-3.5, GPT-4, Llama3.3, Phi4, Deepseek-r1, and Qwen2.5-coder, which are all contemporary models. The paper also discusses OptiMUS, OptLLM, and E-OPT as related work, though direct numerical comparison with those frameworks is not provided."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents a comprehensive ablation study with 5 configurations: Baseline, Baseline + Expert Prompt, Traditional RAG, CHORUS without reasoning, and full CHORUS. This isolates the contributions of expert prompting, retrieval strategy, and structured reasoning."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports four metrics: Accuracy, Syntactic Validity, Semantic Similarity, and Edit Distance, each defined with formulas in Section 3.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code is included. All evaluation is automated (accuracy based on matching objective function values, syntactic parsing, embedding similarity, edit distance). Given that the paper claims code quality improvements, human review of code quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe a train/dev/test split. The NL4Opt dataset has development and test sets, but the paper does not clearly state which split was used for evaluation. Section 3.2 mentions the NL4Opt competition's structure but does not specify a held-out test protocol."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The NL4Opt dataset spans six domains (sales, advertising, investment, production, transportation, sciences) but no per-domain breakdown of results is provided. Only aggregate accuracy across all problems is reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes: smaller models (Llama3.1 8B) suffer accuracy drops with expert prompting due to limited context windows (Section 3.5), and Traditional RAG causes 'fragmented or incomplete contents' that confuse models. The conclusion also acknowledges 'Code generation remains highly sensitive to prompt engineering.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that expert prompting reduces Llama3.1 (8B) accuracy by 61.3% (from 7.96% to 3.08%), and that Traditional RAG consistently hurts performance compared to CHORUS. Edit distance is acknowledged as 'not a particularly reliable metric' that shows no improvement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CHORUS 'improves the performance of open-source LLMs' by a 'significant margin' and allows them to 'outperform or match GPT3.5 and GPT4.' Tables 1 and 2 support these claims: Phi4 with CHORUS achieves 0.6125 accuracy vs GPT-4's 0.6367, and Llama3.3 with CHORUS achieves 0.5675 vs GPT-3.5's 0.5260."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about component contributions (e.g., expert prompting, hierarchical chunking, structured reasoning). The ablation study in Table 2 systematically removes components to measure their individual impact, which is an adequate controlled single-variable manipulation design for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Generating Linear Programming Code' broadly, but the experiments only test on one dataset (NL4Opt-Code) with one solver (Gurobi) in one language (Python). The abstract claims the framework is 'adaptable to any mathematical problem, given an available solver' but this is not tested. The conclusion acknowledges 'alignment of LLMs with other optimization topics...is left for future research' but the title and abstract do not bound the claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, it does not consider whether the gains come simply from providing more context tokens rather than from the specific hierarchical structure, or whether prompt engineering alone (without RAG) could match performance for larger models."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are specified by family and size (e.g., 'Llama3.1 (8B)', 'Phi4 (14B)', 'GPT3.5', 'GPT4') but no specific version strings, snapshot dates, or API versions are provided. For GPT-3.5 and GPT-4, no model ID (e.g., 'gpt-4-0613') is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language ('The system prompt directs the language model to assume the role of an expert...whereas the user prompt outlines strict requirements such as the function name, error handling strategy...' Section 2.4) but the actual prompt text is not provided. The output schema class definition (GurobiSolution) is shown but not the full system/user prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'configuring the sampling temperature to minimize stochasticity' (Section 2.4) but does not report the actual temperature value or any other hyperparameters (top-p, max tokens, top-k retrieval count, embedding model details). The max chunk size of 400 tokens is mentioned but core generation hyperparameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The CHORUS pipeline is described in detail in Section 2: hierarchical tree indexing (Section 2.1), two-stage retrieval with keyword extraction (Section 2.2), cross-encoder reranking selecting top-3 conceptual docs and top-2 code examples (Section 2.3), and structured output parsing (Section 2.4). Figure 1 provides a visual overview."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes how Gurobi documentation is processed into hierarchical tree chunks and metadata-augmented code examples. Section 3.2 describes the dataset annotation process in two stages (LP formulation derivation, then Gurobi code generation) with expert verification."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The conclusion (Section 4) contains a substantial discussion of limitations: 'code generation remains highly sensitive to prompt engineering, and smaller models struggle to fully incorporate all contextual elements within limited context windows. Additionally, the alignment of LLMs with other optimization topics (e.g., integer linear, mixed, or non-linear problems) is left for future research.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations mentioned are specific to some extent (smaller models, prompt sensitivity, scope limited to LP) but there is no dedicated threats-to-validity section and the discussion does not address key threats like single-dataset evaluation, single-run results, potential overfitting of prompts to the specific benchmark, or annotator bias in ground truth creation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the conclusion mentions that other optimization types are 'left for future research,' the paper does not explicitly state what the results do NOT show. It does not bound claims to the specific dataset, solver, or language tested, and the abstract's claim of adaptability to 'any mathematical problem' goes well beyond what was demonstrated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the NL4Opt-Code dataset (their curated extension) nor the raw experimental outputs are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the dataset annotation process: a domain expert (graduate student with optimization coursework) derived LP formulations from NL4Opt problem descriptions, verified by another expert, then generated corresponding Gurobi code."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for the study. The annotators were domain experts involved in dataset creation, not study participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from NL4Opt dataset to NL4Opt-Code is documented in Section 3.2: original problem descriptions → LP formulation derivation by annotator → verification by second expert → Gurobi code generation. The documentation processing pipeline is described in Section 2.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: both authors are from the School of Computing, Queen's University, Ontario, Canada. No evaluated product is affiliated with their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure statement means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs (Llama3.1, Llama3.3, Phi4, Deepseek-r1, Qwen2.5-coder, GPT-3.5, GPT-4) on the NL4Opt-Code benchmark but does not state any training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The NL4Opt dataset was published in 2022 (NeurIPS competition). The models used were trained after 2022 and could have seen this data. No discussion of potential train/test overlap is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The NL4Opt dataset was publicly available before the training cutoff of all tested models. While the authors created new code annotations (NL4Opt-Code), the problem descriptions that the models must understand are from the original NL4Opt dataset, which could be in training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CHORUS enables open-source LLMs to match GPT-4 'while requiring far fewer computational resources' (abstract) but does not report actual inference costs, API costs, tokens consumed, or wall-clock time per problem."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (H100 NVL GPU, Section 3.3) but total compute budget (GPU hours, total inference time, or total cost) is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "CHORUS improves accuracy of open-source LLMs on LP code generation by a significant margin compared to baseline and conventional RAG.",
       "evidence": "Table 1 shows accuracy improvements: Llama3.3 from 0.2289 to 0.5675, Phi4 from 0.1938 to 0.6125, Deepseek-r1 from 0.1073 to 0.5848, Qwen2.5-coder from 0.4644 to 0.5986 (Section 3.4).",
       "supported": "moderate"
     },
     {
       "claim": "CHORUS allows open-source LLMs to outperform or match GPT-3.5 and GPT-4 performance.",
       "evidence": "Table 1: Phi4+CHORUS achieves 0.6125 accuracy vs GPT-4's 0.6367, and Llama3.3+CHORUS achieves 0.5675 vs GPT-3.5's 0.5260 (Section 3.4). However, no model with CHORUS actually exceeds GPT-4's accuracy.",
       "supported": "moderate"
     },
     {
       "claim": "Traditional RAG underperforms CHORUS by 46.14-89.33%.",
       "evidence": "Table 2 shows Traditional RAG accuracy vs CHORUS accuracy across all 5 LLMs. The percentage reduction figures are stated in Section 3.5.",
       "supported": "moderate"
     },
     {
       "claim": "Expert prompting causes 386.95% accuracy improvement for Deepseek-r1 but 61.3% reduction for Llama3.1.",
       "evidence": "Table 2: Deepseek-r1 baseline 0.1073 vs baseline+expert prompt 0.5225; Llama3.1 baseline 0.0796 vs 0.0308 (Section 3.5).",
       "supported": "moderate"
     },
     {
       "claim": "Structured reasoning (reasoning_steps field) provides consistent accuracy gains across LLMs.",
       "evidence": "Table 2 shows CHORUS vs CHORUS (w/o reasoning): improvements of 1.71% to 92.29% across models. The largest gain is for Llama3.1 (8B) from 0.0104 to 0.1349 (Section 3.5).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CHORUS, a RAG framework using hierarchical documentation chunking, metadata-augmented code retrieval, and cross-encoder reranking, substantially improves LP code generation accuracy for open-source LLMs on the NL4Opt-Code benchmark. Phi4 (14B) with CHORUS achieves 0.6125 accuracy, approaching GPT-4's 0.6367, while baseline Phi4 achieves only 0.1938. Ablation studies show that traditional fixed-length RAG actually hurts performance compared to no RAG, while hierarchical chunking, expert prompting, and structured reasoning each contribute positively. Smaller models (8B) struggle to benefit from the framework due to limited context window capacity.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results appear to be single-run numbers with no standard deviations, confidence intervals, or multi-run averaging. Given the acknowledged variability in LLM outputs ('model responses demonstrate significant variability across queries'), single-run results are insufficient to establish reliable performance differences."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'significant' improvements are based on comparing point estimates without any statistical tests. The word 'significant' is used in a colloquial sense rather than a statistical one."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "NL4Opt was published in 2022 and is publicly available. All tested models were trained after 2022 and may have memorized the problem descriptions. Baseline performance could be inflated by data contamination, and CHORUS's relative improvement may differ on uncontaminated data."
     },
     {
       "flag": "Prompts and hyperparameters not disclosed",
       "detail": "The expert prompts described as critical to performance are not provided in full text. Temperature and other generation hyperparameters are not specified. This prevents reproduction and makes it impossible to verify whether the prompts were overfit to the benchmark."
     },
     {
       "flag": "No per-domain breakdown",
       "detail": "The NL4Opt dataset covers 6 domains but results are only reported as aggregates. Performance could vary substantially across domains, and aggregate accuracy could mask poor performance on certain problem types."
     },
     {
       "flag": "Cost claims without evidence",
       "detail": "The abstract claims open-source LLMs with CHORUS require 'far fewer computational resources' than GPT-4, but no actual cost or latency measurements are reported to support this claim."
     }
   ],
   "cited_papers": [
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Abhimanyu Dubey et al."],
       "year": 2024,
       "relevance": "Major open-source LLM family used as baseline and with CHORUS framework, relevant to LLM capability evaluation."
     },
     {
       "title": "Phi-4 Technical Report",
       "authors": ["Marah Abdin et al."],
       "year": 2024,
       "relevance": "Open-source LLM evaluated in the study, relevant to understanding smaller model code generation capabilities."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI et al."],
       "year": 2025,
       "relevance": "Reasoning-focused LLM evaluated for code generation, relevant to LLM reasoning and code generation capabilities."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["Binyuan Hui et al."],
       "year": 2024,
       "relevance": "Code-specialized LLM evaluated in the study, relevant to code generation with specialized models."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI et al."],
       "year": 2024,
       "relevance": "Primary closed-source baseline for code generation performance comparison."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis et al."],
       "year": 2020,
       "relevance": "Foundational RAG paper that CHORUS builds upon, relevant to understanding retrieval-augmented LLM approaches."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei et al."],
       "year": 2022,
       "relevance": "Chain-of-thought prompting technique that CHORUS's structured reasoning extends, relevant to LLM reasoning methodology."
     },
     {
       "title": "OptiMUS: Optimization Modeling Using MIP Solvers and Large Language Models",
       "authors": ["Ali AhmadiTeshnizi", "Wenzhi Gao", "Madeleine Udell"],
       "year": 2023,
       "arxiv_id": "2310.06116",
       "relevance": "Multi-agent LLM framework for optimization problems, directly comparable approach to CHORUS."
     },
     {
       "title": "Let Me Speak Freely? A Study on the Impact of Format Restrictions on Large Language Model Performance",
       "authors": ["Zhi Rui Tam et al."],
       "year": 2024,
       "relevance": "Studies how structured output constraints affect LLM reasoning, directly motivates CHORUS's reasoning_steps design."
     },
     {
       "title": "Benchmarking LLMs for Optimization Modeling and Enhancing Reasoning via Reverse Socratic Synthesis",
       "authors": ["Zhicheng Yang et al."],
       "year": 2024,
       "relevance": "E-OPT benchmark for mathematical programming code generation, provides comparative evaluation framework."
     },
     {
       "title": "Solving General Natural-Language-Description Optimization Problems with Large Language Models",
       "authors": ["Jiahao Zhang et al."],
       "year": 2024,
       "relevance": "OptLLM framework integrating LLMs with solvers for LP problems, directly comparable to CHORUS."
     }
   ]
 }