ai-research-survey

scan.json (24048B)

---

 {
   "paper": {
     "title": "Play by the Type Rules: Inferring Constraints for LLM Functions in Declarative Programs",
     "authors": ["Parker Glenn", "Alfy Samuel", "Daben Liu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.20208",
     "doi": "10.48550/arXiv.2509.20208"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Type-constrained decoding for LLM functions in SQL-like query languages improves accuracy by up to 7% on HybridQA multi-hop QA with 53% latency improvement over comparable systems. Small language models (3b) can approach larger model (8b) performance as function executors when given appropriate type constraints. BlendSQL's DB-first approach reduces latency from 1.7s to 0.76s versus LOTUS on TAG-Bench.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/parkervg/blendsql. Specific version blendsql==0.0.48 mentioned in Appendix B."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available datasets: HybridQA (Chen et al., 2020) and TAG-Bench (Biswal et al., 2024). No proprietary data collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B specifies blendsql==0.0.48, llama-cpp-python 0.3.16, specific llama.cpp commit hash, lotus-ai==1.1.3, ollama 0.6.7, specific model quants (Q4_K_M), GPU hardware (RTX 5080, A10, A100)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and environment details provided, no step-by-step reproduction instructions or scripts to replicate the main experiments are described. Appendix B mentions specific files (tag_queries.py, hand_written.py) but no README or reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports ± for latency (0.76 +/- 0.002) but the main accuracy results in Table 3 and Figure 3 report only point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like '7% accuracy improvement' and comparisons between typing policies are based on raw number differences without any test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute improvements with baselines are provided throughout: e.g., denotation accuracy rises by 6.6 points with type constraints (Section 5.1), 53% latency improvement (Table 2), and full accuracy/F1/denotation tables (Table 3) give baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Uses first 1,000 examples from HybridQA validation set and 60 TAG-Bench questions without justifying why these sizes or subsets were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Latency results in Table 2 are averaged over 5 runs with std dev, but the main accuracy results (Table 3, Figure 3) appear to be single-run with no variance reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: No Context, All Context, RAG, and LOTUS comparison (Table 2, Table 3). Also compares three typing policies (No Type Hints, Type Hints, Type Hints + Constrained Decoding)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LOTUS (Patel et al., 2024) is contemporary. Models used include Llama 3 series (2024) and gemma-3-12b-it (2024). TAG-Bench and HybridQA are established benchmarks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 ablates the three typing policies. Figure 6 ablates documentation vs. few-shot only. Figure 7 ablates CFG-constrained generation. Figure 8 shows hyperparameter sweeps for k values."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports Accuracy, F1, and Denotation Accuracy on HybridQA (Table 3). Reports latency and average tokens per program on TAG-Bench (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via exact match, F1, and denotation accuracy metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported on the HybridQA validation set, not a held-out test set. The paper states 'first 1,000 examples from the HybridQA validation set.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides per-model-size breakdowns across all parsing/execution model combinations. Table 4 categorizes execution errors. Figure 9 shows per-sample latency."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 categorizes execution errors (Empty LLMQA Context, Generic SQLite Syntax, Hallucinated Column, etc.). Section 5.2 discusses 102/1000 execution failures. Appendix A.1 explores syntax errors."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows CFG-constrained generation actually harms 70b model performance. Figure 6 shows removing documentation can moderately improve smaller models. The 1b model shows steep performance drop-off."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims '7% accuracy improvement' and '53% improvement in latency' are supported by Table 3 (type constraints lift) and Table 2 (1.7s to 0.76s). 'Small language models can excel as function executors' is supported by Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about type constraints improving performance are supported by controlled ablation (Figure 3) varying only the typing policy while holding other factors constant. The claim is about the mechanism (constrained decoding), and the ablation isolates this variable."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states the approach 'can be extended to any typed declarative programming language' but evaluation is only on SQLite-based BlendSQL with two datasets (HybridQA and TAG-Bench). The title and framing are broader than the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for observed improvements. For example, no discussion of whether constrained decoding simply eliminates formatting errors vs. actually improving reasoning, or whether improvements are dataset-specific."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims are stated at the granularity of the measurements: denotation accuracy on HybridQA, latency on TAG-Bench. No broader framing like 'reasoning ability' without qualification."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions given: Llama-3.2-1B-Instruct, Llama-3.2-3B-Instruct, Llama-3.1-8B-Instruct, Llama-3.3-70B-Instruct, gemma-3-12b-it. Specific quant (Q4_K_M) and commit hashes provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for LLMQA (Figure 4) and LLMMAP (Figure 5) are provided with complete text including all template variables and formatting."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Max context length of 8000 is stated. Retrieval k values reported. However, LLM generation hyperparameters (temperature, top-p, max tokens) are not reported for the model inference calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The BlendSQL execution pipeline is described in detail: AST parsing, depth-first traversal, function execution flow (Algorithm 1, Figure 2), temporary table creation, and query compilation to SQL."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the evaluation setup: 1,000 HybridQA validation examples, article text split into sentences for search index, lowercase normalization, k values for retrieval. Table 2 setup is described in Appendix B."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section is present in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address specific limitations of the study design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The conclusion broadens claims ('can be extended to any typed declarative programming language') without stating what was NOT tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs, model predictions, or per-example results are released. Only aggregate numbers in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly described: HybridQA validation set (Chen et al., 2020), TAG-Bench from BIRD-SQL (Li et al., 2023). Evaluation procedure is documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The execution pipeline from question to answer is documented: query parsing → AST traversal → function execution → type constraint inference → constrained decoding → SQL compilation (Algorithm 1, Figure 2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present. Authors are from Capital One, a financial company."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with Capital One are clearly stated in the header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Capital One is a financial services company. While not directly selling the evaluated tool, the work is corporate-sponsored research with no funding independence discussion."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the Llama or Gemma models used. HybridQA (2020) and TAG-Bench could potentially be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HybridQA or TAG-Bench data appeared in the training sets of the Llama 3 or Gemma models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HybridQA was published in 2020. The Llama 3 models (2024) were likely trained on data that could include HybridQA. This is not addressed. The 'No Context' baseline showing some accuracy (up to 6.6% for 70b) suggests parametric knowledge overlap."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Latency is reported: 0.76s per sample for BlendSQL vs 1.7s for LOTUS (Table 2). Average tokens per program (76 vs 127) also reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specified: 4x24GB A10 GPUs for smaller models, 4x80GB A100 for 70b, RTX 5080 for latency experiments (Appendix B). Max context length 8000."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Latency results averaged over 5 runs, but accuracy results appear to be single-run with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 states 'average runtime across 5 runs' for latency, but the main accuracy experiments (Table 3, Figure 3) do not state the number of runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Figure 8 shows hyperparameter sweeps for k values but no total search budget or method is described for other hyperparameters."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of k=1 for LLMSEARCHMAP and k=10 for LLMQA is shown in Figure 8 but the selection criterion (which split was used) is not described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applicable. However, many comparisons are made across model sizes and typing policies without any correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors compare their BlendSQL system against LOTUS using their own implementations. No acknowledgment of potential bias in re-implementing baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 explicitly compares performance at matched hardware (same RTX 5080, same quantized model) and across different hardware/model configurations. Latency and tokens per program allow compute-aware comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HybridQA or TAG-Bench actually measure the capabilities claimed (multi-hop reasoning, type alignment). The benchmarks are taken at face value."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "BlendSQL and LOTUS use different scaffolding (SQL compilation vs Pandas API). When comparing systems (Table 2), the scaffold difference is not addressed — performance differences could stem from the scaffold rather than the typing approach."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HybridQA was published in 2020, Llama 3 models trained in 2024. No discussion of whether models saw HybridQA data during training. The 'No Context' baseline achieving up to 6.6% accuracy suggests some memorization."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information, e.g., whether type hints or constrained decoding provide answer-leaking signal."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between HybridQA examples or potential overlap with training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used despite the temporal gap between benchmark creation (2020) and model training (2024)."
       }
     }
   },
   "claims": [
     {
       "claim": "Type Hints + Constrained Decoding improves denotation accuracy by up to 7% over unconstrained generation on HybridQA",
       "evidence": "Figure 3 shows consistent improvements across all model size combinations. Biggest lift: 3b executor with 70b parser, 33.3→45.3 (+12 points) from No Type Hints to Type Hints + Constrained Decoding.",
       "supported": "strong"
     },
     {
       "claim": "BlendSQL achieves 53% latency improvement over LOTUS on TAG-Bench",
       "evidence": "Table 2: 0.76s vs 1.7s on same hardware (RTX 5080) with same model (Llama-3.1-8b-Instruct Q4_K_M), averaged over 5 runs.",
       "supported": "strong"
     },
     {
       "claim": "Small language models (3b) can approach larger model (8b) performance as function executors when given program decompositions",
       "evidence": "Table 3: 3b Program Execution achieves 45.3 denotation accuracy vs 8b RAG at 45.6, though the 3b uses 70b-generated programs while the 8b does its own end-to-end reasoning.",
       "supported": "moderate"
     },
     {
       "claim": "The approach can be extended to any typed declarative programming language",
       "evidence": "Section 7 conclusion claim. Only tested on SQLite-based BlendSQL. No evidence for other languages.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis despite temporal gap",
       "detail": "HybridQA was published in 2020 and models trained in 2024. The 'No Context' baseline shows 6.6% accuracy for the 70b model, suggesting parametric knowledge of the dataset. No contamination analysis is performed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries."
     },
     {
       "flag": "Company evaluating its own system",
       "detail": "Capital One authors evaluating BlendSQL (created by first author Glenn et al., 2024) without acknowledging self-evaluation bias."
     },
     {
       "flag": "No statistical tests for accuracy claims",
       "detail": "All accuracy comparisons are raw number differences with no significance tests, confidence intervals, or multi-run variance."
     },
     {
       "flag": "Validation set used as test set",
       "detail": "Main results are reported on HybridQA validation set, not a held-out test set. Hyperparameter sweeps (Figure 8) appear to use the same data."
     }
   ],
   "cited_papers": [
     {
       "title": "BlendSQL: A Scalable Dialect for Unifying Hybrid Question Answering in Relational Algebra",
       "authors": ["Parker Glenn", "Parag Dakle", "Liang Wang", "Preethi Raghavan"],
       "year": 2024,
       "relevance": "Predecessor system by same authors; foundational work on integrating LLM functions into SQL queries."
     },
     {
       "title": "Semantic Operators: A Declarative Model for Rich, AI-based Analytics over Text Data",
       "authors": ["Liana Patel", "Siddharth Jha", "Parth Asawa"],
       "year": 2024,
       "arxiv_id": "2407.11418",
       "relevance": "LOTUS system compared against in efficiency experiments; key baseline for LLM-DBMS integration."
     },
     {
       "title": "Type-Constrained Code Generation with Language Models",
       "authors": ["Niels Mündler", "Jingxuan He", "Hao Wang", "Koushik Sen", "Dawn Song", "Martin Vechev"],
       "year": 2025,
       "relevance": "Most related prior work on enforcing type safety in LLM code generation."
     },
     {
       "title": "Efficient Guided Generation for Large Language Models",
       "authors": ["Brandon T Willard", "Rémi Louf"],
       "year": 2023,
       "arxiv_id": "2307.09702",
       "relevance": "Foundational work on constrained decoding for LLMs used in the approach."
     },
     {
       "title": "Text2SQL is Not Enough: Unifying AI and Databases with TAG",
       "authors": ["Asim Biswal", "Liana Patel", "Siddarth Jha"],
       "year": 2024,
       "arxiv_id": "2408.14717",
       "relevance": "TAG-Bench dataset used for efficiency evaluation; demonstrates need for LLM-database integration."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "relevance": "Primary model family used across all experiments (1b, 3b, 8b, 70b variants)."
     },
     {
       "title": "Binding Language Models in Symbolic Languages",
       "authors": ["Zhoujun Cheng", "Tianbao Xie", "Peng Shi"],
       "year": 2023,
       "relevance": "Denotation accuracy metric and approach to combining LLMs with symbolic languages."
     },
     {
       "title": "XGrammar: Flexible and Efficient Structured Generation Engine for Large Language Models",
       "authors": ["Yixin Dong", "Charlie F Ruan"],
       "year": 2024,
       "arxiv_id": "2411.15100",
       "relevance": "Optimized constrained decoding engine relevant to the grammar-constrained generation approach."
     },
     {
       "title": "Beyond Quacking: Deep Integration of Language Models and RAG into DuckDB",
       "authors": ["Anas Dorbani", "Sunny Yasser", "Jimmy Lin", "Amine Mhedhbi"],
       "year": 2025,
       "relevance": "Alternative approach to integrating LLMs with DBMS (DuckDB UDFs)."
     }
   ]
 }