ai-research-survey

scan.json (20739B)

---

 {
   "paper": {
     "title": "Cracking SQL Barriers: An LLM-based Dialect Translation System",
     "authors": ["Wei Zhou", "Yuyang Gao", "Xuanhe Zhou", "Guoliang Li"],
     "year": 2025,
     "venue": "Proc. ACM Manag. Data (SIGMOD)",
     "doi": "10.1145/3725278"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/weAIDB/CrackSQL. The abstract states 'The code is available at https://github.com/weAIDB/CrackSQL.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 states 'The evaluation set and the translation result have been released at https://github.com/weAIDB/CrackSQL.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is listed (Xeon E5-2678, RTX 3080 Ti) and database versions (MySQL v8.0, PostgreSQL v14, Oracle 11g), but no requirements.txt, Dockerfile, or detailed software dependency listing is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The GitHub repository is referenced but the paper does not include a 'Reproducing Results' section or specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-7 are reported as point estimates (e.g., '74.19%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CrackSQL outperforms baselines based solely on comparing percentage values. No statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are provided throughout, e.g., '3.22%-21.95% result accuracy improvement than SQLGlot' and absolute accuracy values are given for all methods in Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 248, 142, and 111 query pairs for three translation directions. No justification is given for why these sizes are sufficient, nor is any power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or results across multiple runs are reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 lists baselines: SQLGlot, jOOQ, Ora2Pg, SQLines, GPT-4o, CodeLlama-7B, and Llama3.1-8B-Instruct."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o (2024) and Llama3.1 (2024), which are contemporary. Rule-based tools like SQLGlot are actively maintained and represent the state of the art for rule-based translation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.5 provides ablation studies across all three main components: Functionality-based Query Processing (Table 2), Cross-Dialect Embedding Model (Table 4), and Local-To-Global Translation validation mechanisms (Table 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two main metrics are used: Executable Ratio (AccEX) and Result Accuracy (AccRES), plus Retrieval Precision for the embedding model."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of translation quality is included. Evaluation is entirely automated via execution and result comparison against ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.5.4 states 'None of the samples in the evaluation set is included in the training datasets' for the fine-tuning experiments. The evaluation set is separate from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 provides error distributions broken down by translation type (Syntax Rules, Functions & Operators, Keywords, Data Types, Others). Table 1 shows per-dialect-pair breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 discusses translation error distributions in detail, and Table 3 provides specific examples of translations that different tools fail on. Section 7.3 discusses specific failure cases like incorrect TO_TIMESTAMP translation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.5.4 reports that fine-tuning Llama3.1-8B can hurt performance: 'Llama3.1-8B after fine-tuning (i.e., 38.10% accuracy on average) even performs worse over MySQL→PG data samples than the original version (i.e., 54.76% accuracy on average).'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CrackSQL 'significantly outperforms existing methods (e.g., by up to 77.42%)'. Table 1 supports this with improvements up to 77.42% in some scenarios."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by ablation studies (Table 2) that use controlled single-variable manipulation (removing one component at a time)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds scope to MySQL, PostgreSQL, and Oracle, and Section 9 explicitly states limitations: 'translation types supported by CrackSQL are constrained by the available BNF grammar' and notes issues with new dialects that have incomplete documents."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for why CrackSQL outperforms baselines. For example, whether the improvement comes primarily from additional LLM calls (more compute) vs. the proposed techniques is not analyzed. No threats-to-validity section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4o' without specifying an API version or snapshot date. CodeLlama-7B and Llama3.1-8B-Instruct link to HuggingFace model pages but no specific version/checkpoint is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 states 'The detailed LLM instruction prompts are provided at https://github.com/weAIDB/CrackSQL' with a footnote link. This is an external repository with actual prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 reports hyperparameters: k=3 for retrieved specifications, LoRA for fine-tuning, learning rate 1e-4, training epoch 50. Embedding model components (StarEncoder, bge-large-en-v1.5, etc.) are specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding (iterative LLM translation with validation, operation extension, retry logic) is described in detail in Section 6 and Algorithm 1, including the Local-to-Global strategy with syntactic and semantic validation steps."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 describes data collection from three sources (NL2SQL benchmarks, GitHub repos, StackExchange), de-duplication of query templates, and LLM-assisted extraction from StackExchange. Dataset statistics are provided."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Conclusion) contains a substantive limitations discussion: 'Despite its effectiveness, CrackSQL has three main limitations' covering BNF grammar constraints, precision loss issues, and human intervention for new dialects."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations in Section 9 are specific to this system: BNF grammar constraints limiting translation types, precision loss issues (DOUBLE vs. DOUBLE PRECISION), and dependency on well-structured dialect documentation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly states scope: 'we mainly consider the common translation types found in existing dialect translation tools, i.e., translating (1) syntax rules, (2) keywords, (3) built-in functions and operators, (4) data types.' Section 9 notes UDFs and stored procedures are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation set and translation results are released at the GitHub repository (Section 7.2 footnote 9)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 describes three data sources: BIRD NL2SQL benchmark, GitHub repositories (SQLGlot, jOOQ issues and test cases), and StackExchange dumps. Collection methods for each are described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Data comes from publicly available benchmarks, code repositories, and online sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 describes the pipeline: collecting from three sources, de-duplicating query templates, expanding to Oracle, and final dataset statistics (248, 142, 111 query pairs). Section 4.1 documents syntax specification extraction."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists funding: National Key R&D Program of China, NSF of China, Shenzhen Project, Zhongguancun Lab, Huawei, and BNRist."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Shanghai Jiao Tong University and Tsinghua University. These are academic institutions, not product vendors being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders include Chinese government research programs and Huawei. The paper evaluates open-source tools (SQLGlot, jOOQ) and general LLMs (GPT-4o, Llama). Huawei does not have a direct stake in the CrackSQL system's performance vs. these baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o, CodeLlama, and Llama3.1 are used for translation, but no training data cutoff dates are stated. The benchmark queries could potentially be in LLM training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LLMs may have seen the benchmark queries (from BIRD, StackExchange, SQLGlot test cases) during pre-training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark draws from public sources (BIRD benchmark published 2023, StackExchange, SQLGlot repo) that could be in GPT-4o's training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "CrackSQL makes multiple LLM API calls (up to 3-7 trials per operation) but no API costs, token counts, or per-query latency figures are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is listed but no total GPU hours, API spend, or training time for the embedding model is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "CrackSQL significantly outperforms existing methods by up to 77.42% in translation accuracy.",
       "evidence": "Table 1 shows AccRES improvements: 3.22%-21.95% over SQLGlot, 3.22%-39.69% over jOOQ, 9.75%-64.51% over SQLines, 7.69%-34.93% over GPT-4o across six dialect translation scenarios.",
       "supported": "moderate"
     },
     {
       "claim": "Query Segmentation significantly enhances translation accuracy by 48.91% on average.",
       "evidence": "Table 2 ablation study comparing CrackSQL with and without Query Segmentation on Oracle→PG and Oracle→MySQL scenarios.",
       "supported": "moderate"
     },
     {
       "claim": "The Cross-Dialect Embedding Model achieves 86.34% higher retrieval precision than baselines on average.",
       "evidence": "Table 4 shows retrieval precision across PG→MySQL, MySQL→PG, Oracle→PG at k=1,3,5 compared against BM25, StarEncoder, MiniLM, stella, and MultiEmbed.",
       "supported": "moderate"
     },
     {
       "claim": "CrackSQL is robust across different underlying LLMs.",
       "evidence": "Table 5 shows CrackSQL improves all tested LLMs: 6.67%-13.33% for GPT-4o, 15.39%-20% for Llama3.1, 23.08%-50% for CodeLlama. CrackSQL(Llama3.1) achieves similar performance to CrackSQL(GPT-4o).",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning LLMs can negatively affect translation generalizability.",
       "evidence": "Table 6 shows Llama3.1-8B after fine-tuning achieves 38.10% accuracy on average for MySQL→PG, worse than the original 54.76%. Section 7.5.4 discusses this.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CrackSQL is an LLM-based SQL dialect translation system that outperforms rule-based tools (SQLGlot, jOOQ, SQLines) and direct LLM use (GPT-4o) across six translation scenarios between MySQL, PostgreSQL, and Oracle. The system uses three components: functionality-based query processing to segment and simplify queries, a cross-dialect embedding model for matching equivalent syntax across dialects, and a local-to-global translation strategy with hybrid validation. Ablation studies show query segmentation provides the largest accuracy gain (~49%), and the system works with different underlying LLMs including smaller open-source models.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparisons between CrackSQL and baselines are based on raw percentage differences with no significance tests, confidence intervals, or variance across runs. With dataset sizes of 111-248 query pairs, observed differences may not be statistically significant."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "The evaluation dataset draws from public sources (BIRD benchmark, StackExchange, SQLGlot test cases) that may appear in GPT-4o's training data. This could affect both the baseline GPT-4o results and CrackSQL's own performance since it uses GPT-4o internally. This is not discussed."
     },
     {
       "flag": "No cost analysis",
       "detail": "CrackSQL makes multiple LLM API calls per query (up to 7 trials) plus embedding model inference, but no cost or latency comparison is provided. This makes practical deployment assessment impossible."
     },
     {
       "flag": "Self-evaluation by authors' own system",
       "detail": "The benchmark dataset was partially constructed by the authors (6 human months), and Oracle translations were expanded by 'verify and collect the successfully translated queries.' The selection process could inadvertently favor CrackSQL's capabilities."
     }
   ],
   "cited_papers": [
     {
       "title": "Can LLM Already Serve as A Database Interface? A BIg Bench for Large-Scale Database Grounded Text-to-SQLs",
       "authors": ["Jinyang Li", "Binyuan Hui", "Ge Qu"],
       "year": 2023,
       "relevance": "BIRD benchmark used as a data source for the dialect translation evaluation dataset."
     },
     {
       "title": "Lost in Translation: A Study of Bugs Introduced by Large Language Models while Translating Code",
       "authors": ["Rangeet Pan", "Ali Reza Ibrahimzada", "Rahul Krishna"],
       "year": 2024,
       "relevance": "Studies bugs introduced by LLMs during code translation, directly relevant to LLM reliability in code generation."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "relevance": "StarEncoder used as the syntax structure encoder in CrackSQL; relevant to code LLM capabilities."
     },
     {
       "title": "Large language models: A survey",
       "authors": ["Shervin Minaee", "Tomas Mikolov"],
       "year": 2024,
       "relevance": "Survey of LLM capabilities cited to motivate using LLMs for SQL translation tasks."
     },
     {
       "title": "LLM-R2: A Large Language Model Enhanced Rule-based Rewrite System for Boosting Query Efficiency",
       "authors": ["Zhaodonghui Li", "Haitao Yuan"],
       "year": 2024,
       "relevance": "Related work on using LLMs to enhance rule-based database systems, closely related to the LLM-augmented approach."
     },
     {
       "title": "Query Rewriting via Large Language Models",
       "authors": ["Jie Liu", "Barzan Mozafari"],
       "year": 2024,
       "relevance": "LLM-based query rewriting via iterative self-reflection, related approach to using LLMs for SQL transformation."
     },
     {
       "title": "D-Bot: Database Diagnosis System using Large Language Models",
       "authors": ["Xuanhe Zhou", "Guoliang Li"],
       "year": 2024,
       "relevance": "LLM-based database diagnosis system, part of the broader trend of using LLMs for database tasks."
     },
     {
       "title": "GPTuner: A Manual-Reading Database Tuning System via GPT-Guided Bayesian Optimization",
       "authors": ["Jiale Lao", "Yibo Wang"],
       "year": 2024,
       "relevance": "LLM-augmented database tuning system, relevant to LLM applications in database management."
     }
   ]
 }