ai-research-survey

scan.json (26184B)

---

 {
   "paper": {
     "title": "Auto-Cypher: Improving LLMs on Cypher generation via LLM-supervised generation-verification framework",
     "authors": [
       "Aman Tiwari",
       "Shiva Krishna Reddy Malay",
       "Vikas Yadav",
       "Masoud Hashemi",
       "Sathwik Tejaswi Madhusudhan"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2412.12612"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link or code repository is provided. The paper mentions a HuggingFace dataset link (https://huggingface.co/datasets/ServiceNow-AI/SynthCypher) but does not release the pipeline code, training scripts, or evaluation code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The SynthCypher dataset is publicly available on HuggingFace at https://huggingface.co/datasets/ServiceNow-AI/SynthCypher, as stated on the first page of the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section is provided. The paper mentions training hyperparameters (learning rate, batch size, epochs) but not the software environment or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts to replicate experiments are provided. The pipeline is described at a high level, and prompts are in the appendix, but there are no instructions for a researcher to reproduce the full pipeline or training runs."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 and Figure 3 report only point estimates (e.g., '71.4' accuracy) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SynthCypher leads to 'significant improvements' and '40% absolute improvement' but provides no statistical significance tests (no p-values, t-tests, or similar). Comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage improvements with baseline context, e.g., 'up to 40% accuracy improvement on 7B & 8B models' and Table 2 shows both baseline and fine-tuned scores (e.g., Llama-3.1-8B goes from 30.9 to 71.4 on SynCy-test), allowing readers to assess the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the training set size of 29.8k or test set size of 2k/4k. No power analysis or discussion of whether these sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across experimental runs. Results appear to be from single training runs with no indication of multiple seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 includes three baseline setups: (1) Base IFT (models fine-tuned on generic instruction data without SynthCypher), (2) off-the-shelf Instruct model versions, and (3) comparison with Neo4j Labs (tomasonjo_gpt4o) dataset in Figure 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines use contemporary models (Llama 3.1, Qwen 2.5, Mistral v0.2) and the only other existing Text2Cypher dataset (Neo4j Labs/tomasonjo_gpt4o from 2024). The paper acknowledges that Text2Cypher is an underexplored area with limited prior work."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented. The pipeline has multiple components (schema generation, LLM-As-Database-Filler, chain-of-thought Cypher generation, validation), but no experiments isolate the contribution of individual components. For example, no experiment shows what happens without the validation step or without the database-filling strategy."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only a single metric is used: LLM-as-Judge exact match (GPT-4o gives 1 if all requested information is present, 0 otherwise). No other metrics such as syntactic validity rate, execution success rate, or partial credit are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated Cypher queries or model outputs is performed. The only human involvement is manual vetting of 25% of generated schemas during data creation (Appendix B.2), which evaluates the dataset itself, not the fine-tuned model's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'For testing, we employed a separate dataset of 4k samples, covering all 109 query types across 165 schemas not included in train.' Additionally, the SPIDER-Cypher benchmark is a separate external test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Despite having 109 query types of varying complexity, the paper only reports overall accuracy numbers in Table 2 and Figure 3. No per-query-type or per-complexity-level breakdown is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis, qualitative examples of failures, or discussion of where the fine-tuned models produce incorrect Cypher queries. The paper only shows aggregate success rates."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement from SynthCypher training. No approaches that were tried and abandoned, or configurations that failed, are discussed. No negative results or unsuccessful variations are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 40% on the Text2Cypher test split and 30% on the SPIDER benchmark.' Table 2 supports these: Llama-3.1-8B improves from 30.9 to 71.4 on SynCy-test (~40%) and from 30.8 to 62.2 on SPIDER (~31%). The other abstract claims about pipeline quality and dataset diversity are supported by the methodology description."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are of the form 'fine-tuning on SynthCypher improves performance.' This is supported by a controlled comparison: same models fine-tuned with vs. without SynthCypher data (Table 2, comparing Base IFT vs. Base + SynCy rows). This is a standard ablation-style controlled comparison adequate for this type of causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Improving LLMs on Cypher generation' is broader than what was tested. Results are only for 7B/8B parameter models; the Limitations section acknowledges 'we have limited this research to smaller models and it is not clear if the same strategy would work on larger models,' but neither the title nor the abstract bounds claims to this model size range."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, no consideration of whether the improvement comes from the additional training data volume rather than data quality, or whether the LLM-as-Judge metric might favor certain query styles. The Limitations section discusses bias risks and model size constraints but not alternative explanations for the observed results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models are partially versioned (e.g., 'Mistral-v0.2-7B', 'Llama-3.1-8B', 'Qwen2-7B', 'QwenCoder-2.5-7B'). However, 'GPT-4' used in the pipeline (Section 3) and 'GPT-4o' used as LLM-as-Judge have no snapshot date or API version specified. These are critical components whose behavior varies across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The appendix (Figures 4-14) provides the full prompt text for all major pipeline steps: schema generation (Fig 4-5), question generation (Fig 6), ground truth generation (Fig 7), database infilling (Fig 8-9), Cypher generation steps (Fig 10-13), and the LLM-as-Judge prompt (Fig 14). These are actual prompts with placeholders whose fill values are specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Training hyperparameters are partially reported: 'learning rate=1e-05, batch size of 128 over three epochs' (Section 4). However, temperature/sampling settings for GPT-4 and GPT-4o used in the pipeline and evaluation are not stated, nor are temperature settings for Mixtral during schema/question generation (Appendix B mentions 'higher temperature (0.8)' for domain generation only)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-step pipeline is described in detail in Section 3 (Steps 1-5) and the appendix. The chain-of-thought Cypher generation process using four sequential LLM calls (analyze question, identify nodes, incorporate best practices, generate final Cypher) is documented in Appendix F and Figures 10-13."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data generation pipeline is documented in Section 3 with five explicit steps. Table 1 shows final dataset sizes (29,838 train, 2,000 test). The validation step describes how queries are retried up to 5 times before discarding, and only execution-correct queries are retained. Schema generation, question generation, and database population processes are all documented with prompts in the appendix."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' is present with substantive discussion of synthetic data biases, model size limitations, and benchmark contamination risks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 7) discusses specific threats: (1) synthetic data may reinforce model biases and not model real-world scenarios, (2) research is limited to smaller (7B/8B) models and may not transfer to larger models, (3) SPIDER test dataset was publicly released and may be in pre-training data. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section mentions smaller models and synthetic data risks, it does not explicitly state what the results do NOT show. For example, it does not state that results do not generalize to production Neo4j databases with real (not synthetic) schemas, or that the LLM-as-Judge metric may not capture all quality dimensions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The SynthCypher dataset is publicly available on HuggingFace (https://huggingface.co/datasets/ServiceNow-AI/SynthCypher), allowing independent verification of the training and test data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is thoroughly described in Section 3 (Steps 1-5) and Appendices B-F, covering schema generation, question generation, ground truth generation, database population, Cypher generation, and validation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is synthetically generated using LLMs and standard benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from schema generation through validation is documented in Section 3 with five explicit steps. Table 1 shows final dataset counts. The paper explains that only queries producing correct results are retained after up to 5 retries. However, the exact attrition numbers at each step (how many were generated vs. retained) are not fully reported."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. All authors are from ServiceNow, but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are listed as affiliated with ServiceNow, with ServiceNow email addresses, clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are ServiceNow employees. ServiceNow offers graph database products and AI capabilities, giving them a potential commercial interest in demonstrating strong Text2Cypher performance. No statement of funder independence is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (Llama 3.1, Mistral, Qwen, CodeLlama, GPT-4, GPT-4o). This is relevant because the models are evaluated on SPIDER-Cypher, derived from the public SPIDER benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 7) states: 'SPIDER test dataset has been publicly released as of Feb-2024 and it is not clear if any of that data went into the pre-training of base models or the Instruct models we considered.' This explicitly flags potential train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper acknowledges the SPIDER contamination risk in the Limitations section, no mitigation steps are taken (e.g., no canary strings, no temporal analysis, no comparison with contamination-safe benchmarks). The acknowledgment is present but the contamination is not addressed — just flagged."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency, or per-example costs are reported for either the data generation pipeline (which uses GPT-4 and Mixtral) or the fine-tuned models at inference time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report GPU hours for training, API costs for GPT-4/GPT-4o calls during data generation and evaluation, or hardware specifications used."
       }
     }
   },
   "claims": [
     {
       "claim": "SynthCypher fine-tuning leads to up to 40% absolute accuracy improvement on the Text2Cypher test split over base IFT models.",
       "evidence": "Table 2 shows Llama-3.1-8B improves from 30.9 (Base IFT) to 71.4 (Base + SynCy) on SynCy-test, a 40.5% absolute improvement. Similar improvements are seen across other models (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "SynthCypher fine-tuning leads to up to 30% improvement on the SPIDER benchmark adapted for Cypher.",
       "evidence": "Table 2 shows Llama-3.1-8B improves from 30.8 (Base IFT) to 62.2 (Base + SynCy) on SPIDER, a 31.4% absolute improvement (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "SynthCypher outperforms the Neo4j Labs (tomasonjo_gpt4o) dataset when training data size is controlled.",
       "evidence": "Figure 3 shows Llama-3.1-8B fine-tuned on down-sampled SynthCypher (~7.7k) outperforms the same model fine-tuned on Neo4j Labs data (7.7k) on both SynthCypher and SPIDER test splits (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "Off-the-shelf instruct LLMs and base IFT models achieve very low performance on Text2Cypher, demonstrating the need for task-specific data.",
       "evidence": "Table 2 shows instruct models range from 27.7% to 40.2% on SynCy-test and 25.2% to 50.8% on SPIDER. Base IFT models range from 14.6% to 50.85% on SynCy-test (Section 5).",
       "supported": "strong"
     },
     {
       "claim": "The LLM-As-Database-Filler is a novel strategy not explored in prior literature.",
       "evidence": "Section 3, Step 3 states: 'To the best of our knowledge, this strategy of filling the database conditioned on an arbitrarily chosen dummy ground truth has not been explored in literature before.' No prior work citation contradicts this claim.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Auto-Cypher presents a synthetic data generation pipeline for Text2Cypher that produces the SynthCypher dataset (29.8k training instances across 700 domains and 109 query types). Fine-tuning 7B/8B LLMs on SynthCypher yields up to 40% absolute accuracy improvement on the Text2Cypher test set and 30% on an adapted SPIDER benchmark, compared to models fine-tuned only on generic instruction data. The pipeline's key innovation is the LLM-As-Database-Filler strategy, which generates synthetic Neo4j databases conditioned on expected answers to validate Cypher query correctness through execution.",
   "red_flags": [
     {
       "flag": "No variance or error bars",
       "detail": "All results appear to be single-run numbers with no standard deviations, confidence intervals, or multiple seeds. Given the well-known variance in LLM fine-tuning outcomes across random seeds, the reported improvements may not be robust."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "The only metric is an LLM-as-Judge (GPT-4o) binary match. This metric itself is not validated against human judgment for Cypher queries, and it may systematically favor certain query styles or miss partial correctness."
     },
     {
       "flag": "No ablation study",
       "detail": "The pipeline has multiple components (schema diversity, database filling, chain-of-thought generation, execution validation), but no ablation study isolates which components drive the improvement. The improvement could come simply from having more in-domain training data rather than from the pipeline's specific innovations."
     },
     {
       "flag": "Evaluating on own synthetic test set",
       "detail": "The primary evaluation (SynCy-test) is on a test split of the authors' own synthetically generated dataset, using the same pipeline that generated the training data. The test set may share distributional biases with the training set, inflating reported performance. The SPIDER-Cypher results partially mitigate this concern."
     },
     {
       "flag": "Undisclosed costs",
       "detail": "The pipeline uses GPT-4 for multiple steps (schema validation, database population code, Cypher generation, validation) and GPT-4o as the evaluation judge, but no API costs are reported. This is relevant for assessing whether the pipeline is practically scalable."
     },
     {
       "flag": "Corporate affiliation without conflict disclosure",
       "detail": "All authors are ServiceNow employees. ServiceNow has commercial interests in AI and data management products. No competing interests statement or conflict of interest disclosure is present."
     }
   ],
   "cited_papers": [
     {
       "title": "Can LLM Already Serve as a Database Interface? A Big Bench for Large-Scale Database Grounded Text-to-SQLs",
       "authors": ["Jinyang Li", "Binyuan Hui", "Ge Qu"],
       "year": 2023,
       "arxiv_id": "2305.03111",
       "relevance": "Major benchmark (BIRD) for evaluating LLMs on text-to-SQL, directly relevant to understanding LLM code generation capabilities for database queries."
     },
     {
       "title": "Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task",
       "authors": ["Tao Yu", "Rui Zhang", "Kai Yang"],
       "year": 2018,
       "relevance": "Foundational text-to-SQL benchmark adapted for Cypher evaluation in this paper; widely used in code generation evaluation."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "GPT-4 is used as a core component of the data generation pipeline, relevant to understanding LLM capabilities in code generation tasks."
     },
     {
       "title": "WizardCoder: Empowering Code Large Language Models with Evol-Instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Influential code LLM training approach using synthetic instruction evolution, relevant to LLM code generation methodology."
     },
     {
       "title": "Mistral 7B",
       "authors": ["Albert Q Jiang", "Alexandre Sablayrolles", "Arthur Mensch"],
       "year": 2023,
       "arxiv_id": "2310.06825",
       "relevance": "One of the base models fine-tuned in this work; relevant as a widely-used open-source LLM for code tasks."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Laurens Van Der Maaten et al."],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Llama 3.1 is the best-performing base model in the paper's experiments; relevant as a state-of-the-art open-source LLM."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "QwenCoder is one of the code-specialized models evaluated, relevant to understanding specialized code LLMs."
     },
     {
       "title": "Mixtral of Experts",
       "authors": ["Albert Q. Jiang", "Alexandre Sablayrolles"],
       "year": 2024,
       "arxiv_id": "2401.04088",
       "relevance": "Mixtral 8x22B is used for schema and question generation in the pipeline; relevant as a mixture-of-experts architecture for code generation."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper relevant to understanding instruction-following capabilities of LLMs used in code generation."
     },
     {
       "title": "Synthet2C: Generating Synthetic Data for Fine-Tuning Large Language Models on the Text2Cypher Task",
       "authors": ["Ziije Zhong", "Linqing Zhong", "Zhaoze Sun"],
       "year": 2024,
       "arxiv_id": "2406.10710",
       "relevance": "Concurrent work on synthetic Text2Cypher data generation; directly comparable approach limited to medical domain."
     },
     {
       "title": "A Survey on Employing Large Language Models for Text-to-SQL Tasks",
       "authors": ["Liang Shi", "Zhengju Tang", "Nan Zhang"],
       "year": 2024,
       "arxiv_id": "2407.15186",
       "relevance": "Survey of LLM text-to-SQL approaches, relevant to understanding the broader landscape of LLM-based database query generation."
     }
   ]
 }