ai-research-survey

scan.json (24235B)

---

 {
   "paper": {
     "title": "Automated Discovery of Test Oracles for Database Management Systems Using LLMs",
     "authors": ["Qiuyang Mang", "Runyuan He", "Suyang Zhong", "Xiaoxuan Liu", "Huanchen Zhang", "Alvin Cheung"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.06663"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The tool Argus is described but no release link is given."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release the bug reports, generated CAQ pairs, SQL snippet corpus, or any other experimental data. Bug report links to individual DBMS issue trackers are not provided as a dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '64 cores and 128 GB memory running on Ubuntu 24.04' (Section 7) but does not provide library versions, dependency lists, requirements.txt, or any reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The algorithms are described at a conceptual level but there are no concrete instructions for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates for coverage metrics and bug counts without confidence intervals or error bars. Coverage figures (e.g., '19.9% higher line coverage') and metamorphic coverage ratios are presented without uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Argus outperforms baselines in coverage and bug detection but does not apply any statistical significance tests to these comparisons. Differences are reported as raw ratios (e.g., '5.473x', '6.43x') without tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout: e.g., '19.9% and 18.1% higher line and branch coverage than SQLancer++' (Section 7.2), '5.473x, 6.431x, and 5.571x higher' metamorphic coverage (Table 3), '3.33x more unique logic bugs' (Section 7.3), with absolute values provided in figures and tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Key parameters like '10 test suites, each with 100 test cases' and '20 bug reports' for the false positive analysis are used without justification for why these sample sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The metamorphic coverage experiment uses 10 test suites and reports averages (Table 3), but no standard deviation, variance, or spread measure is reported. Coverage experiments appear to be single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Argus against SQLancer and SQLancer++ for code coverage (Section 7.2), a composite baseline of 11 oracles from four prior works (TLP, NoREC, EET, DQP) for bug finding (Section 7.3), and an LLM-as-a-judge baseline for the prover ablation (Section 7.4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "SQLancer and SQLancer++ are actively maintained state-of-the-art DBMS testing tools. The composite baseline includes oracles from 2020-2024 works (TLP 2020, NoREC 2020, EET 2024, DQP 2024). The LLM baseline uses GPT-5."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes multiple ablation-like studies: comparing Argus-50 vs Argus-5000 to evaluate oracle quantity (Section 7.3), evaluating the SQL equivalence prover vs LLM-as-a-judge (Section 7.4), and comparing the CAQ-based approach vs a naive baseline for cost/efficiency (Section 7.5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses line coverage, branch coverage, metamorphic coverage (line/function/branch), number of unique bugs found, false positive rate, false negative rate, cost, and throughput across different experiments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Bug reports were manually adjudicated: 'we then manually adjudicated each report as a true positive or a false positive' (Section 7.4). Additionally, all 40 bugs were reported to DBMS developers, with 36 confirmed and 26 fixed by developers."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a software testing tool evaluation, not a machine learning model evaluation. There is no train/test split concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-DBMS breakdowns of bugs found (Dolt, DuckDB, MySQL, PostgreSQL, TiDB), categorized by bug status (fixed, confirmed, duplicate, pending) and bug type (logic vs other). Coverage results are broken down by DBMS and metric type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 discusses false positive and false negative cases in detail, including a specific false positive example (Listing 11). Section 8 discusses prover bugs and limitations. The paper acknowledges Argus-50 underperformed the baseline."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: Argus-50 found fewer bugs than the human-designed baseline (Section 7.3), the prover has a high false negative rate (8/10 rejections were false negatives, Section 7.4), and Argus slightly underperforms SQLancer on PostgreSQL line coverage (Section 7.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '40 previously unknown bugs, 35 of which are logic bugs, with 36 confirmed and 26 already fixed' — directly supported by Table 2. Claims of coverage improvement ('up to 1.19x in code coverage, and 6.43x in metamorphic coverage') are supported by Section 7.2 and Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about why Argus works (the CAQ abstraction enables scalability, the prover ensures soundness) and supports these through controlled ablation studies: removing the prover (Section 7.4), varying oracle count (Section 7.3), comparing CAQ vs naive approach (Section 7.5). The ablation designs involve single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Limitations) explicitly bounds generalization: 'Argus currently focuses on relational DBMS and SQL' and acknowledges the prover 'currently supports only a subset of SQL features.' The scope is clearly limited to the five tested DBMSs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why Argus finds more bugs. For example, it does not consider whether the improvement is due to the LLM's creativity, the formal verification, the larger number of test cases, or the specific SQL features tested. The discussion section covers limitations but not confounding factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'o4-mini' and 'GPT-5' but does not provide specific API versions, snapshot dates, or version identifiers (e.g., no model checkpoint or API version string). These are marketing names without version pinning."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompting strategy conceptually: 'The prompt qualitatively asks the LLM to generate novel queries that are different from the provided examples' (Section 5.2) and mentions 'Documentation-augmentation Generation' (Section 6.1), but the actual prompt text is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported. Algorithm parameters like StopThreshold (initially 3), MaxDepth, N (number of schemas), and K (test cases per oracle) are mentioned but their experimental values are not all specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: Algorithms 1 and 2 describe the full pipeline including iterative prompting with in-context learning from Equal/Fail sets, k-means clustering of query plans for example selection, the SQL equivalence prover verification loop, and the corpus synthesis and instantiation process."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents preprocessing steps: grammar-based seed generation (Section 5.1), runtime validation of LLM-generated snippets (Section 6.1, Listing 4), filtering via determinism/null-preserving/empty-preserving constraints (Section 6.2), and cross-combination of expressions (Section 6.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Discussion) contains a dedicated 'Limitations and future work' subsection that discusses three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) the SQL equivalence prover supports only a subset of SQL features, limiting oracle scope; (2) the approach is limited to relational DBMS and SQL; (3) no mechanism for prioritizing or ranking generated oracles. Additionally, prover bugs are discussed as a real encountered issue."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states scope boundaries: 'Argus currently focuses on relational DBMS and SQL' and 'does not provide a mechanism to prioritize or rank the generated oracles.' The paper also notes that the prover 'does not understand the semantics of most functions, treating them as uninterpreted functions.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. Bug reports, generated CAQ pairs, snippet corpora, coverage measurements, and experimental logs are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The bug collection process is well documented: 'We continuously ran Argus on the five DBMSs for a three-month testing campaign. We used the latest development versions and reported bugs only when they could be reproduced on their latest versions' (Section 7.1). The deduplication policy is also described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study tests automated tools against DBMSs with no human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from schema generation through CAQ generation, prover verification, corpus synthesis, instantiation, and bug detection is documented in Algorithms 1 and 2. Section 7.1 describes the bug reporting workflow including deduplication against known issues."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Section 11 (Acknowledgements) thanks individuals for feedback and support but does not disclose any funding sources, grants, or sponsorships."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: UC Berkeley and National University of Singapore. The paper tests third-party open-source DBMSs, not products by the authors' institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, making it impossible to assess funder independence. The absence of a funding disclosure is itself a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. LLMs are used as a generation tool within the Argus pipeline, not evaluated for their standalone performance."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper tests DBMSs for bugs, not LLM benchmark performance. Whether the LLM has seen SQL in training does not affect the validity of formally verified equivalent query pairs."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation of model capabilities. The LLM is a tool for generating candidate queries, not the subject of evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates an automated testing tool."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.5 reports detailed cost information: 'generating the CAQ pairs costs about $3 in LLM calls, and instantiating test cases costs roughly $1 per 1,000, leveraging a reusable corpus of 100,000 snippets built for only $12.' Figure 5 shows cumulative cost over time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the machine specification (64 cores, 128 GB memory, Ubuntu 24.04) and testing duration (three-month campaign, 24-hour coverage runs), but does not quantify total compute budget (total CPU/GPU hours, total API spend across all experiments)."
       }
     }
   },
   "claims": [
     {
       "claim": "Argus discovered 40 previously unknown bugs across five DBMSs, including 35 logic bugs, with 36 confirmed and 26 fixed by developers.",
       "evidence": "Table 2 in Section 7.1 provides the full breakdown by DBMS, bug status, and bug type. Individual bug examples are shown in Listings 6-10.",
       "supported": "strong"
     },
     {
       "claim": "Argus achieves up to 5.47x higher metamorphic coverage than SQLancer on DuckDB.",
       "evidence": "Table 3 reports average metamorphic coverage across 10 test suites: Argus 17.820% vs SQLancer 3.256% for lines (5.473x), 7.910% vs 1.230% for functions (6.431x), 7.315% vs 1.313% for branches (5.571x).",
       "supported": "moderate"
     },
     {
       "claim": "Argus's new oracles detect 3.33x more unique logic bugs than the union of prior oracles within 6 hours.",
       "evidence": "Figure 4 shows Argus-5000 finding 10 unique bugs vs the baseline's 3 bugs on Dolt v1.0.0 over 6 hours. Unique bugs were verified using git-bisect to link to fix commits.",
       "supported": "moderate"
     },
     {
       "claim": "The SQL equivalence prover is necessary to eliminate false positives — all 20 bug reports from the LLM-as-a-judge approach were false positives.",
       "evidence": "Section 7.4 describes the ablation: 'we performed an ablation that disables the prover while using an LLM-as-a-judge approach, and ran Argus on DuckDB until collecting 20 bug reports. We then manually adjudicated each report... all of them were false positives.'",
       "supported": "strong"
     },
     {
       "claim": "Argus is cost-effective: generating CAQ pairs costs ~$3 in LLM calls and instantiating test cases costs ~$1 per 1,000.",
       "evidence": "Section 7.5 and Figure 5 provide cost breakdowns for a 1-hour run on Dolt, showing the two-phase approach with specific dollar amounts.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Argus automates test oracle discovery for DBMS testing by combining LLM-generated Constrained Abstract Query (CAQ) pairs with formal SQL equivalence verification. Evaluated on five widely-used DBMSs (Dolt, DuckDB, MySQL, PostgreSQL, TiDB), it discovered 40 previously unknown bugs including 35 logic bugs, 36 of which were confirmed by developers. The approach achieves up to 5.47x higher metamorphic coverage than state-of-the-art tools and demonstrates that the formal prover is essential, as an LLM-as-a-judge approach produced 100% false positive bug reports in comparison.",
   "red_flags": [
     {
       "flag": "No code or data release",
       "detail": "Despite describing a complete system (Argus) with detailed algorithms, no source code, generated oracles, snippet corpus, or bug report dataset is released, making independent reproduction impossible."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "Coverage and bug-finding comparisons report only point estimates without confidence intervals, error bars, or significance tests. The metamorphic coverage experiment averages 10 test suites without reporting variance. Single-run coverage experiments do not account for randomness in the generation process."
     },
     {
       "flag": "Small sample in key ablation",
       "detail": "The prover vs LLM-as-a-judge comparison (Section 7.4) uses only 20 bug reports and 10 rejected pairs per configuration. These small samples support strong conclusions about false positive/negative rates without statistical justification."
     },
     {
       "flag": "Prompts and hyperparameters not provided",
       "detail": "The actual prompts used for LLM-based CAQ generation and snippet synthesis are not provided, only described qualitatively. No LLM hyperparameters (temperature, top-p) are reported, despite these significantly affecting generation quality."
     }
   ],
   "cited_papers": [
     {
       "title": "Fuzz4all: Universal fuzzing with large language models",
       "authors": ["Chunqiu Steven Xia", "Matteo Paltenghi", "Jia Le Tian", "Michael Pradel", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Demonstrates LLM-based universal fuzzing for automated testing, directly relevant to LLM-aided software testing methodology."
     },
     {
       "title": "Whitefox: White-box compiler fuzzing empowered by large language models",
       "authors": ["Chenyuan Yang", "Yinlin Deng", "Runyu Lu", "Jiayi Yao", "Jiawei Liu", "Reyhaneh Jabbarvand", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Uses LLMs for compiler testing with white-box techniques, relevant to LLM-aided testing methodology."
     },
     {
       "title": "KernelGPT: Enhanced kernel fuzzing via large language models",
       "authors": ["Chenyuan Yang", "Zijie Zhao", "Lingming Zhang"],
       "year": 2025,
       "relevance": "Applies LLMs to kernel fuzzing, relevant to understanding LLM capabilities in automated testing domains."
     },
     {
       "title": "Automated program repair via conversation: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Evaluates LLM-based program repair with cost analysis, relevant to LLM-aided software engineering productivity."
     },
     {
       "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery",
       "authors": ["Alexander Novikov"],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "LLM-based agent for algorithm discovery, relevant to agentic AI capabilities in code generation."
     },
     {
       "title": "Enhancing static analysis for practical bug detection: An LLM-integrated approach",
       "authors": ["Haonan Li", "Yu Hao", "Yizhuo Zhai", "Zhiyun Qian"],
       "year": 2024,
       "relevance": "Integrates LLMs with static analysis for bug detection, relevant to LLM-aided software quality."
     },
     {
       "title": "Testing Database Systems with Large Language Model Synthesized Fragments",
       "authors": ["Suyang Zhong", "Manuel Rigger"],
       "year": 2025,
       "arxiv_id": "2505.02012",
       "relevance": "Uses LLMs to synthesize SQL fragments for DBMS testing, directly complementary to Argus's approach."
     },
     {
       "title": "Can large language models discover metamorphic relations? A large-scale empirical study",
       "authors": ["Jiaming Zhang", "Chang-ai Sun", "Huai Liu", "Sijin Dong"],
       "year": 2025,
       "relevance": "Empirically evaluates LLM capability in discovering metamorphic relations for testing, directly related to Argus's test oracle discovery."
     },
     {
       "title": "LLM-SQL-Solver: Can LLMs determine SQL equivalence?",
       "authors": ["Fuheng Zhao", "Lawrence Lim", "Ishtiyaque Ahmad", "Divyakant Agrawal", "Amr El Abbadi"],
       "year": 2023,
       "arxiv_id": "2312.10321",
       "relevance": "Evaluates LLM capability in SQL equivalence determination, the core verification task in Argus's pipeline."
     },
     {
       "title": "S*: Test time scaling for code generation",
       "authors": ["Dacheng Li"],
       "year": 2025,
       "arxiv_id": "2502.14382",
       "relevance": "Explores test-time scaling for LLM code generation, relevant to LLM-aided programming capabilities."
     }
   ]
 }