ai-research-survey

scan.json (27244B)

---

 {
   "paper": {
     "title": "Featurized-Decomposition Join: Low-Cost Semantic Joins with Guarantees",
     "authors": [
       "Sepanta Zeighami",
       "Shreya Shankar",
       "Aditya Parameswaran"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.05399",
     "doi": "10.48550/arXiv.2512.05399"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "FDJ reduces semantic join costs by up to 10x over BARGAIN by automatically constructing featurized decompositions — logical expressions in CNF that extract and compare features from text records instead of relying on embedding similarity. The method provides statistical guarantees on precision and recall. The approach is most effective when join conditions can be decomposed into discrete features (e.g., names, dates, locations), but offers limited gains on classification-style joins where the mapping is complex. The paper also proves that finding the minimum-cost featurized decomposition is NP-hard and provides novel statistical results for multi-dimensional threshold selection.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Detailed prompts for all our LLM powered functions are available in our source code' (Appendix I) but provides no repository URL or archive link in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Most datasets used are publicly available: Products [33,42], BioDEX [14,46], Citations [38], Categorize [40,56], IMDB [23]. The paper references these public sources directly."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided. The paper mentions using OpenAI models but gives no environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating experiments are provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 3 and Figures 7-10 report point estimates only. No confidence intervals or error bars on cost ratios or recall values."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims FDJ 'improves over BARGAIN across all datasets' (Sec 8.2) based on comparing numbers in Table 3 without any statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports cost ratios with reduction factors in parentheses (e.g., '6.80 (0.24×)' for Citations, '6.70 (0.09×)' for Movies), providing both absolute and relative magnitudes with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper justifies system parameters (250 positive pairs, 50 for featurizations) but provides no power analysis or justification for the number of datasets or experimental runs in the evaluation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 shows 'Avg. % Failed' implying multiple runs, and Sec 8.1 mentions 'every run' for LOTUS. However, no standard deviations, IQR, or spread measures are reported for any results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against BARGAIN [65] (state-of-the-art model cascade), optimal cascade (oracle lower bound), and discusses LOTUS [46] (excluded for failing guarantees). See Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "BARGAIN [65] is from SIGMOD 2026 (to appear), LOTUS [46] from 2024. These represent the current state-of-the-art for semantic joins with guarantees."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Fig 9 provides cost breakdowns and Sec 8.3-8.4 study parameter sensitivity. However, no ablation removes or modifies individual FDJ components (e.g., iterative feature generation, CNF formulation, threshold selection) to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports cost ratio, observed recall, and failure rate (Table 2). Precision is guaranteed at 100% by design. Cost breakdowns are provided (Fig 9)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of join quality. All evaluation uses ground-truth labels: 'instead of actually performing the LLM call, we simulate it by returning the known ground-truth' (Sec 8.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For Products: 'we evaluate the join condition on the test set and use the training set to create the featurized decomposition' (Sec 8.1). For other datasets, FDJ uses internal samples S and S' (Alg. 6) with fresh samples for threshold setting, evaluating on the full cross product."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-dataset results. Sec 8.2 categorizes datasets into three types with analysis of each. Fig 9 shows per-component cost breakdowns. Sec 8.4 provides per-characteristic analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sec 8.2 discusses where FDJ provides limited gains: 'Categorize and BioDEX are classification tasks. In such datasets the join relationship may not be easily decomposable into a set of featurizations.' BioDEX shows only 0.99× improvement."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "BioDEX shows FDJ at 73.8 vs BARGAIN at 73.9 (0.99× — essentially no improvement). LOTUS is shown to fail meeting recall targets on every run (Table 2). Categorize shows only 0.83× improvement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'up to 10 times reduction in cost' — supported by Table 3 (Movies: 6.70 vs 69.9 = ~10x). 'providing the same quality guarantees' — supported by Theorem 7.1 and Table 2 empirical validation."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (FDJ reduces cost) are supported by controlled experimental comparison with baselines on 6 datasets, theoretical analysis with proofs (Theorems 4.2, 6.1, 7.1), and systematic analysis of data characteristics (Sec 8.4)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims: 'up to 10 times' (not always), discusses when gains are limited (classification tasks, Sec 8.2), and systematically analyzes factors affecting performance (number of attributes, text length, Sec 8.4)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper explains why FDJ works and why embeddings fail but does not consider alternative explanations for observed results, such as whether dataset-specific tuning or the choice of LLM/embedding model could explain the performance differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures cost ratio and recall/precision directly — these exactly match the claimed contributions (cost reduction with quality guarantees). No proxy gap between measurement and claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'GPT-4.1 for feature extraction and performing join, O3 for generating candidate featurizations; and text-embedding-large as the embedding model' (Sec 8.1) but provides no snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix I provides summaries of prompts for LLM functions, but the paper states 'our actual prompts include additional content for passing examples and defining output format and instruction' — the full prompts are only in the source code (no URL provided)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "System parameters are reported (250 positive pairs, γ=0.05, T_R=90%, T_P=100%, δ=10%, β=0) but LLM hyperparameters (temperature, top-p, max tokens) for GPT-4.1 and O3 are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. FDJ is an algorithmic pipeline, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sec 8.1 describes dataset preparation: subsampling procedures for BioDEX/Categorize/Citations ('sampled 20,000 ground-truth pairs'), Movies dataset creation ('crawling Wikipedia pages'), Products train/test setup, and synthetic data generation (Sec 8.4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The paper proceeds from experiments (Sec 8) to related work (Sec 9) to conclusion (Sec 10) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity discussion. No mention of threats specific to this study's design or evaluation methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While Sec 8.2 discusses dataset categories where FDJ is less effective, there are no explicit scope boundary statements."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Most source datasets are public, but the specific subsamples, extracted features, and experimental artifacts used in evaluation are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Each dataset's source and preparation is described in Sec 8.1: original references for public datasets, subsampling procedures, Movies dataset creation from IMDB/Wikipedia, and Police Records from the Police Records Access Project [1]."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and established datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents data preparation: subsampling 20,000 ground-truth pairs from larger datasets, creating join prompts (Appendix I), cost simulation methodology, and synthetic data generation process (Sec 8.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed as UC Berkeley affiliations with email addresses. They are not employees of OpenAI (whose models are used in evaluation)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper. Two authors (Shankar, Parameswaran) are co-authors of LOTUS [46], a baseline system."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a system (FDJ) for optimizing semantic joins, not a pre-trained model's capability on benchmarks. LLM calls are simulated using ground-truth labels."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's knowledge on benchmarks. The evaluation measures FDJ system performance using simulated LLM calls."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination is not applicable — the evaluation tests the FDJ algorithm's cost-quality tradeoff, not whether the LLM has seen the test data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost ratio is the primary metric throughout: Table 3 reports cost ratios for all datasets, Fig 9 shows cost breakdowns, and Sec 2 defines the cost model based on monetary cost of LLM/embedding tokens."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports relative cost ratios but does not state absolute compute budgets (total API spend, GPU hours). Prop E.1 gives cost complexity in terms of tokens but no actual numbers."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 3 are single-run numbers. The algorithm involves random sampling (Alg. 6, Lines 1 and 5) but sensitivity to randomness is not reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports 'Avg. % Failed' and the text mentions 'every run' for LOTUS, implying multiple runs. But the exact number of runs is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "System parameters (k, k', γ, β) are set to fixed values justified by reasoning, but no hyperparameter search was conducted and no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "System parameters are justified: '250 positive pairs' based on statistical significance needs, γ=0.05 'to avoid adding featurizations that have marginal difference' (Sec 6.2), '50 positive samples' based on empirical observation that LLMs stop generating new featurizations (Appx E)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (FDJ) and implement the baselines. Two authors (Shankar, Parameswaran) are co-authors of LOTUS [46], which is excluded from the main comparison. No discussion of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Fig 8 shows cost ratio vs target recall across datasets — performance as a function of quality target, which is the cost-quality tradeoff. Fig 9 breaks down costs. Methods are compared at matched quality targets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper categorizes datasets into three types (Sec 8.2) but does not formally discuss whether the benchmarks are representative of real-world semantic join workloads or whether the evaluation setup (simulated LLM calls) affects validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. FDJ is an algorithmic pipeline, not an agent-based system with scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LLM calls are simulated with ground truth (Sec 8.1), but the feature extraction and featurization generation steps use real GPT-4.1/O3 calls. No discussion of whether the LLM's training data could affect feature extraction quality on these public datasets."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The featurization generation process uses labeled samples from the same dataset being evaluated on, but the statistical framework is designed to account for this. No explicit discussion of feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Samples for construction (S, S') are drawn from the same cross product being evaluated on. While the statistical guarantees account for this, independence between construction and evaluation data is not explicitly discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. The paper relies on its statistical framework for guarantees but does not use canary strings, membership inference, or decontamination."
       }
     }
   },
   "claims": [
     {
       "claim": "FDJ provides up to 10x cost reduction over BARGAIN, the state-of-the-art model cascade method with statistical guarantees.",
       "evidence": "Table 3: On Citations dataset, FDJ costs 6.80 vs BARGAIN's 28.0 (0.24×). On Movies, 6.70 vs 69.9 (0.09×, approximately 10x). Average ~50% cost across all 6 datasets.",
       "supported": "strong"
     },
     {
       "claim": "FDJ provides up to 8x cost reduction over the optimal cascade, which represents a lower bound on any cascade-based approach.",
       "evidence": "Table 3: On Movies dataset, FDJ costs 6.70 vs optimal cascade's 52.5 (approximately 8x). Optimal cascade cheats by using all ground-truth pairs to set thresholds.",
       "supported": "strong"
     },
     {
       "claim": "Finding the minimum-cost featurized decomposition (MCFD) is NP-hard.",
       "evidence": "Theorem 4.2 with full proof in Appendix H.1 via reduction from Set Cover.",
       "supported": "strong"
     },
     {
       "claim": "FDJ provides statistical guarantees on recall: P(R(Y) < T) ≤ δ.",
       "evidence": "Theorem 7.1 with proof via Theorem 6.1 and Lemma 6.2. Empirically validated in Table 2: FDJ meets 90% recall target with only 7% failure rate (target δ=10%).",
       "supported": "strong"
     },
     {
       "claim": "Embedding-based approaches fail when text records contain multiple attribute values or irrelevant information.",
       "evidence": "Fig 10: Optimal cascade cost increases from ~0 to ~0.5 as number of persons mentioned grows from 1 to 5. Cost jumps from ~0.1 to ~0.4 when just 2 additional sentences are added to text.",
       "supported": "strong"
     },
     {
       "claim": "LOTUS fails to provide statistical guarantees in practice, missing the 90% recall target on every run.",
       "evidence": "Table 2: LOTUS has 100% failure rate and average recall of only 75.4% at T_R=90%. Limited to BioDEX dataset only.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars on main results",
       "detail": "Table 3, the central results table, reports single-point cost ratios with no uncertainty quantification. The FDJ algorithm involves random sampling (Alg 6 Lines 1, 5) which should produce variance across runs, but this is not reported."
     },
     {
       "flag": "LLM calls simulated with ground truth",
       "detail": "Sec 8.1: 'for every invocation of Lp, instead of actually performing the LLM call, we simulate it by returning the known ground-truth and calculate the cost by creating the prompt.' This means the evaluation does not capture real LLM failure modes (hallucinations, inconsistencies) that could affect FDJ's performance in practice."
     },
     {
       "flag": "Authors' prior work excluded from comparison",
       "detail": "Two authors (Shankar, Parameswaran) are co-authors of LOTUS [46], which is excluded from the main experiments. While the exclusion is justified (LOTUS fails guarantees per Table 2), this means the paper's proposed method is not compared against its authors' own prior system in the main evaluation."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section discussing threats to validity, boundary conditions, or potential failure modes beyond what is shown in the experiments."
     }
   ],
   "cited_papers": [
     {
       "title": "Lotus: Enabling semantic queries with llms over tables of unstructured and structured data",
       "authors": ["Liana Patel", "Siddharth Jha", "Carlos Guestrin", "Matei Zaharia"],
       "year": 2024,
       "arxiv_id": "2407.11418",
       "relevance": "Core baseline system for LLM-powered semantic queries including joins, directly compared against in this paper."
     },
     {
       "title": "Cut Costs, Not Accuracy: LLM-Powered Data Processing with Guarantees",
       "authors": ["Sepanta Zeighami", "Shreya Shankar", "Aditya Parameswaran"],
       "year": 2026,
       "relevance": "State-of-the-art model cascade method (BARGAIN) with statistical guarantees for LLM-powered data processing, primary baseline."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Proposes model cascade framework for cost-efficient LLM usage, foundational work for the cascade approach that FDJ builds upon."
     },
     {
       "title": "DocETL: Agentic Query Rewriting and Evaluation for Complex Document Processing",
       "authors": ["Shreya Shankar", "Tristan Chambers", "Tarak Shah", "Aditya G Parameswaran", "Eugene Wu"],
       "year": 2024,
       "arxiv_id": "2410.12189",
       "relevance": "Agentic system for processing unstructured documents with LLMs, supports semantic operations including joins."
     },
     {
       "title": "Approximate Selection with Guarantees using Proxies",
       "authors": ["Daniel Kang", "Edward Gan", "Peter Bailis", "Tatsunori Hashimoto", "Matei Zaharia"],
       "year": 2020,
       "relevance": "SUPG framework providing statistical guarantees for proxy-based selection, theoretical foundation that FDJ generalizes to higher dimensions."
     },
     {
       "title": "Can foundation models wrangle your data?",
       "authors": ["Avanika Narayan", "Ines Chami", "Laurel Orr", "Simran Arora", "Christopher Ré"],
       "year": 2022,
       "arxiv_id": "2205.09911",
       "relevance": "Evaluates foundation models for data wrangling tasks including entity matching, relevant to the LLM-powered data processing paradigm."
     },
     {
       "title": "ThriftLLM: On Cost-Effective Selection of Large Language Models for Classification Queries",
       "authors": ["Keke Huang", "Yimin Shi", "Dujian Ding"],
       "year": 2025,
       "arxiv_id": "2501.04901",
       "relevance": "Cost-effective LLM selection via ensembling, related approach to reducing LLM-powered data processing costs."
     },
     {
       "title": "SMART: Automatically Scaling Down Language Models with Accuracy Guarantees for Reduced Processing Fees",
       "authors": ["Saehan Jo", "Immanuel Trummer"],
       "year": 2024,
       "arxiv_id": "2403.13835",
       "relevance": "Model profiling for cost reduction with accuracy guarantees in LLM-powered data processing."
     },
     {
       "title": "LLM-Powered Proactive Data Systems",
       "authors": ["Sepanta Zeighami", "Yiming Lin", "Shreya Shankar", "Aditya Parameswaran"],
       "year": 2025,
       "arxiv_id": "2502.13016",
       "relevance": "Broader vision for LLM-powered data systems that includes semantic joins as a core operation."
     },
     {
       "title": "A declarative system for optimizing AI workloads",
       "authors": ["Chunwei Liu", "Matthew Russo", "Michael Cafarella"],
       "year": 2024,
       "arxiv_id": "2405.14696",
       "relevance": "System for optimizing LLM-powered data processing workflows, related approach to cost-efficient AI data management."
     }
   ]
 }