ai-research-survey

scan.json (28639B)

---

 {
   "paper": {
     "title": "eSapiens: A Platform for Secure and Auditable Retrieval-Augmented Generation",
     "authors": [
       "Isaac Shi",
       "Zeyuan Li",
       "Fan Liu",
       "Wenli Wang",
       "Lewei He",
       "Yang Yang",
       "Tianyu Shi"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2507.09588",
     "doi": "10.48550/arXiv.2507.09588"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "eSapiens is a commercial RAG platform evaluated on legal QA retrieval (four LegalBench subsets) and generation quality (TRACe metrics across five LLMs). The retrieval experiment finds chunk size 1000 generally outperforms 500 for recall, while the generation experiment shows eSapiens improves Context Relevance over a FAISS baseline but at the cost of higher hallucination rates and lower completeness. The paper's own data shows the FAISS baseline outperforms eSapiens on 3 of 5 TRACe metrics, undermining the paper's positive framing.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository is provided. The paper links to https://www.esapiens.ai/ which is the commercial product website, not open-source code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available datasets: four LegalBench subsets (PrivacyQA, CUAD, MAUD, ContractNLI) listed in Table 2, and RAGtruth for the generation quality evaluation (Table 5)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions technology components (Elasticsearch 8.x, LangChain, FastAPI, OpenAIEmbeddings) in Section 6 but provides no requirements.txt, Dockerfile, or versioned dependency list sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The evaluation setup is described at a high level but there are no steps, scripts, or commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3, 4, and 5 report only point estimates. No confidence intervals, error bars, or uncertainty measures appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims eSapiens 'outperforms' the FAISS baseline and shows 'up to 23% improvement' but no statistical significance tests are applied to any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract mentions 'up to 23% improvement in factual alignment' but this is cherry-picked from the best-case model (GPT-4o-mini Context Relevance). No systematic effect size reporting across models or metrics. For DeepSeek R1, eSapiens Context Relevance is actually 24.8% worse than FAISS."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The generation quality experiment uses '100 random questions from RAGtruth' (Table 5 caption). No justification is given for why 100 is sufficient. The retrieval benchmark uses existing dataset sizes (712–3950 Q-A pairs) without power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B compares eSapiens against a 'baseline FAISS-based DEREK pipeline' across five LLMs on TRACe metrics (Table 5). Appendix A compares two chunk sizes."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The sole baseline is the authors' own simpler FAISS-based pipeline ('faiss+top-2+short'). No comparison against contemporary external RAG systems, frameworks, or academic baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (DEREK, THOR, hybrid retrieval, reranking, CO-STAR prompting, LangGraph validation) but no ablation study isolates their individual contributions. The chunk size comparison (500 vs 1000) tests only one parameter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The retrieval experiment reports Recall@k and Precision@k at multiple k values (Tables 3–4). The generation experiment reports five TRACe metrics: Completeness, Utilization, Context Relevance, pc_hallucinated, and Accuracy (Table 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The Accuracy metric in Table 5 is described as 'Human-graded alignment with ground truth' (Section B.1), indicating human evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of train/validation/test splits. The chunk size comparison (500 vs 1000) appears to be evaluated directly on the test data with no held-out validation set for parameter selection."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Retrieval results are broken down per dataset (PrivacyQA, CUAD, MAUD, ContractNLI) in Tables 3–4. Generation results are broken down per model in Table 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B discusses where eSapiens underperforms: higher hallucination rates than FAISS ('FAISS baseline consistently achieves lower hallucination rates'), lower completeness ('FAISS baseline yields higher completeness scores overall'), and explains why (e.g., 'eSapiens sometimes applies structural formatting or abstraction, leading to partial omission')."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 and the analysis in Appendix B honestly report that FAISS outperforms eSapiens on hallucination and completeness metrics. The paper notes the baseline 'yields stricter factual alignment with fewer hallucinations.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'chunk size of 512 tokens yields the highest retrieval precision (Top-3 accuracy: 91.3%)' but the experimental tables show chunk sizes of 500 and 1000 (not 512), and the number 91.3% does not appear in any results table. This is a significant discrepancy between abstract and presented data."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims eSapiens 'delivers more context-consistent outputs with up to 23% improvement in factual alignment.' This implies a causal effect of the platform, but eSapiens and the FAISS baseline differ in multiple ways (retrieval strategy, reranking, prompting) with no controlled isolation of which component drives the difference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims effectiveness 'for high-stakes domains like legal and finance' but tests only on legal QA datasets and RAGtruth. Finance, healthcare, and insurance are invoked repeatedly (Sections 1, 2, 3, 4) with no supporting evaluation. The title claims 'Secure and Auditable' but security is not experimentally tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed results. For example, the Context Relevance improvement could be due to the hybrid retrieval rather than the full platform, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The abstract calls Context Relevance improvements 'factual alignment' and frames retrieval metrics as evidence of 'trustworthy, auditable AI workflows.' The gap between the proxy (TRACe metrics on 100 questions) and the claimed outcome (trustworthy enterprise workflows) is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 lists 'GPT-4o mini, GPT-4o, GPT-4.1, Claude 3.7 Sonnet, Gemini 1.5 Pro, DeepSeek-R1 and V3.' These are marketing names without specific API versions or snapshot dates (e.g., no 'gpt-4o-2024-05-13')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.6 mentions using the 'CO-STAR prompt format' and that GPT-4o rewrites queries, but the actual prompt text is never provided. Appendix C shows generated SQL queries but not the prompts used to generate them."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Chunk sizes (500/1000 tokens) and overlap (150 tokens) are stated. However, no LLM hyperparameters are reported: temperature, top-p, max tokens, or any generation parameters for the five models tested."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 5.6–5.7 and 7.2–7.4 describe the multi-agent architecture in detail: supervisor agent routing, DEREK retrieval pipeline (query refinement → hybrid retrieval → answer generation → LangGraph validation), THOR SQL generation pipeline (orchestration → SQL generation → self-correction → result interpretation), with workflow diagrams in Figures 1 and 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Document chunking parameters are stated (Section 5.6: 1000 tokens, 150 overlap), but no detail on how the evaluation datasets were preprocessed: how the '100 random questions' were selected from RAGtruth, how the LegalBench corpora were ingested and indexed, or any filtering steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations, threats-to-validity, or discussion section in the paper. The paper ends with Appendix C (SQL examples) with no reflection on limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper broadly claims applicability to 'finance, healthcare, legal services' and 'high-stakes domains' without bounding what the evaluation does and does not show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data is released. The evaluation datasets are public benchmarks, but the eSapiens outputs, human evaluation scores, and retrieval results are not made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 lists the four LegalBench datasets with Q-A pair counts, and Table 5 mentions '100 random questions from RAGtruth.' However, no detail is given on how random selection was performed, what subset of RAGtruth was used, or how human grading for the Accuracy metric was conducted."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The Accuracy metric is described as 'Human-graded alignment with ground truth' but the paper does not describe who the human evaluators were, how many there were, or how they were recruited."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the data pipeline from benchmark datasets to final results. The paper does not explain how documents were indexed, how retrieval was performed at evaluation time, or how the TRACe scores were computed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, acknowledgments section, or grant information appears anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed under 'eSapiens Team' with the company URL https://www.esapiens.ai/. The affiliation with the evaluated product is clearly visible."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The authors are employees of the company whose product is being evaluated. The eSapiens company has a direct financial interest in positive evaluation results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The authors clearly have financial interests (as employees/founders of eSapiens) but this is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five LLMs used. While the evaluation is RAG-based, the models could have memorized benchmark answers."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LegalBench or RAGtruth benchmark content appeared in any model's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LegalBench and its subsets (CUAD, MAUD, ContractNLI, PrivacyQA) are publicly available benchmarks that predate the models' training. No contamination risk is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects. The human grading of Accuracy is an evaluation methodology, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported for any experiment despite using five commercial LLM APIs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications are stated for the evaluation experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated for any experiment."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Only two chunk sizes (500, 1000) are compared. No search budget, search method, or other configurations tried are reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper recommends chunk=1000 for production use based on test set performance without a separate validation set or justification for the selection process."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple models and metrics are compared across two pipelines but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The eSapiens team evaluates their own platform against their own FAISS baseline. No acknowledgment of self-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "eSapiens uses hybrid retrieval with reranking while the FAISS baseline uses simpler top-2 retrieval. The compute cost difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LegalBench subsets or RAGtruth adequately measure what eSapiens claims to deliver (trustworthy, auditable enterprise workflows). The benchmarks test retrieval/generation quality but the claims extend far beyond."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "eSapiens uses hybrid retrieval + reranking + CO-STAR prompting while the FAISS baseline uses simple top-2 retrieval with a restrictive prompt. Multiple scaffold differences are confounded but the performance difference is attributed to 'eSapiens' as a whole."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs' training data includes the benchmark content. LegalBench subsets (CUAD from 2021, ContractNLI from 2021) predate most models tested."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The RAG pipeline provides retrieved context that may contain answer-adjacent information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap or similarity between questions used for tuning the retrieval pipeline and the 100 test questions from RAGtruth."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Chunk size of 512 tokens yields the highest retrieval precision with Top-3 accuracy of 91.3%",
       "evidence": "Abstract states this claim, but experimental tables (Tables 3 and 4) show chunk sizes of 500 and 1000 — not 512. The number 91.3% does not appear in any results table.",
       "supported": "unsupported"
     },
     {
       "claim": "eSapiens delivers up to 23% improvement in factual alignment compared to FAISS baseline",
       "evidence": "Table 5 Context Relevance: eSapiens-gpt4o-mini (0.3785) vs faiss-gpt4o-mini (0.3090) = ~22.5% improvement. However, this is the best-case model. For DeepSeek R1, eSapiens is 24.8% worse (0.2581 vs 0.3430). Average improvement across models is much smaller.",
       "supported": "weak"
     },
     {
       "claim": "eSapiens outperforms traditional FAISS-based methods in context relevance, utilization, accuracy, and naturalness",
       "evidence": "Table 5 shows eSapiens wins on Context Relevance for 4/5 models and Accuracy for 4/5 models. However, FAISS wins on Completeness (5/5 models), Hallucination rate (5/5 models), and Utilization (3/5 models). The paper's own data contradicts its framing of general superiority.",
       "supported": "weak"
     },
     {
       "claim": "Chunk size 1000 is better aligned with production needs, offering broader coverage and lower information loss",
       "evidence": "Tables 3–4 and Appendix A analysis show chunk=1000 outperforms 500 on recall for CUAD, MAUD, and ContractNLI at high k values, while chunk=500 is better for PrivacyQA. Reasonable conclusion for 3/4 datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Monthly financial reporting time fell from two hours to twelve minutes",
       "evidence": "Section 1 (Executive Summary) states this as an early adopter report. No methodology, sample size, measurement procedure, or data is provided.",
       "supported": "unsupported"
     },
     {
       "claim": "Development costs slashed by up to 60 percent",
       "evidence": "Section 1 (Executive Summary) states this as a benefit. No data, methodology, or supporting evidence is provided.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All seven authors are from the 'eSapiens Team' and evaluate the eSapiens commercial platform. No independent evaluation or acknowledgment of self-evaluation bias. The paper functions partly as marketing material with extensive product description."
     },
     {
       "flag": "Abstract claims do not match presented data",
       "detail": "The abstract claims 'chunk size of 512 tokens yields the highest retrieval precision (Top-3 accuracy: 91.3%)' but the experimental tables show chunk sizes of 500 and 1000, and 91.3% appears nowhere in the results. This is a factual discrepancy between the abstract and the paper's own data."
     },
     {
       "flag": "Marketing claims without evidence",
       "detail": "The Executive Summary claims '60% cost reduction,' 'reporting time from 2 hours to 12 minutes,' '40% accuracy improvement in ticket categorization,' and 'double-digit' lead-to-deal velocity gains. None of these are supported by any presented data, methodology, or even citations."
     },
     {
       "flag": "Cherry-picked 'up to' improvements",
       "detail": "The '23% improvement in factual alignment' comes from the single best model (GPT-4o-mini on Context Relevance). For DeepSeek R1, eSapiens is actually 24.8% worse on the same metric. The 'up to' framing hides that the FAISS baseline outperforms eSapiens on 3 of 5 TRACe metrics across most models."
     },
     {
       "flag": "Selective framing of results",
       "detail": "The FAISS baseline achieves lower hallucination rates (consistently, across all 5 models) and higher completeness (all 5 models), yet the paper concludes eSapiens shows 'clear gains' and 'effectiveness in enabling trustworthy, auditable AI workflows.' The word 'trustworthy' is undercut by higher hallucination rates."
     },
     {
       "flag": "No error bars or statistical tests on any comparison",
       "detail": "All results are single-point estimates with no uncertainty quantification. Claims of superiority rest on comparing raw numbers without any significance testing, despite small margins on some metrics."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, discussion, or threats-to-validity section. It acknowledges no weaknesses in the evaluation design or scope of claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis"],
       "year": 2020,
       "relevance": "Foundational RAG paper establishing the retrieval-augmented generation paradigm that eSapiens builds upon."
     },
     {
       "title": "Distilling Knowledge from Reader to Retriever for Question Answering",
       "authors": ["Gautier Izacard", "Edouard Grave"],
       "year": 2021,
       "relevance": "FiD architecture for tightly coupling retrievers with generators, cited as early RAG research prototype."
     },
     {
       "title": "LangChain: Language Models in Chains",
       "authors": ["Harrison Chase"],
       "year": 2022,
       "relevance": "LLM orchestration framework used as the backbone of eSapiens' prompt orchestration and agent workflows."
     },
     {
       "title": "LlamaIndex (GPT Index)",
       "authors": ["Jerry Liu"],
       "year": 2022,
       "relevance": "Modular toolchain for building LLM-powered applications, cited as comparable developer-centric framework."
     },
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "authors": ["Chinmay H. Patil"],
       "year": 2023,
       "arxiv_id": "2305.15334",
       "relevance": "Demonstrates autonomous agent behaviors and tool use capabilities relevant to LLM-based agentic workflows."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Demonstrates LLM tool-use capabilities relevant to agentic AI systems and API-calling workflows."
     },
     {
       "title": "ChatLaw: Open-Source Legal Large Language Model Trained on Chinese Legal Documents",
       "authors": ["Xiaofei Gao"],
       "year": 2023,
       "arxiv_id": "2305.14251",
       "relevance": "Domain-specific legal LLM relevant to evaluating AI in high-stakes professional domains."
     },
     {
       "title": "Lawyer LLM: An Expert-Level Chinese Legal Large Language Model",
       "authors": ["Weicheng Fan"],
       "year": 2023,
       "arxiv_id": "2310.10472",
       "relevance": "Another domain-specific legal AI system, relevant for understanding vertical AI evaluation in professional settings."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Enterprise RAG platform with clear practical use cases (legal QA, SQL generation), but commercial product — not open-source or freely usable."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms standard expectations about RAG improving LLM outputs with no surprising or contrarian findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or security concerns raised; the paper emphasizes security as a selling point rather than exposing vulnerabilities."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward product evaluation paper."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Commercial website exists (esapiens.ai) with demos mentioned, but no open-source code or free trial evident from the paper."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "eSapiens is an unknown startup; none of the authors are widely recognized in the AI research community."
     }
   }
 }