ai-research-survey

scan.json (22581B)

---

 {
   "paper": {
     "title": "Does It Tie Out? Towards Autonomous Legal Agents in Venture Capital",
     "authors": ["Pierre Colombo", "Malik Boudiaf", "Allyn Sweet", "Michael Desa", "Hongxi Wang", "Kevin Candra", "Syméon del Marmol"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.18658",
     "doi": "10.48550/arXiv.2512.18658"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper formalizes capitalization table tie-out as a verification problem and evaluates three approaches on 4 anonymized datarooms (Seed to Series B). The proposed Equall system (eager world model construction via an Event Graph) achieves 85.1% F1 vs 42.1% for agentic+structured and 29.0% for pure agentic baselines. Equall shows a 22x per-check speed advantage over agentic approaches due to amortized reasoning costs. The paper also characterizes empirical complexity scaling: verification steps nearly triple from Seed (~2,700) to Series B (~8,000).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The four datarooms are anonymized and proprietary. No dataset download link or supplementary data is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions or scripts are provided. The system is proprietary."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Figure 11 shows 95% error bars for time comparisons, but the main F1 results in Figure 8 have no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims like 'significantly outperforming' (Section 5.1) are based on raw number comparisons without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute F1 scores and differences are reported with baseline context: Equall 85.1% vs agentic+structured 42.1% vs pure agentic 29.0%. Speed comparison provides concrete ratios (22x per check)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only 4 datarooms are used with no justification for this sample size and no acknowledgment that N=4 may be too small for generalizable conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for the F1 results. Each dataroom appears evaluated once."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: a pure agentic RAG baseline (GPT-5.1 with iterative RAG) and an ablation (agentic + Equall's structured representation)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The agentic baseline uses GPT-5.1, a contemporary model. However, no other commercial legal AI systems are compared."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The 'Agentic + Structured Repr.' baseline serves as an ablation, isolating the effect of Equall's Event Graph (Stage 2) from the full pipeline."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Precision, recall, and F1 are reported per flag category (Figure 8). Speed/latency comparisons are also provided (Figure 9)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Ground-truth flags were annotated by experienced legal professionals (Section 5). The evaluation compares system output against expert human annotations."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "With only 4 datarooms, there is no held-out test set. All datarooms appear to be used for both development and evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 breaks down precision, recall, and F1 across four flag categories: Data Discrepancy, Issuance Missing, Board Approval Missing, and Cap Table Missing."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses where agentic baselines fail: 'Missing Documentation' and 'Missing from Cap Table' categories requiring global reasoning. Equall's limitations on these categories are also visible in results."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results for Equall are reported. Every experiment shows Equall outperforming baselines. No failed approaches or configurations are mentioned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that existing agentic systems fail at tie-out and proposes a world model architecture. The results (29% F1 for agentic vs 85% for Equall) support these claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about why Equall outperforms ('pre-computed knowledge graph turns these complex reasoning chains into reliable graph queries') but the study design (4 datarooms, no controlled manipulation of individual components beyond one ablation) is insufficient for strong causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Venture Capital' broadly but results are from only 4 US-focused datarooms (Seed to Series B). No discussion of whether results generalize to other jurisdictions, deal types, or company sizes."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The Equall system may benefit from domain-specific engineering advantages not shared with the generic agentic baseline, but this is not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures F1 on anomaly detection against expert annotations and frames this as 'tie-out automation,' but does not discuss whether matching expert flags is the same as producing legally reliable tie-out (experts may disagree, annotations may be incomplete)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The agentic baseline uses 'GPT-5.1' but no snapshot date or API version is given. The LLM used inside Equall's extraction pipeline is not specified at all."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts or system instructions are provided for any of the LLM components (agentic baseline, Equall's extractors, or classifiers)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, chunk size for RAG, etc.) are reported for any system."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes both paradigms in detail: the agentic RAG pipeline (query generation → retrieval → reasoning → verification) and Equall's three-stage pipeline (foundational extraction → inductive event modeling → neuro-symbolic verification)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how raw dataroom documents were preprocessed (OCR, text extraction, chunking for RAG, etc.). Section 3 mentions OCR quality issues but does not describe the actual preprocessing pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations or threats-to-validity section. The paper moves directly from results to conclusion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries. The paper does not state what the results do NOT show or what settings were excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The datarooms are proprietary and anonymized. No supplementary data files are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the four datarooms with statistics (pages, documents, securities, shareholders) across financing stages. The dataroom composition is characterized by document type distribution (Figures 2, 5)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the four companies/datarooms were selected. Were they Equall customers? Were they chosen to be representative? Selection criteria are absent."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw documents to ground-truth annotations is not documented. How legal experts annotated ground-truth flags, inter-annotator agreement, and annotation guidelines are all absent."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list Equall email addresses ({firstname}@equall.com), making the company affiliation clear."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are from Equall, the company whose product is being evaluated. Equall has a direct financial interest in the system appearing to outperform alternatives."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. The authors presumably hold equity or employment stakes in Equall but this is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff is stated for GPT-5.1 or any other model used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models could have seen similar legal documents during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis. Although the datarooms are proprietary (reducing contamination risk), this is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates automated systems against expert-annotated ground truth."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 reports per-check inference time (45 sec agentic vs 2 sec Equall) and total times for 100 and 500 checks. Figure 11 reports end-to-end time for Equall-assisted workflows (64m to 300m)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, API costs, total spend) is stated. Only relative timing comparisons are provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple-seed experiments are reported. Each system appears to be run once per dataroom."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for any system."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how configurations were selected for any of the three systems."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own product (Equall) against baselines they implemented. No acknowledgment of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 explicitly compares speed (a compute proxy) vs. paradigm, showing the trade-off between eager (15 min indexing, 2 sec/check) and lazy (2 min indexing, 45 sec/check) approaches."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether matching expert-annotated flags constitutes valid measurement of tie-out quality. Inter-annotator agreement is not reported."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The agentic baseline uses a generic RAG scaffold while Equall uses a highly engineered domain-specific pipeline. The performance difference could be due to engineering quality rather than architectural paradigm, but this confound is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-5.1 may have seen similar legal documents or patterns in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "With only 4 datarooms, independence between evaluation instances is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is described."
       }
     }
   },
   "claims": [
     {
       "claim": "Equall achieves 85.1% average F1 on flag detection, significantly outperforming agentic+structured (42.1%) and pure agentic (29.0%) baselines.",
       "evidence": "Figure 8, Section 5.1. Precision, recall, and F1 broken down by flag category across three systems.",
       "supported": "moderate"
     },
     {
       "claim": "Equall provides a 22x speed advantage per verification check over the agentic baseline (2 sec vs 45 sec).",
       "evidence": "Figure 9, Section 5.1. Speed comparison table on a 300-document dataroom.",
       "supported": "moderate"
     },
     {
       "claim": "The evidentiary burden in tie-out scales super-linearly with document volume, with securities increasing 7x while documents only double from Seed to Series B.",
       "evidence": "Figure 4, Section 3. Statistics across four companies at different financing stages.",
       "supported": "weak"
     },
     {
       "claim": "Verification workload nearly triples from ~2,700 steps at Seed to ~8,000 at Series B.",
       "evidence": "Figure 7, Section 3. Verification step counts across four companies.",
       "supported": "weak"
     },
     {
       "claim": "Equall-assisted workflow reduces tie-out time from 5-27 hours (manual) to 64-300 minutes.",
       "evidence": "Figure 11. Time comparison with 95% intervals from customer/partner data vs internal benchmarks.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are Equall employees evaluating Equall's product against baselines they implemented. No independent evaluation or third-party replication. This is the classic self-evaluation bias documented by Lucic et al."
     },
     {
       "flag": "Tiny sample size (N=4)",
       "detail": "Only 4 datarooms are used for evaluation. This is far too small to draw generalizable conclusions about architectural paradigm superiority. No sample size justification is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or discussion of what the results do NOT show. This is a significant omission for a paper making strong architectural claims."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "The agentic baseline is a generic RAG system, while Equall is a heavily engineered domain-specific pipeline with specialized parsers and a custom knowledge graph. The comparison conflates architectural paradigm with engineering investment."
     },
     {
       "flag": "No statistical rigor",
       "detail": "Claims of 'significant' outperformance are made without any statistical tests, confidence intervals (for F1), or multi-run experiments on N=4 datarooms."
     },
     {
       "flag": "Proprietary and unreproducible",
       "detail": "No code, data, prompts, hyperparameters, or ground-truth annotations are released. The entire evaluation is unverifiable."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Directly relevant to understanding failure modes of agentic LLM systems, which this paper claims to address."
     },
     {
       "title": "Evaluating AI for Law: Bridging the Gap with Open-Source Solutions",
       "authors": ["Rohan Bhambhoria", "Samuel Dahan", "Jonathan Li", "Xiaodan Zhu"],
       "year": 2024,
       "arxiv_id": "2404.12349",
       "relevance": "Evaluates AI capabilities for legal tasks, relevant to understanding LLM performance on legal reasoning."
     },
     {
       "title": "LegalBench: A Collaboratively Built Benchmark for Measuring Legal Reasoning in Large Language Models",
       "authors": ["Neel Guha", "Julian Nyarko", "Daniel E Ho"],
       "year": 2023,
       "arxiv_id": "2308.11462",
       "relevance": "Major legal AI benchmark that this paper positions against as insufficient for real-world legal workflows."
     },
     {
       "title": "SaulLM-54B & SaulLM-141B: Scaling Up Domain Adaptation for the Legal Domain",
       "authors": ["Pierre Colombo", "Telmo Pires", "Malik Boudiaf"],
       "year": 2024,
       "relevance": "Domain-adapted legal LLM by the same group, relevant to understanding domain-specific LLM development."
     },
     {
       "title": "GPT-4 Passes the Bar Exam",
       "authors": ["Daniel Martin Katz", "Michael James Bommarito", "Shang Gao", "Pablo Arredondo"],
       "year": 2023,
       "relevance": "Landmark evaluation of LLM legal reasoning capability on professional bar examination."
     },
     {
       "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models",
       "authors": ["Zhihong Shao", "Peiyi Wang", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Referenced for RL-driven training methodology that the paper proposes applying to legal verification."
     }
   ]
 }