ai-research-survey

scan.json (24825B)

---

 {
   "paper": {
     "title": "Traceable Latent Variable Discovery Based on Multi-Agent Collaboration",
     "authors": ["Huaming Du", "Tao Hu", "Yijie Huang", "Yu Zhao", "Guisong Liu", "Tao Gu", "Gang Kou", "Carl Yang"],
     "year": 2026,
     "venue": "WWW '26",
     "arxiv_id": "2602.14456",
     "doi": "10.1145/3774904.3792244"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is available at https://github.com/HYJ9999/TLVD.git as stated in Appendix A."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The WCHSU hospital datasets are not publicly available ('newly collected and has not been made publicly available online'). The two benchmark datasets are from prior work but the paper doesn't provide download links or its processed versions."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA GeForce A6000 GPU with 48GB of memory' but does not provide requirements.txt, Dockerfile, or detailed library version specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 3, 5, 6 all report results with ± notation (e.g., '0.833±0.000')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TLVD outperforms baselines but provides no statistical significance tests (no p-values, t-tests, etc.). Comparisons are based solely on comparing means."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '131.68% improvement over MiniMax' and 'average improvements of 32.67% in Acc, 62.21% in CAcc, and 26.72% in ECit' (Section 4.2, Abstract)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 5 runs with different random seeds were chosen, or why the specific dataset sizes were used."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across 5 runs in all main results tables (Tables 2, 3, 5, 6)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 lists 10 baselines across four categories: single LLMs (GPT-5), deep research agents, multi-agent platforms, and multi-LLM reasoning frameworks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-5, Gemini-deepresearch, OpenAI-deepresearch, WideSearch, and other recent systems from 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 presents ablation studies with 7 variants (TLVD-v, TLVD-d, TLVD-I, TLVD-R1, TLVD-R2, TLVD-R3) testing different components and reward signals."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: ACC, CAcc, and ECit (defined in Appendix A)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "For the WCHSU datasets lacking ground truth, 'five experts from relevant fields at Asia's largest hospital' performed consistency evaluation of inferred latent variables (Appendix A)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test split for the datasets. The paper evaluates on entire datasets with no mention of held-out test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (Tables 2, 3, 5), per model configuration (Table 4), and per data source (Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7.2 provides detailed failure attribution analysis using the MAST taxonomy across 100 execution traces, identifying three main failure categories (Figure 7)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 shows that performance declines when using more than 6 execution LLMs, and heterogeneous models perform worse than homogeneous ones (Table 4). Ablation variants show degraded performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '32.67% in Acc, 62.21% in CAcc, and 26.72% in ECit' improvements are supported by Tables 3 and 5 across five datasets."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by controlled ablation studies (Section 4.4) with single-variable manipulation. The paper also provides theoretical analysis (Section 3.2.3) for convergence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and framework are presented as general ('Traceable Latent Variable Discovery') but results are only on 5 datasets (3 medical, 2 social science). No discussion of generalizability boundaries to other domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why TLVD outperforms baselines. For instance, the game-theoretic framework's advantages are asserted but not compared against simpler coordination mechanisms with equal computational budget."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "ACC is determined by expert consensus for WCHSU datasets, but the paper does not discuss the gap between expert agreement as a proxy and actual causal correctness. CAcc and ECit measure evidence retrieval quality, not causal validity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'GPT-5', 'LLaMA3.1 8B', 'LLaMA3.1 70B', 'Qwen2.5 7B/72B', 'GPT-oss-120B', 'DeepSeek-v3' but does not specify exact version snapshots or API dates for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes what queries are sent to the LLMs (e.g., Figure 5 shows a sample query) but does not provide the full system prompts, coordinator prompts, or executor prompt templates used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 reports: episodes=100, buffer size=32, optimizer=Adam, learning rate=0.001, discount factor=0.99, entity dimension=256, belief state dimension=128, MLP hidden size=256, λ_m=0.1."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent architecture is described in detail: coordinator LLM, executor LLMs, belief networks, mixing network, reward design (Section 3.2). Workflow diagrams in Figures 2-4."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes how WCHSU-Cancer (n=22) was curated by scoring variable correlation using LLMs with scores 0-5 and filtering >5, and WCHSU-Cancer (n=12) was screened by lung cancer specialists. De-identification and IRB approval mentioned."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section in the paper. The conclusion (Section 5) contains no limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state what the results do NOT show or what settings/domains are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "WCHSU datasets are not publicly available. The benchmark datasets are from prior work but the paper does not provide its processed versions."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes WCHSU-Cancer as coming from a hospital health management center with 200,000 participants for lung cancer screening, and WCHSU-Pain from the same hospital with 1,568 patients' perioperative data."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the expert evaluation, '5 experts from relevant fields' are mentioned but how they were recruited and selected is not described. For the hospital data, patient selection criteria are not detailed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw data to latent variable discovery is documented: variable selection (LLM scoring or expert screening), causal graph construction (RLCD), multi-agent inference, and web-based validation (Section 3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Acknowledgements) lists multiple funding sources including NSFC, Xiangjiang Laboratory, China Postdoctoral Science Foundation, and Sichuan Science and Technology Program."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Southwestern University of Finance and Economics, Hunan University of Technology and Business, Xiangjiang Laboratory, and Emory University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Chinese government agencies and academic foundations with no apparent financial stake in the specific outcomes of latent variable discovery performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-5, LLaMA3.1, Qwen2.5, and others but does not state their training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes 'the WCHSU dataset is newly collected and has not been made publicly available online, making it unlikely to have been used in training any existing LLMs, and therefore posing no risk of data leakage' (Section 4.1.1). For benchmark datasets, Appendix D acknowledges they 'might have been included in the training data of LLMs.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D explicitly acknowledges that benchmark datasets may be in LLM training data and argues the comparison is still fair since all baselines are also LLM-based."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Expert evaluation was for ground-truth construction, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "IRB approval is mentioned for the patient dataset usage, but there are no human participants in the study itself. The IRB covers data use, not human experimentation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 10 shows token consumption across different modules and sample sizes for the method and baselines."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA GeForce A6000 GPU with 48GB' but does not state total GPU hours, training time, or API costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1: 'we conduct five experiments with different random seeds and report the average performance.' Standard deviations reported in all tables."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 explicitly states 'five experiments with different random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported but no search budget (number of configurations tried, search method) is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how the reported hyperparameter configuration was selected or whether it was tuned on a validation set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their system against baselines without acknowledging potential bias from implementing/configuring the baselines themselves."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Token consumption is shown in Figure 10 but performance is not plotted as a function of compute budget. No matched-compute comparisons are made."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ACC (expert-consensus-based) actually measures causal correctness, or whether the benchmark datasets are valid proxies for real-world latent variable discovery."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TLVD uses a complex scaffold (belief networks, mixing network, coordinator) while baselines use their own different scaffolds. The scaffold contribution vs. the game-theoretic coordination is not isolated."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses temporal leakage for WCHSU by noting it is newly collected and not publicly available. For benchmark datasets, the fairness argument (all baselines are LLM-based) partially addresses this."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information to the LLMs beyond what would be available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark datasets' variables or structure could overlap with LLM training data in ways that create non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The WCHSU novelty argument is a prevention method, but no detection is applied to the benchmark datasets."
       }
     }
   },
   "claims": [
     {
       "claim": "TLVD achieves average improvements of 32.67% in Acc, 62.21% in CAcc, and 26.72% in ECit across five datasets compared to baselines.",
       "evidence": "Tables 2, 3, 5 show TLVD outperforming all baselines on WCHSU-Cancer, WCHSU-Pain, Multitasking Behaviour Study, and Teacher's Burnout Study datasets (Sections 4.2, Appendix D).",
       "supported": "moderate"
     },
     {
       "claim": "TLVD achieves performance improvements with fewer communication tokens compared to CAMEL, Multi-Agent Debate, and Multi-Agent Majority.",
       "evidence": "Figure 10 shows token consumption comparison. Section 4.2 attributes efficiency to the belief network design.",
       "supported": "moderate"
     },
     {
       "claim": "Heterogeneous execution LLMs perform worse than homogeneous ones due to increased difficulty in reaching BNE.",
       "evidence": "Table 4 shows homogeneous LLaMA3.1 70B achieving 0.833 ACC vs heterogeneous LLaMA3.1 70B + Qwen2.5 72B achieving 0.500 ACC on WCHSU-Cancer (n=12).",
       "supported": "moderate"
     },
     {
       "claim": "The multi-agent collaboration framework converges to Bayesian Nash Equilibrium with sublinear regret O(N√T/(1-γ)).",
       "evidence": "Theoretical proof in Section 3.2.3 and Appendices B-C, relying on Glicksberg's Fixed Point Theorem and standard stochastic approximation theory.",
       "supported": "moderate"
     },
     {
       "claim": "ArXiv data sources contribute more to TLVD performance than Wikipedia or databases for medical domains.",
       "evidence": "Figure 6 shows removing arXiv (W/O ARR) causes the largest performance drop (ACC from 0.80 to 0.25) on WCHSU-Pain.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TLVD integrates causal discovery algorithms with multi-LLM collaboration modeled as a Bayesian Nash Equilibrium game to discover latent variables and their semantics, then validates them via web evidence retrieval. On 5 datasets (3 medical, 2 social science), TLVD substantially outperforms single LLMs, deep research agents, and multi-agent baselines. The framework shows that game-theoretic coordination of LLM agents can improve causal reasoning compared to simpler multi-agent strategies like debate or majority voting. Failure analysis using MAST taxonomy reveals specification issues and task verification as primary failure modes.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundaries discussion despite making broad claims about a novel framework."
     },
     {
       "flag": "Expert-consensus ground truth for WCHSU",
       "detail": "ACC on the main WCHSU datasets is determined by 5 experts' consensus, but no inter-rater reliability metrics are reported, and expert selection process is not described. This makes the primary evaluation metric difficult to verify."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Despite comparing against 10 baselines with claims of superiority, no statistical tests are reported. Some reported standard deviations overlap between methods."
     },
     {
       "flag": "Scaffold confound",
       "detail": "TLVD uses a sophisticated scaffold (belief networks, mixing network, RL training) while baselines use their native implementations. Performance gains could be due to the scaffold rather than the game-theoretic principles claimed."
     },
     {
       "flag": "Suspiciously large improvements",
       "detail": "Claims of 131.68% and 124.30% improvement over some baselines, while many baselines score 0.000 on CAcc and ECit, suggest potential issues with baseline configuration or metric design rather than genuine methodological superiority."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM used as baseline and component in AI agent systems."
     },
     {
       "title": "DeepSeek-R1 incentivizes reasoning in LLMs through reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "relevance": "RL-based LLM reasoning improvement relevant to AI capability evaluation."
     },
     {
       "title": "A survey on hallucination in large language models",
       "authors": ["Lei Huang"],
       "year": 2025,
       "relevance": "Survey on LLM hallucination — key limitation addressed by multi-agent frameworks."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework for software engineering tasks."
     },
     {
       "title": "CAMEL: Communicative agents for mind exploration of large language model society",
       "authors": ["Guohao Li"],
       "year": 2023,
       "relevance": "Multi-agent LLM collaboration framework used as baseline."
     },
     {
       "title": "More Agents Is All You Need",
       "authors": ["Junyou Li"],
       "year": 2024,
       "relevance": "Study on scaling multi-agent LLM systems, used as baseline (Multi-Agent Majority)."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du"],
       "year": 2024,
       "relevance": "Multi-agent debate framework for LLM reasoning, used as baseline."
     },
     {
       "title": "Autogen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu"],
       "year": 2024,
       "relevance": "Multi-agent LLM platform used as baseline."
     },
     {
       "title": "From Debate to Equilibrium: Belief-Driven Multi-Agent LLM Reasoning via Bayesian Nash Equilibrium",
       "authors": ["Xie Yi"],
       "year": 2025,
       "relevance": "Game-theoretic multi-agent LLM reasoning framework that this paper builds upon."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Failure taxonomy for multi-agent LLM systems used in this paper's error analysis."
     },
     {
       "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
       "authors": ["Yujia Qin"],
       "year": 2024,
       "relevance": "LLM-based tool use and agent framework."
     },
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["Lei Wang"],
       "year": 2024,
       "relevance": "Survey of LLM-based autonomous agents relevant to agentic AI research."
     }
   ]
 }