ai-research-survey

scan.json (19994B)

---

 {
   "paper": {
     "title": "Co-Sight: Enhancing LLM-Based Agents via Conflict-Aware Meta-Verification and Trustworthy Reasoning with Structured Facts",
     "authors": ["Hongwei Zhang", "Ji Lu", "Shiqing Jiang", "Chenxiang Zhu", "Li Xie", "Chen Zhong", "Haoran Chen", "Yurui Zhu", "Yongsheng Du", "Yanqin Gao", "Lingjun Huang", "Baoli Wang", "Fang Tan", "Peng Zou"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.21557"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Code available here' (footnote 1) but no actual URL is provided in the extracted text. The link appears to be a placeholder or broken reference."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: GAIA, HLE, and Chinese-SimpleQA, all with citations to their public sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or step-by-step guide is provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (e.g., 84.4%, 35.5%, 93.8%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Co-Sight outperforms baselines (e.g., 'exceeding the next best system by 1.0%') but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '+8.6% over the baseline' in Table 2, and '13.4% over the backbone model' for HLE, giving readers enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the benchmark sizes used. GAIA has 300 questions and Chinese-SimpleQA has 3,000, but there is no discussion of whether these are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run statistics are reported. It is unclear whether experiments were run multiple times."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: Skywork Deep Research v2, AWorld, Su Zero Ultra on GAIA; Tongyi DeepResearch, Kimi-Researcher, Gemini Deep Research, OpenAI Deep Research on HLE; and majority voting, simple verification, oracle pass@N on Chinese-SimpleQA."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include 2024-2025 systems such as Skywork Deep Research v2, OpenAI Deep Research, and Gemini Deep Research, which are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides a detailed component ablation on Chinese-SimpleQA with N=2 experts, isolating CAMV, TRSF, and SV contributions. Table 1 varies ensemble size."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is reported across all benchmarks. No secondary metrics (e.g., cost, latency, F1) are used for evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluations are automated benchmark accuracy scores. Given claims about 'trustworthy' and 'auditable' reasoning, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "GAIA results are reported on the official test set via the HuggingFace leaderboard. HLE is also a separate test benchmark."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "GAIA results are broken down by difficulty level (Level 1, 2, 3) in Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is presented. The paper does not show examples where Co-Sight fails or discuss failure modes beyond the brief limitations section."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows that CAMV underperforms oracle pass@N at larger ensemble sizes (N>=3), which is honestly reported and explained as a budget limitation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims state-of-the-art on GAIA (84.4%) and HLE (35.5%), and strong results on Chinese-SimpleQA (93.8%). These are supported by Figures 3, 4, and Tables 1-2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'CAMV and TRSF jointly yield an overall gain of +8.6%' are supported by controlled ablation studies in Table 2 that manipulate single variables."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to enhance 'LLM-Based Agents' generally, but results are on three specific benchmarks with one backbone model (Gemini 2.5 Pro). The paper does not explicitly bound its claims to this setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the improvement could stem from the specific backbone model choice or the additional compute from running multiple expert agents, but these are not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Gemini 2.5 Pro' as the backbone but provides no snapshot date or API version. This is a marketing name without a precise version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper describes the system architecture and algorithms but does not include the prompts used for expert agents, the meta-verifier, or the planner/actor components."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters are missing: temperature settings for conservative vs radical agents, the consensus threshold θ, the budget Bmax, and sampling parameters are not specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: Section 3 covers the planner-actor-toolkit-facts architecture, DAG-based task decomposition, ReAct execution, the four-stage CAMV pipeline, and the three-tier TRSF context compression."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing steps are documented. It is unclear how benchmark inputs were prepared or whether any filtering was applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 'Limitations' provides a dedicated discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses specific limitations: incomplete expert plans can miss salient errors, multimodal pipeline bounded by vision/parsing module accuracy, and robustness in safety-critical settings not yet established."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Limitations mention safety-critical settings but do not bound claims to the tested benchmarks, backbone model, or task types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data, intermediate outputs, or detailed per-question results are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used are well-described with citations: GAIA (300 questions, 3 levels), HLE (100+ fields), Chinese-SimpleQA (3,000 questions, 6 domains)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; all data comes from standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from benchmark input to final score is not documented. How expert agent outputs are collected, how the meta-verifier processes them in practice, and how final scores are computed against ground truth are not described in operational detail."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Zhongxing Telecom Equipment (ZTE), China, which is clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Since all authors are from ZTE (a corporate entity), the funder's independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for Gemini 2.5 Pro is not stated, making it impossible to assess whether benchmark data was in the training set."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. GAIA was published in 2024 and Chinese-SimpleQA in 2024, both potentially within the training data of Gemini 2.5 Pro."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark contamination is not discussed despite using public benchmarks with a model that may have trained on them."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported despite the system using multiple expert agents and a meta-verification agent, which implies significant computational overhead."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API costs, or hardware specifications are stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Co-Sight achieves state-of-the-art accuracy of 84.4% on the GAIA test benchmark.",
       "evidence": "Figure 3 shows Co-Sight at 84.4% overall, exceeding Skywork Deep Research v2 (83.4%), AWorld (81.7%), and Su Zero Ultra (80.4%).",
       "supported": "moderate"
     },
     {
       "claim": "Co-Sight achieves 35.5% on the HLE benchmark, outperforming all competitors.",
       "evidence": "Figure 4 shows Co-Sight at 35.5% vs Tongyi DeepResearch (32.9%) and others. The improvement is 2.6% over the strongest competitor.",
       "supported": "moderate"
     },
     {
       "claim": "The synergy between CAMV and TRSF produces an 8.6% gain over the baseline on Chinese-SimpleQA.",
       "evidence": "Table 2 shows the full Co-Sight (CAMV+TRSF) at 91.2% vs baseline 82.6%, with controlled ablations isolating each component's contribution.",
       "supported": "strong"
     },
     {
       "claim": "CAMV outperforms oracle pass@N for small ensemble sizes (N<=2).",
       "evidence": "Table 1 shows CAMV at 88.3% vs pass@N at 85.0% for N=1, and 91.2% vs 90.0% for N=2.",
       "supported": "moderate"
     },
     {
       "claim": "Co-Sight provides a scalable paradigm for reliable long-horizon reasoning.",
       "evidence": "Scalability claim is supported only by benchmark results on three datasets. No analysis of computational scaling, latency, or cost is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Co-Sight introduces a conflict-aware meta-verification (CAMV) and structured facts (TRSF) framework for LLM-based multi-agent reasoning. It achieves state-of-the-art on GAIA (84.4%) and HLE (35.5%) benchmarks using Gemini 2.5 Pro as the backbone. Ablation studies on Chinese-SimpleQA demonstrate that both CAMV and TRSF contribute complementary gains, with the full system achieving +8.6% over the baseline. The approach concentrates verification on disagreement hotspots among expert agents rather than full reasoning chains.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or variance across runs. The margins of improvement (e.g., 1.0% on GAIA) could be within noise."
     },
     {
       "flag": "Missing cost analysis",
       "detail": "The paper claims scalability and cost-efficiency but reports no actual costs, latency, or token consumption. Running multiple expert agents plus a meta-verification agent with Gemini 2.5 Pro implies significant compute cost that is never quantified."
     },
     {
       "flag": "Single backbone model",
       "detail": "All experiments use Gemini 2.5 Pro. It is unclear whether Co-Sight's improvements generalize to other LLMs."
     },
     {
       "flag": "Corporate evaluation of own product",
       "detail": "All authors are from ZTE. While they are not evaluating a ZTE product per se, Co-Sight appears to be a ZTE system being promoted without independent evaluation or disclosed funding."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Critical hyperparameters (temperature settings for conservative/radical agents, consensus threshold θ, audit budget Bmax) are defined formally but their actual values are never reported."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Foundational agent scaffolding paradigm that Co-Sight builds upon."
     },
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Thomas Wolf", "Yann LeCun", "Thomas Scialom"],
       "year": 2024,
       "relevance": "Primary benchmark used for evaluating agentic reasoning systems."
     },
     {
       "title": "Humanity's last exam",
       "authors": ["Long Phan", "Alice Gatti", "Ziwen Han"],
       "year": 2025,
       "arxiv_id": "2501.14249",
       "relevance": "Frontier-difficulty benchmark used to evaluate advanced reasoning capabilities."
     },
     {
       "title": "Graph of thoughts: Solving elaborate problems with large language models",
       "authors": ["Maciej Besta"],
       "year": 2024,
       "relevance": "Graph-structured prompting method for LLM reasoning that Co-Sight extends."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2023,
       "relevance": "Self-verification approach for LLM agents relevant to Co-Sight's meta-verification."
     },
     {
       "title": "Improving factuality and reasoning in language models through multi-agent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba", "Joshua B Tenenbaum", "Igor Mordatch"],
       "year": 2024,
       "relevance": "Multi-agent debate framework that Co-Sight's conflict-aware approach improves upon."
     },
     {
       "title": "Detecting hallucinations in large language models using semantic entropy",
       "authors": ["Sebastian Farquhar", "Jannik Kossen", "Lorenz Kuhn", "Yarin Gal"],
       "year": 2024,
       "relevance": "Hallucination detection method that motivates Co-Sight's verification approach."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Self-consistency voting method used as a baseline comparison."
     },
     {
       "title": "PlanGEN: A multi-agent framework for generating planning and reasoning trajectories for complex problem solving",
       "authors": ["Mihir Parmar"],
       "year": 2025,
       "arxiv_id": "2502.16111",
       "relevance": "Multi-agent planning framework that represents the current state of agentic reasoning."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analysis of failure modes in multi-agent LLM systems, directly relevant to understanding reliability of agentic approaches."
     }
   ]
 }