ai-research-survey

scan.json (26330B)

---

 {
   "paper": {
     "title": "Many AI Analysts, One Dataset: Navigating the Agentic Data Science Multiverse",
     "authors": ["Martin Bertran", "Riccardo Fogliato", "Zhiwei Steven Wu"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.18710"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Fully autonomous AI analysts built on LLMs reproduce the defining signature of many-analyst studies: wide dispersion in effect sizes, p-values, and binary hypothesis support across independent runs on the same dataset. This dispersion is steerable—varying analyst persona or base model shifts outcome distributions by 34–66 percentage points even after auditor-based filtering. Across 4,946 runs (3,303 compliant) on three datasets, confirmation-seeking personas produce systematically lower p-values and higher support rates, with recognizable analytic choices (weighting, outlier removal, clustering) driving the differences.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The underlying datasets (soccer, METR RCT, ANES) are public, but the paper does not release its own generated transcripts, analyst outputs, or extracted decision codebooks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the Inspect AI framework and specific LLMs but provides no requirements.txt, Dockerfile, or versioned dependency list."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The experimental setup is described but not in a runnable form."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are shown in the specification curves (Figures 1, 4, 5) for each AI analyst-produced analysis."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports percentage point differences in support rates across personas (34–66 pp) but does not apply any formal statistical test to its own comparisons between conditions. Claims like 'persona choice alone produces up to a 47 percentage point swing' are based on raw comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Support rate differences are reported with context: 'comparing the most skeptical (Negative) to the most confirmation-seeking (Strong CS) persona yields support rate differences ranging from 34 percentage points (anes-views) to 66 percentage points (metr-rct)' (Section 4.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper targets ~30 runs per cell but provides no power analysis or justification for why 30 is sufficient. Total N=4,946 runs (3,303 compliant) is stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Distributions of p-values (Figure 3), effect sizes (Figures 1, 4, 5), and support rates (Figure 2) are shown, making variance visible across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The Standard (neutral) persona serves as the baseline condition against which Negative, Positive, CS, and Strong CS personas are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Contemporary models used: Claude Sonnet 4.5, Haiku 4.5, Qwen3 Coder 480B, Qwen3 235B A22B — all recent at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Persona and model are systematically varied as independent variables, functioning as a factorial experiment that isolates the contribution of each factor to outcome variation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple outcome measures reported: hypothesis support rate, p-value distributions, effect size distributions, exclusion rates (Table 2), and extracted analytic decision patterns."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "'Two authors manually reviewed a stratified subset of runs and found no major disagreements that would change substantive conclusions with respect to the default full-transcript auditor' (Section 3.3)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The study is not a train/test ML evaluation — it studies analytical variability across agent runs. No held-out set concept applies."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (3 datasets), persona (5 conditions), and model (4 models) in Table 2 and Figure 2."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Exclusion patterns are discussed in detail: hallucinated results in pilot runs, truncated reports (38% for Qwen), variable misidentification (42% for Qwen), and confirmation-seeking personas showing elevated exclusion rates (Section 4.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that auditing 'does not eliminate the dispersion' and that Qwen3 Coder 480B had 48% exclusion rate overall and 82–84% for CS personas (Table 2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about wide dispersion, structured analytic diversity, and steerability are all supported by Figures 1–5, Table 2, and the specification curve analyses in Sections 4.1–4.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims persona and model choice 'shift' outcome distributions. Since persona is a manipulated independent variable in a controlled experiment (holding dataset and estimand fixed), causal language is appropriate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'These results highlight a central challenge for an AI-automated future of empirical science' — generalizing from 4 LLMs and 3 datasets to all of empirical science. The paper does not explicitly bound its claims to the tested models or task types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses contamination as an alternative explanation (soccer dataset results may reflect training data recall, not analysis), model-specific failure modes (Qwen truncation vs. deliberate choices), and notes that auditor filtering may itself introduce bias (Section 5)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures what it claims: analytical variability in effect sizes, p-values, and support decisions. It also explicitly fixes estimands to ensure comparability, acknowledging this 'restricts the portion of the multiverse that analysts can explore' (Section 3.1)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'Claude Sonnet 4.5', 'Haiku 4.5', 'Qwen3 Coder 480B', 'Qwen3 235B A22B' — marketing names without snapshot dates or API version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all five analyst personas and the auditor system/user prompts are provided in Appendix A, including the exact modifications for each variant."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "'All AI analysts use sampling temperature T = 1.0, chosen deliberately so that stochastic sampling contributes to analytic diversity, and are capped at 250 messages or 60 minutes per run' (Section 3.2)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "'Analysts are implemented as tool-using ReAct agents in the Inspect AI framework. Each analyst has access to a persistent Python session with standard data-science libraries, a stateful shell, and a file editor' (Section 3.2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The auditing pipeline is documented: compliance screening criteria (validity verdict + required outputs), exclusion rates by model and persona (Table 2), and the two-criterion filter (Section 4.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Some limitations are discussed within the Conclusion and Discussion (Section 5), but they are embedded in the broader discussion rather than receiving substantive standalone treatment."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: soccer dataset contamination ('AI analysts reproducing qualitative conclusions before inspecting the data'), auditor limitations ('any definition of a reasonable analysis depends on chosen analytic standards'), and Qwen-specific failure modes (Section 4.1)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it did NOT test or claim. It generalizes to 'an AI-automated future of empirical science' without bounding to the tested models, domains, or task types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No release of raw transcripts, analyst outputs, auditor evaluations, or extracted decision codebooks. Only aggregated results are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full experimental procedure: how AI analysts are configured, what inputs they receive, how runs are audited, and how decisions are extracted."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 'subjects' are AI agent runs, whose generation procedure is described in Section 3.2."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: 4,946 total runs → compliance screening → 3,303 compliant (67%). Exclusion rates broken down by model and persona in Table 2. Decision extraction codebook process described in Section 3.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section listing grants or corporate sponsorship. Authors are at Amazon AWS but no explicit funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Amazon AWS (Bertran, Wu), 'Work done while at Amazon AWS' (Fogliato), and Carnegie Mellon University (Wu)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are Amazon AWS employees. Amazon has a commercial interest in AI systems and their perceived reliability. The paper's findings about analytical variability and steerability could affect perceptions of AI-powered analytics products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses contamination qualitatively (soccer = 'high-contamination', metr-rct = 'unlikely to appear in current training corpora') but does not state specific training data cutoff dates for any of the four models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "'In pilot runs we frequently observed AI analysts reproducing qualitative conclusions before inspecting the data' (Section 3.3). The paper explicitly uses contamination level as a design variable across the three datasets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a core design variable: the soccer dataset is 'high-contamination', metr-rct is 'recent and unlikely to appear in current training corpora', and anes-views is 'designed to be low-contamination' (Section 3.1). The auditor filters runs that recall results without analysis."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses AI agents as the subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-run costs reported despite running 4,946 agent sessions across four commercial/large LLMs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. The 250-message/60-minute cap per run is mentioned but total API spend or GPU hours are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature T=1.0 with ~30 independent runs per cell explicitly produces stochastic variation. The entire paper is about characterizing this variation — dispersion across runs IS the result."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "'We target approximately 30 independent compliant runs per (dataset × model × persona) cell' (Section 3.2). Total: 4,946 runs, 3,303 compliant."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search was conducted for the agent framework itself. T=1.0 and 250-message cap are stated as design choices without exploring alternatives."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "The study does not select a best configuration — it characterizes the distribution across all configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparisons across 5 personas × 4 models × 3 datasets without applying any multiple comparison correction. No formal statistical tests are used for its own comparisons, so correction is not applied."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not claim its system is better than alternatives — it studies a phenomenon (analytical variability). Self-comparison bias is not relevant."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The study does not compare methods at different compute levels. All conditions use the same per-run budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares AI analyst variability against human many-analyst studies (Silberzahn et al., Breznau et al.) to validate that the framework captures real analytical variability. The soccer dataset replication explicitly tests construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models use the same Inspect AI scaffold with identical tool access. Model comparisons are within the same scaffold, isolating the model variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly addressed: soccer dataset is 'high-contamination' because results are widely known, metr-rct is recent and unlikely in training corpora, anes-views is 'designed to be low-contamination' (Section 3.1)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the task prompt or dataset structure leaks information about expected results beyond the explicit contamination discussion."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether runs within the same model-persona cell are truly independent given shared model weights and prompt structure."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The auditor detects contamination behaviorally: 'some analysts could fully recall the results from Silberzahn et al. and reported those' — these runs are flagged and filtered (Section 3.3)."
       }
     }
   },
   "claims": [
     {
       "claim": "AI analysts reproduce the defining signature of many-analyst studies: systematic dispersion in defensible conclusions under a shared hypothesis, dataset, and estimand.",
       "evidence": "Across 3,303 compliant runs on 3 datasets, AI analysts show wide dispersion in effect sizes, p-values, and binary support decisions (Figures 1–5, Section 4).",
       "supported": "strong"
     },
     {
       "claim": "The outcome distribution is steerable: varying persona or base model shifts the distribution of evidential outcomes even after auditor-based filtering.",
       "evidence": "Comparing Negative to Strong CS persona yields support rate differences of 34 pp (anes-views) to 66 pp (metr-rct). Figure 2 shows systematic shifts by persona and model (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Confirmation-seeking personas produce systematically lower p-values.",
       "evidence": "Figure 3 shows p-value distributions shifted downward for CS and Strong CS personas across datasets, both before and after compliance filtering.",
       "supported": "strong"
     },
     {
       "claim": "Auditing filters invalid runs but does not eliminate dispersion.",
       "evidence": "34% of runs excluded (Table 2), but compliant runs still show wide dispersion in Figures 1–5. The separation in p-value distributions persists after filtering (Figure 3 bottom panel).",
       "supported": "strong"
     },
     {
       "claim": "These results highlight a central challenge for an AI-automated future of empirical science.",
       "evidence": "Based on 4 LLMs and 3 datasets. The generalization to 'empirical science' broadly is not empirically supported beyond these settings.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company conflict not disclosed",
       "detail": "All authors are affiliated with Amazon AWS. Amazon sells AI services and has a commercial interest in how AI analytical tools are perceived. No competing interests statement is provided."
     },
     {
       "flag": "No formal statistical tests on own claims",
       "detail": "The paper reports large percentage-point differences between conditions (34–66 pp) but applies no significance tests, confidence intervals, or effect size measures to its own between-condition comparisons. The statistical rigor applied to analyzing AI analyst outputs is not applied to the paper's own experimental comparisons."
     },
     {
       "flag": "No data or code release",
       "detail": "Despite running 4,946 agent sessions generating reproducible transcripts, neither the raw transcripts, extracted decisions, auditor evaluations, nor the analysis code are released. This limits independent verification of the reported results."
     },
     {
       "flag": "Unbounded generalization",
       "detail": "The paper generalizes from 4 LLMs on 3 datasets to claims about 'an AI-automated future of empirical science' without bounding the scope of its conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 ai on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "The METR coding RCT, one of the three datasets used in this study; directly relevant to AI programming productivity evaluation."
     },
     {
       "title": "Swe-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major LLM code generation benchmark referenced in the related work on agentic evaluation."
     },
     {
       "title": "Data interpreter: An llm agent for data science",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "arxiv_id": "2402.18679",
       "relevance": "LLM agent for automated data science, directly relevant to agentic AI coding and analysis workflows."
     },
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao"],
       "year": 2022,
       "relevance": "The ReAct agent framework used as the computational scaffold for the AI analysts in this study."
     },
     {
       "title": "Swe-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Agentic software engineering agent referenced in the related work on LLM-based coding tools."
     },
     {
       "title": "BLADE: Benchmarking language model agents for data-driven science",
       "authors": ["Ken Gu"],
       "year": 2024,
       "arxiv_id": "2408.09667",
       "relevance": "Most closely related work — benchmarks LLM agents on data science tasks; this paper explicitly positions against it."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick"],
       "year": 2023,
       "relevance": "Foundational work on tool-using LLMs, underlying the agentic approach used in this study."
     },
     {
       "title": "Measuring ai ability to complete long tasks",
       "authors": ["Thomas Kwa"],
       "year": 2025,
       "arxiv_id": "2503.14499",
       "relevance": "Benchmark for long-horizon AI task completion, relevant to evaluating agentic AI capabilities."
     },
     {
       "title": "The ai scientist-v2: Workshop-level automated scientific discovery via agentic tree search",
       "authors": ["Yutaro Yamada"],
       "year": 2025,
       "arxiv_id": "2504.08066",
       "relevance": "Automated scientific discovery agent relevant to AI-driven research and the risks of automated analysis."
     },
     {
       "title": "Ds-agent: Automated data science by empowering large language models with case-based reasoning",
       "authors": ["Siyuan Wang"],
       "year": 2024,
       "arxiv_id": "2402.17753",
       "relevance": "LLM-powered data science agent directly relevant to the agentic data analysis paradigm studied."
     },
     {
       "title": "Discovering language model behaviors with model-written evaluations",
       "authors": ["Ethan Perez"],
       "year": 2023,
       "relevance": "Documents sycophancy and other LLM behavioral issues relevant to understanding AI analyst bias."
     }
   ]
 }