ai-research-survey

scan.json (28136B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "paper": {
     "title": "How Do AI Agents Do Human Work? Comparing AI and Human Workflows Across Diverse Occupations",
     "authors": ["Zora Zhiruo Wang", "Yijia Shao", "Omar Shaikh", "Daniel Fried", "Graham Neubig", "Diyi Yang"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.22780",
     "doi": "10.48550/arXiv.2510.22780"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in footnote 1: https://github.com/zorazrw/workflow-induction-toolkit"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No explicit dataset download link provided. The 112 collected trajectories (human and agent activities) are not stated to be publicly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section listing library versions is mentioned. The paper states experiments run in TAC's sandboxed environments but does not provide reproducible environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README with commands described. The paper references the TAC setup but does not provide specific instructions for reproducing the full study."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (success rates in Table 8, efficiency comparisons) are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "T-statistics and p-values are reported for workflow alignment measures in Tables 5 and 6. The paper states 'We conducted significance tests to ensure the statistical reliability of all quantitative findings' in the Limitations section."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: agents are 88.3% faster, cost 90.4-96.2% less, human-agent alignment is 83.0%, AI augmentation accelerates work by 24.3% while automation slows by 17.7%."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 3 human workers per task, 4 agent frameworks, or 16 tasks were chosen. No power analysis. The Limitations section acknowledges the limited number of tasks but does not justify sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations or variance measures reported across runs or workers. Results are presented as averages without spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human workers serve as the baseline for comparison against 4 agent frameworks. Multiple agent frameworks (OpenHands-GPT, OpenHands-Claude, ChatGPT, Manus) are compared against each other and against humans."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Agent frameworks tested (ChatGPT Agent, Manus, OpenHands with gpt-4o and claude-sonnet-4) are contemporary state-of-the-art systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The workflow induction toolkit components are not systematically ablated to measure their contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: task success rate, workflow alignment (matching steps %, order preservation %), time elapsed, action count, cost, and progress rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual verification of workflows and their evaluation results is performed, showing 'substantial agreement with human judgments — 0.637 and 0.781 in Cohen's Kappa for consistency and modularity metrics' (§3.3). Workflow alignment pairs are also 'manually refined for accuracy' (§3.4)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a train/test evaluation paradigm. The study compares human and agent workflows on the same tasks without a training phase."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by skill category (data analysis, engineering, computation, writing, design) throughout the paper, including Tables 6, 8, and Figures 9, 10, 19."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 extensively discusses agent failure modes: data fabrication (Figure 6a), tool misuse (Figure 6b), computation errors, format transformation failures, and limited visual capabilities."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: AI automation slows humans by 17.7% (§4.2), agents produce lower-quality work (§5), agents fabricate data (§5.1), engineering tasks surprisingly show low agent quality despite being the focus of agent development."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about programmatic approach, inferior quality with fabrication, and 88.3% faster/90.4-96.2% cheaper are all supported by results in §4 and §5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'AI automation markedly reshapes workflows and slows human work by 17.7%' and 'agents deliver results 88.3% faster' but the study design is observational — no randomization of workers to conditions, no control for worker expertise differences. Workers self-selected AI tool usage."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims comparison 'Across Diverse Occupations' but the study uses only 16 tasks with 48 human workers and 4 agent frameworks. The paper extrapolates to '287 computer-using U.S. occupations and 71.9% of their daily work activities' based on skill category coverage, which is a substantial generalization from 16 tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: AI automation slowdown may be due to '(i) additional time spent verifying, debugging, and correcting AI solutions' and '(ii) lower task expertise among human workers who are more likely to rely on AI for full-process automation' (§4.2)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses program verifier checkpoints as a proxy for 'work quality' and action count/time as proxies for 'efficiency', but does not discuss limitations of these proxies. Checkpoint-based evaluation is acknowledged as limited in §6.2 but this is brief and does not distinguish proxy from outcome for the main quality claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Agent models are specified as 'gpt-4o' and 'claude-sonnet-4' without snapshot dates or API versions. The workflow induction model is 'claude-sonnet-3.7' without a version identifier. These are marketing names, not pinned versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompts for workflow induction steps are provided in Appendix B: segment merging (§B.1), action-goal consistency evaluation, and modularity evaluation (§B.2)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or sampling parameters reported for any of the LLM calls (workflow induction, agent runs, or evaluation)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Agent frameworks are described: OpenHands is described as coding-oriented with specific action spaces (Table 4), ChatGPT Agent and Manus are described with their available tools and distinctive behaviors (search_web, check_with_users). The workflow induction pipeline is described in detail in §3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Human activity preprocessing is documented in §2.2 and §A.3: merging consecutive keypress/scroll actions, double-click detection within 0.1 seconds, reducing action count by 83.2%. Agent activity processing described in §A.4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present (after §8) discussing coverage of work activities, O*NET database limitations, and understanding AI's impact on work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: 'our study does not yet capture the full diversity of real-world working contexts', 'the inherent difficulty in constructing and maintaining such a large-scale database may introduce inaccuracies', limited number of tasks and workers, O*NET may not reflect current workforce distribution."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: communication and decision-making skills are excluded (§A.1), study focuses on computer-using occupations only, and acknowledges the need for 'additional task instances with greater breadth'."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 112 collected trajectories (raw mouse/keyboard actions and screenshots) are not stated to be publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: §2.2 for human workers (recording tool capturing actions and screenshots), §2.3 for agents (direct access for OpenHands, UI-based capture for ChatGPT/Manus), and §A.3-A.4 for processing details."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Human worker recruitment described in §2.2 and §A.3: hired from Upwork, screened based on professional qualifications and work portfolios, 3 workers per task, required relevant educational backgrounds and current professional experience."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw activities to workflows is documented: raw recording → preprocessing (merging keypresses, scroll actions, double-click detection, 83.2% reduction) → trajectory segmentation (MSE-based) → semantic merging → hierarchical goal annotation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed in Acknowledgments: Google PhD Fellowship, Sloan Foundation, ONR grant N000142412532, Open Philanthropy."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (Carnegie Mellon University, Stanford University) are clearly listed. No authors are affiliated with the agent companies being evaluated (OpenAI, Anthropic, Manus)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (Google, Sloan Foundation, ONR, Open Philanthropy) do not have a direct financial stake in the comparative outcomes of ChatGPT vs. Manus vs. OpenHands. Google is a competitor but the first author's fellowship does not create a direct outcome dependency."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate pre-trained model capability on a benchmark in the traditional sense. It evaluates agent frameworks on work tasks — the concern is agent capability, not whether models memorized benchmark answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as training_cutoff_stated — the study evaluates agent work behavior on novel realistic tasks, not model knowledge on existing benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Tasks are custom-designed or adopted from TheAgentCompany with sandboxed environments, making traditional benchmark contamination less relevant."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration. The study involves 48 human participants but no pre-registration link is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite recruiting 48 human workers and collecting their computer-use activities."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Worker demographics are not reported beyond stating they have 'relevant educational backgrounds and current professional experience.' No experience levels, gender, geographic distribution, or other demographics provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Inclusion criteria stated in §2.2 and §A.3: 'relevant educational backgrounds and currently doing jobs related to each task and skill', screened based on 'professional qualifications', 'work portfolios and prior client ratings'."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a randomized experiment — workers are not assigned to conditions. Each worker completes their task using their preferred tools."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not applicable — this is an observational comparison of human vs. agent workflows, not an experiment with treatment conditions."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information provided. The paper states 48 human trajectories were collected (3 per task × 16 tasks) but does not mention whether any workers started but did not finish."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Agent costs reported: OpenHands-GPT $0.94/task, OpenHands-Claude $2.39/task. Human workers charge $24.79/task on average (§5.2)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated for the workflow induction pipeline (which uses claude-sonnet-3.7 extensively) or for the overall study."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Agent runs appear to be single-run. No mention of running agents multiple times with different seeds to assess result stability."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of agent runs per task is not explicitly stated. It appears each agent framework was run once per task (64 agent trajectories = 4 agents × 16 tasks)."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning is performed — agents are used as-is with their default configurations."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection is performed — agents are evaluated with their default settings."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple t-tests are reported across skill categories (Tables 5, 6) without mention of multiple comparison correction (Bonferroni, etc.)."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors evaluate third-party agent systems (ChatGPT, Manus, OpenHands), not their own system. Self-comparison bias is not applicable."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Compute differences between agents are negligible relative to the study's goals — the focus is on workflow comparison, not compute-matched performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses construct validity: tasks are designed to cover 70.1-95.2% of employment per skill category using O*NET data (§2.1, Figure 2). The Limitations section discusses O*NET database limitations and coverage gaps."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly compares the same model backbone (GPT, Claude) across different scaffolds (OpenHands vs ChatGPT/Manus), and discusses how different frameworks lead to different behaviors despite shared backbones (§2.3, §4.1)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Contamination module not activated — see contamination section rationale."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Contamination module not activated."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Contamination module not activated."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Contamination module not activated."
       }
     }
   },
   "claims": [
     {
       "claim": "Agents take an overwhelmingly programmatic approach across all work domains, with 93.8% program-use rate, even for open-ended visual tasks like design.",
       "evidence": "Figure 4(c) and Figure 10 show program-use rates across tasks. Agents exhibit 27.8% stronger alignment with program-using human steps than non-programmatic ones (Table 7, §4.1).",
       "supported": "strong"
     },
     {
       "claim": "Human workflows are substantially altered by AI automation but not by augmentation. AI augmentation accelerates work by 24.3%, while AI automation slows humans by 17.7%.",
       "evidence": "§4.2 and Figure 5. Workflow alignment with independent humans: 40.3% for automation scenarios vs 76.8% for augmentation scenarios.",
       "supported": "moderate"
     },
     {
       "claim": "Agents produce work of inferior quality, with success rates 32.5-49.5% lower than humans, including concerning behaviors like data fabrication.",
       "evidence": "Table 8 and Figure 7(a). Concrete fabrication examples in Figure 6(a) where agent creates fake receipt data instead of parsing actual bill images.",
       "supported": "strong"
     },
     {
       "claim": "Agents deliver results 88.3% faster and at 90.4-96.2% lower cost than human workers.",
       "evidence": "§5.2: restricted to tasks completed by both, agents take 88.3% less time and 96.4% fewer actions. OpenHands-GPT costs $0.94/task vs human $24.79/task.",
       "supported": "strong"
     },
     {
       "claim": "Human-agent teaming at the workflow step level improves efficiency by 68.7% while maintaining task correctness.",
       "evidence": "§5.3 and Figure 7(c): demonstrated on data analysis tasks where human handles file navigation and agent performs analysis. Single example demonstrated.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational", "qualitative"],
   "key_findings": "AI agents approach human work through a programmatic lens (93.8% program use rate), even for visual tasks, diverging from humans' UI-centric workflows. Despite high workflow alignment (83% matching steps), agents produce 32.5-49.5% lower quality work, with concerning behaviors like data fabrication. However, agents are 88.3% faster and 90.4-96.2% cheaper, suggesting value in step-level delegation where humans handle navigation and verification while agents handle programmable steps.",
   "red_flags": [
     {
       "flag": "Very small sample size for broad claims",
       "detail": "Only 16 tasks, 48 human workers (3 per task), and 4 agent frameworks are used to make claims about '287 computer-using U.S. occupations and 71.9% of their daily work activities.' The coverage argument via skill categories is indirect."
     },
     {
       "flag": "Single-run agent evaluation",
       "detail": "Each agent appears to be run only once per task. Given the stochasticity of LLM outputs, single-run results may not be representative of typical agent performance."
     },
     {
       "flag": "No IRB for human subjects research",
       "detail": "The study recruits 48 human workers from Upwork and records their screen activities, but no IRB or ethics approval is mentioned."
     },
     {
       "flag": "Teaming claim based on single example",
       "detail": "The human-agent teaming benefit (68.7% efficiency gain) is demonstrated on data analysis tasks failed by one agent (Manus), which is a very limited demonstration for a headline finding."
     },
     {
       "flag": "LLM-as-judge without calibration",
       "detail": "Workflow quality evaluation and alignment are performed using claude-sonnet-3.7 as an evaluator. While Cohen's Kappa against human judgments is reported (0.637 for consistency), this is only 'substantial' agreement — the modularity Kappa of 0.781 is better but the evaluation still relies heavily on an LLM judge."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. R. Narasimhan"],
       "year": 2024,
       "relevance": "Key benchmark for evaluating LLM agents on software engineering tasks, directly relevant to agentic coding evaluation."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["X. Wang", "B. Li", "Y. Song"],
       "year": 2025,
       "relevance": "One of the agent frameworks evaluated in this study; open-source platform for AI coding agents."
     },
     {
       "title": "TheAgentCompany: benchmarking LLM agents on consequential real world tasks",
       "authors": ["F. F. Xu", "Y. Song", "B. Li"],
       "year": 2024,
       "relevance": "Benchmark environment used in this study for evaluating agents on work-related tasks across occupations."
     },
     {
       "title": "GPTs are GPTs: An early look at the labor market impact potential of large language models",
       "authors": ["T. Eloundou", "S. Manning", "P. Mishkin", "D. Rock"],
       "year": 2023,
       "arxiv_id": "2303.10130",
       "relevance": "Foundational study on LLM labor market impact potential, directly relevant to AI workforce effects."
     },
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["J. Becker", "N. Rush", "E. Barnes", "D. Rein"],
       "year": 2025,
       "relevance": "Studies AI impact on developer productivity, finding verification overhead can negate efficiency gains — corroborates this paper's automation slowdown finding."
     },
     {
       "title": "Which economic tasks are performed with AI? Evidence from millions of Claude conversations",
       "authors": ["K. Handa", "A. Tamkin"],
       "year": 2025,
       "relevance": "Large-scale analysis of AI use in economic tasks, relevant to understanding AI's role in human work."
     },
     {
       "title": "Code with me or for me? How increasing AI automation transforms developer workflows",
       "authors": ["V. Chen", "A. Talwalkar", "R. Brennan", "G. Neubig"],
       "year": 2025,
       "relevance": "Directly studies how AI automation vs augmentation changes developer workflows — complementary to this paper."
     },
     {
       "title": "Navigating the jagged technological frontier: Field experimental evidence of the effects of AI on knowledge worker productivity and quality",
       "authors": ["F. Dell'Acqua"],
       "year": 2023,
       "relevance": "Field experiment on AI effects on knowledge worker productivity, finds AI helps for some tasks but hurts for others."
     },
     {
       "title": "R2e-gym: Procedural environment generation and hybrid verifiers for scaling open-weights SWE agents",
       "authors": ["N. Jain", "J. Singh"],
       "year": 2025,
       "relevance": "Training environment for SWE agents, relevant to understanding agent development for software engineering tasks."
     },
     {
       "title": "Future of work with AI agents: Auditing automation and augmentation potential across the US workforce",
       "authors": ["Y. Shao", "H. Zope"],
       "year": 2025,
       "relevance": "Audits AI automation/augmentation potential across US occupations using same O*NET framework as this study."
     },
     {
       "title": "The AI Scientist: Towards fully automated open-ended scientific discovery",
       "authors": ["C. Lu", "C. Lu", "R. T. Lange"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Attempts fully automated scientific research with AI agents, relevant to understanding agent capabilities and limitations."
     },
     {
       "title": "Re-bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",
       "authors": ["H. Wijk", "T. Lin", "J. Becker"],
       "year": 2024,
       "relevance": "Directly compares AI agents against human experts on R&D tasks, complementary human-agent comparison study."
     }
   ]
 }