ai-research-survey

scan.json (25636B)

---

 {
   "paper": {
     "title": "From Benchmarks to Business Impact: Deploying IBM Generalist Agent in Enterprise Production",
     "authors": ["Segev Shlomov", "Alon Oved", "Sami Marreed", "Ido Levy", "Offer Akrabi", "Avi Yaeli", "Łukasz Strak", "Elizabeth Koumpan", "Yinon Goldshtein", "Eilam Shapira", "Nir Mashkif", "Asaf Adi"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.23856",
     "doi": "10.48550/arXiv.2510.23856"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "CUGA achieves state-of-the-art on WebArena (61.7%) and AppWorld Test-Challenge (48.2% scenario completion). In a BPO talent acquisition pilot with 26 tasks across 13 APIs, CUGA reached 87% accuracy. The authors report preliminary estimates of ~90% reduction in development time and ~50% cost reduction compared to specialized agents, though these are based on controlled simulations without statistical significance testing.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states CUGA has been open-sourced with a GitHub link: https://github.com/cuga-project/cuga-agent (footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The BPO-TA benchmark is described in detail (26 tasks, 13 APIs) but no download link or public release is provided. The benchmark data itself is not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the architecture but does not give instructions for replicating benchmark results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are point estimates (e.g., 61.7% WebArena, 87% BPO-TA) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly acknowledges results are 'not formally tested for statistical significance (Dror et al. 2018, 2020)' in Section 6.2."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context: '~90% improvement' in time-to-answer (from ~20 min to ~2-5 min), reproducibility improvement from ~60% to ~95%, and provenance coverage from ~40% to ~92% (Table 4)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 26 BPO-TA tasks or the specific benchmark sizes are adequate. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported for any results. Single-run numbers throughout."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "WebArena and AppWorld results compared against published agents (Table 5: Operator, Jace.AI, ScribeAgent, etc.; Table 7: Chen et al., Gupta et al., ReAct). BPO-TA compared against 'vanilla ReAct baseline' (62% vs 79% valid-first-try)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent systems: OpenAI Operator (2025), Jace.AI (2024), ScribeAgent (2024), and concurrent AppWorld entries."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 mentions ablations: 'reflective retries (-11 points without) and variable tracking (-15 reproducibility without).' However, only two ablation results are reported with minimal detail."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Task Goal Completion and Scenario Goal Completion for AppWorld; accuracy, valid-first-try rate, provenance log rate, latency, and analyst-reported reproducibility for BPO-TA (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 reports qualitative feedback from BPO architects and analysts. Table 3 includes 'Analyst-Reported Reproducibility' score of 4.6/5. However, this is informal feedback, not a structured user study."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "WebArena and AppWorld are established benchmarks with fixed test sets. AppWorld distinguishes Test-Normal and Test-Challenge."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "WebArena results broken down per application (Table 1: GitLab, Map, Reddit, Shopping, etc.). AppWorld broken down by difficulty level (Table 2). BPO-TA task categories described (Section 6.1)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 notes 'failures concentrated on unsupported cross-application queries where graceful degradation is expected.' The BPO-TA benchmark explicitly includes graceful failure tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multi-App performance on WebArena is notably low (35.4% vs 61.7% overall). The paper acknowledges the system is 'still on its journey toward full production deployment' and discusses limitations of early architectures (Section 3.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are hedged: 'approached the accuracy of specialized agents,' 'indicating potential,' 'preliminary evaluations.' Results in Tables 1-4 support these hedged claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation claims ('reflective retries -11 points,' 'variable tracking -15 reproducibility') are causal but reported without detail on experimental design. The '90% reduction in development time' is stated as an estimate without causal methodology."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to bound claims: 'preliminary evaluations,' 'pilot-level results,' 'controlled test environments and limited analyst feedback rather than full production deployment' (Section 6.2). Title says 'Enterprise Production' but body consistently hedges."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. The improvement over ReAct baseline could be due to many factors (more compute, better prompts, etc.) but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures task accuracy on BPO-TA and frames it as 'enterprise readiness' and 'business value' without discussing the gap between benchmark accuracy on 26 curated tasks and actual business impact in production."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "AppWorld results use 'GPT-4.1' (Table 7) but no snapshot date or API version. WebArena model not specified in the paper. No version details for the LLMs used in CUGA's various sub-agents."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes 'schema-grounded prompting' and prompt design principles but does not provide actual prompt text used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters reported: no temperature, top-p, max tokens, or other LLM settings mentioned anywhere."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The architecture is described in detail: hierarchical planner-executor loops, chat layer, outer/inner loops, API Sub Agent, Browser Sub Agent, Plan Controller, ShortlisterAgent, CodeAgent, etc. (Section 5, Figures 2-4, Appendix A-B)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The BPO-TA benchmark is described at a high level (task categories, API endpoints) but the data preprocessing steps, how tasks were curated, and how gold-standard answers were created are not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Some limitations are scattered in Section 7 (Lessons Learned) and Section 6.2, but there is no substantive 'Limitations' or 'Threats to Validity' section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity discussed. The paper acknowledges results are 'preliminary' and 'pilot-level' but does not discuss specific threats like selection bias in BPO-TA task design, single-domain evaluation, or potential measurement artifacts."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope: 'still on its journey toward full production deployment,' 'controlled test environments and limited analyst feedback rather than full production deployment' (Section 6.2), pilot is read-only only. Section 7 outlines 'next steps' implying what was NOT done."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data available. BPO-TA benchmark data, detailed per-task results, and analyst feedback data are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "BPO-TA benchmark design is described: 26 tasks across 13 read-only APIs, task categories (lookup, join, looped reasoning, provenance, graceful failure), drawn from analyst practice (Section 6.1, Table 9)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'recruiters and analysts' provided feedback but does not describe how many, how they were selected, or the feedback collection process."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the pipeline from task creation to benchmark evaluation. How gold-standard answers were generated, how accuracy was scored, and how the 26 tasks were selected from possible tasks is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section. All authors are IBM Research or IBM Consulting employees, but no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors clearly listed as IBM Research or IBM Consulting with institutional emails."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "IBM employees are evaluating IBM's own CUGA system on IBM's own BPO business. IBM has a direct financial interest in showing CUGA works for enterprise deployment."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. IBM authors evaluating IBM's product with potential commercial implications — this conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff stated for GPT-4.1 or any models used in CUGA's pipeline."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WebArena or AppWorld tasks could appear in the training data of the underlying LLMs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WebArena was published in 2023 and AppWorld in 2024. Models trained after these dates may have seen benchmark content. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The qualitative feedback from analysts is informal and does not constitute a human subjects study requiring pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study conducted. Informal analyst feedback does not require IRB."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No formal human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No formal human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No formal human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports 'Average Latency per Query: 11.2s' for BPO-TA. Table 2 reports average interactions per task level for AppWorld. However, no monetary cost is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated — no GPU hours, API costs, or total spend for running the benchmarks or pilot."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of experimental runs not stated for any benchmark evaluation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported despite the system having many configurable components."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the final system configuration was selected or how many configurations were tried."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "IBM authors evaluate their own CUGA system against baselines without acknowledging self-evaluation bias. Lucic et al. (2018) concern applies directly."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of compute budgets between CUGA and baselines. CUGA's hierarchical multi-agent architecture likely uses substantially more compute than simpler baselines, but this is not quantified or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WebArena and AppWorld actually measure enterprise readiness. The paper argues enterprise deployment requires different evaluation but does not question the construct validity of the academic benchmarks it uses for SOTA claims."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CUGA is a complex scaffold compared to baselines (ReAct, etc.). The paper does not separate model capability from scaffold contribution. Comparisons in Tables 5 and 7 mix different scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. WebArena (2023) and AppWorld (2024) existed before model training — potential solutions could be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and benchmark content."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied."
       }
     }
   },
   "claims": [
     {
       "claim": "CUGA achieves state-of-the-art performance on WebArena with 61.7% accuracy",
       "evidence": "Table 1 and Table 5 show per-application breakdown and leaderboard comparison. CUGA leads with 61.7% vs Operator at 58.1%.",
       "supported": "moderate"
     },
     {
       "claim": "CUGA achieves state-of-the-art on AppWorld Test-Challenge with 48.2% scenario goal completion",
       "evidence": "Table 2 and Table 7 show per-level results. CUGA leads with 57.6% TGC / 48.2% SGC on Challenge vs next best Chen et al. at 47.2% / 28.8%.",
       "supported": "moderate"
     },
     {
       "claim": "CUGA achieves 87% accuracy on the BPO-TA benchmark (26 tasks)",
       "evidence": "Table 3 reports 87% task accuracy. However, this is on a 26-task benchmark designed by the same team, with no external validation.",
       "supported": "weak"
     },
     {
       "claim": "CUGA can reduce development time by up to 90% and cost by up to 50% compared to specialized agents",
       "evidence": "Section 7 and Table 4 present these as 'estimated benefits' from 'controlled simulations' and 'internal projections.' The paper explicitly notes these are 'not formally tested for statistical significance.'",
       "supported": "weak"
     },
     {
       "claim": "Reflective retries improve accuracy by 11 points and variable tracking improves reproducibility by 15 points",
       "evidence": "Section 6.1 mentions ablation results but provides no details on methodology, base rates, or statistical testing.",
       "supported": "weak"
     },
     {
       "claim": "Generalist agents can enable measurable business value in enterprise contexts",
       "evidence": "Based on a pre-deployment pilot with read-only access and 26 curated tasks. The system is 'under consideration for production rollout' (Section 3.2), not yet deployed.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are IBM employees evaluating IBM's CUGA system on IBM's BPO business. The BPO-TA benchmark was designed by the same team. No independent evaluation or third-party validation."
     },
     {
       "flag": "Claims exceed evidence",
       "detail": "The title says 'Enterprise Production' and 'Business Impact' but the system is in pre-deployment pilot with 26 curated read-only tasks. The '90% time reduction' and '50% cost reduction' are projections from simulations, not measured production outcomes."
     },
     {
       "flag": "No statistical rigor",
       "detail": "Paper explicitly acknowledges results are 'not formally tested for statistical significance' yet makes comparative claims. No error bars, no variance, no multiple runs reported."
     },
     {
       "flag": "Self-designed benchmark",
       "detail": "The BPO-TA benchmark (26 tasks) was created by the same team that built the system being evaluated. Task selection, gold-standard answers, and evaluation criteria are all internal with no external review."
     },
     {
       "flag": "Benchmark SOTA claims without contamination analysis",
       "detail": "WebArena (2023) and AppWorld (2024) were published before model training cutoffs. No contamination analysis despite claiming state-of-the-art."
     },
     {
       "flag": "Tiny benchmark for business impact claims",
       "detail": "26 tasks is very small for claims about enterprise readiness and business impact. The qualitative evidence is from unstructured feedback with no reported methodology."
     }
   ],
   "cited_papers": [
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R. Narasimhan", "Yuan Cao"],
       "year": 2022,
       "relevance": "Foundational agentic paradigm combining chain-of-thought reasoning with environment actions."
     },
     {
       "title": "Executable code actions elicit better llm agents",
       "authors": ["Xingyao Wang"],
       "year": 2024,
       "relevance": "CodeAct approach for code-centric agent execution, relevant to agentic code generation."
     },
     {
       "title": "Autogen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu"],
       "year": 2024,
       "relevance": "Multi-agent framework for orchestrating agent conversations, directly compared architecture pattern."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["Shuyan Zhou"],
       "year": 2023,
       "relevance": "Primary benchmark used to evaluate CUGA's web agent capabilities."
     },
     {
       "title": "AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents",
       "authors": ["Harsh Trivedi"],
       "year": 2024,
       "relevance": "Primary benchmark for multi-application API agent evaluation."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn"],
       "year": 2023,
       "relevance": "Reflective retry mechanism used in CUGA's architecture."
     },
     {
       "title": "St-webagentbench: A benchmark for evaluating safety and trustworthiness in web agents",
       "authors": ["Ido Levy"],
       "year": 2024,
       "relevance": "Safety/trustworthiness benchmark for web agents, from same research group."
     },
     {
       "title": "Tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
       "authors": ["Shunyu Yao"],
       "year": 2025,
       "relevance": "Benchmark for tool-agent-user dynamics and policy/instruction following."
     },
     {
       "title": "Magentic-one: A generalist multi-agent system for solving complex tasks",
       "authors": ["Adam Fourney"],
       "year": 2024,
       "arxiv_id": "2411.04468",
       "relevance": "Competing generalist multi-agent system for complex task solving."
     },
     {
       "title": "The hitchhiker's guide to testing statistical significance in natural language processing",
       "authors": ["Rotem Dror", "Gili Baumer", "Segev Shlomov", "Roi Reichart"],
       "year": 2018,
       "relevance": "Statistical testing methodology for NLP — cited by authors as the standard they did NOT follow."
     },
     {
       "title": "Towards enterprise-ready computer using generalist agent",
       "authors": ["Sami Marreed"],
       "year": 2025,
       "arxiv_id": "2503.01861",
       "relevance": "Companion paper describing CUGA's hierarchical architecture in detail."
     },
     {
       "title": "From grounding to planning: Benchmarking bottlenecks in web agents",
       "authors": ["Segev Shlomov"],
       "year": 2024,
       "arxiv_id": "2409.01927",
       "relevance": "Identifies planning as dominant bottleneck in web agents, motivating CUGA's planner-executor design."
     }
   ]
 }