ai-research-survey

scan.json (32595B)

---

 {
   "paper": {
     "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
     "authors": [
       "Tianbao Xie",
       "Danyang Zhang",
       "Jixuan Chen",
       "Xiaochuan Li",
       "Siheng Zhao",
       "Ruisheng Cao",
       "Toh Jing Hua",
       "Zhoujun Cheng",
       "Dongchan Shin",
       "Fangyu Lei",
       "Yitao Liu",
       "Yiheng Xu",
       "Shuyan Zhou",
       "Silvio Savarese",
       "Caiming Xiong",
       "Victor Zhong",
       "Tao Yu"
     ],
     "year": 2024,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2404.07972",
     "doi": "10.48550/arXiv.2404.07972"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code, environment, baseline models, and data are publicly available at https://os-world.github.io' (Abstract, Section 1) and 'We open-source OSWORLD environment and benchmark, including environment initial state setup, reliable evaluation scripts, documentation, and our implementation of baseline models' (Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark of 369 tasks with initial state configs and evaluation scripts is publicly released. The paper states data is available at https://os-world.github.io. Task configurations, evaluation functions, and VM snapshots are included."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Detailed environment specifications are provided: Ubuntu 22.04, LibreOffice 7.3.7.2, Thunderbird 115.6.0, specific VM configurations, screen resolution 1920x1080. Software dependencies including pyautogui, pyatspi, openpyxl, python-docx, python-pptx, Playwright are documented throughout Appendix B."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed documentation of the environment setup (Section 2.2), task configuration files (Figure 2), evaluation scripts, and releases baseline implementations. The open-source release includes documentation for reproducing experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports only point estimates of success rates (e.g., 12.24%, 5.26%) with no confidence intervals or error bars. No uncertainty quantification is provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'GPT-4V falls far behind', model X outperforms model Y) based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw success rate percentages. No formal effect sizes (Cohen's d, odds ratios, etc.) are computed. The paper compares numbers directly (e.g., 12.24% vs 72.36%) without formal effect size analysis."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 369 tasks constitute a sufficient sample. No power analysis is discussed. The ablation studies (Figures 5, 7, 8) are run on subsets described as '10% of examples' or '28 tasks' without justifying these sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run numbers. No standard deviations, variance across seeds, or interquartile ranges are reported in any table or figure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes extensive baseline comparisons across 8+ models (GPT-4V, GPT-4, Gemini-Pro, Claude-3 Opus, Mixtral, Llama-3, CogAgent, Qwen-Max) and human performance (Table 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4V, GPT-4o, Gemini-Pro-1.5, Claude-3 Opus, and Llama-3 — all state-of-the-art at time of writing (2024). Models represent both open-source and closed-source frontiers."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies are conducted: input modality variations (a11y tree, screenshot, SoM, combined in Table 5), screenshot resolution (Figure 5), trajectory history length (Figure 7), and window perturbation robustness (Figure 8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary and essentially only metric is success rate. While the paper breaks results down by category, the underlying metric is the same binary task completion rate. No complementary metrics (e.g., partial credit distribution, step efficiency, action accuracy) are systematically reported in main results."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human performance evaluation is conducted (Section 3.4): 'We conduct human evaluations on each example in our dataset, with annotators being computer science major college students.' Human accuracy (72.36%) and completion time (median 111.94s) are reported (Figure 4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark tasks are a clean evaluation set — all agent evaluations are zero-shot with no training on benchmark tasks. The tasks were created specifically for evaluation and models were not fine-tuned on them."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides breakdowns by OS, Office, Daily, Professional, and Workflow categories. Table 14 (Appendix) provides per-app breakdowns (Calc, Impress, Writer, Chrome, VLC, etc.). Table 6 breaks down by difficulty, feasibility, and number of apps."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 provides extensive qualitative analysis of failure cases with specific examples (Figures 9, 10, 16-18). Common error types are catalogued: mouse click inaccuracies (>75% of failures), repetitive clicks, environmental noise, lack of domain knowledge."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: SoM hurts GPT-4V performance vs screenshot+a11y tree (Section 4.2), screenshot-only history doesn't improve performance (Figure 7), few-shot prompting failed (2.79%, Section 4.1), window perturbation causes 60-80% performance drops (Figure 8)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'first-of-its-kind scalable, real computer environment' (Table 4 comparison), '369 computer tasks' (Table 3), 'over 72.36% of the tasks' by humans (Section 3.4, Figure 4), 'best model achieves only 12.24%' (Table 5, GPT-4 with a11y tree)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'higher resolution leads to improved performance' (Figure 5) and 'longer text history improves performance' (Figure 7) are supported by controlled single-variable manipulation experiments. Ablation studies in Table 5 isolate input modalities. The study designs are adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Open-Ended Tasks in Real Computer Environments' and the abstract says the environment supports 'various operating systems such as Ubuntu, Windows, and macOS,' but the benchmark evaluation is almost entirely Ubuntu-based (369 tasks) with only 43 supplementary Windows tasks and no macOS tasks. The conclusion generalizes to 'autonomous digital agents' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides explanations for results (e.g., SoM fails due to noise from too many elements, Claude struggles with grounding) but doesn't systematically consider alternative explanations. No discussion of confounds like model size, training data differences, or API version effects on comparative results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures success rate on 369 curated tasks but frames results in terms of 'ability to serve as computer assistants' (Section 4.2) and 'generalist capable computer agents' (Section 1). The gap between curated benchmark performance and real-world computer assistance is not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1 specifies 'gpt-3.5-turbo-16k, gpt-4-0125-preview, and gpt-4-vision-preview' and 'gemini-pro and gemini-pro-vision.' These are specific API version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix C.2 (Sections C.2.1 and C.2.2) for all four input settings: a11y tree, screenshot, screenshot+a11y tree, and Set-of-Mark."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section C.1 states: 'temperature parameter to 1.0, and top_p to 0.9, and the maximum number of tokens for generation is set to 1500. We set the maximum steps of interaction to 15 and the maximum time limits to 30 minutes.' Screen resolution 1920x1080 is also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agent scaffolding is described in detail: observation space (Section 2.3), action space via pyautogui (Section 2.4, Table 2), agent interaction loop with POMDP formalization (Section 2.1), history encoding using recent 3 observations in chat mode (Section 4.1), and special actions (WAIT, FAIL, DONE)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "A11y tree filtering is documented in detail (Section C.3, Table 13) with specific criteria for keeping nodes (tag types, visibility, availability, etc.). Screenshot downsampling procedures are described (Section 5.2). SoM implementation is detailed (Section C.4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Conclusion and Future Work' includes substantive discussion of limitations across four dedicated subsections: VLM capabilities, agent methodologies, safety challenges, and data/environment expansion. Quality control (Section 3.2) also acknowledges false positives/negatives."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: a11y tree quality varies across applications (Section 7), copyright prevents macOS evaluation (Appendix B.1), evaluation may have false positives/negatives that 'further investment of time and a more red teaming could further reduce' (Section 3.2), few-shot prompting scheme limitation (Section 4.1)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Scope boundaries are stated: Ubuntu focus with limited Windows supplementary tasks (Section 3.1), 8 representative applications selected with stated criteria (Appendix B.2), macOS excluded due to copyright (Appendix B.1), max 15 steps per task (Section 4.1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full benchmark data is released: task configurations, evaluation scripts, VM snapshots, and baseline implementations are available at https://os-world.github.io. Individual task results can be verified through the execution-based evaluation framework."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes task collection in detail: sources include 'forums, tutorials, guidelines' (listed in Table 9), selection criteria based on 'popularity, helpfulness, and diversity, revealed by the views and votes,' with cross-checking by two non-annotating authors per task."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluators are described as 'computer science major college students who possess basic software usage skills but have not been exposed to the samples or software before' (Section 3.4). However, how they were recruited, the total number, and whether they were compensated is not described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: source collection (Table 9), task selection criteria, annotation process (Section 3.2), cross-checking by two additional authors, four rounds of quality control checks with 400+ man-hours, human evaluation and baseline experiment iterations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgements section thanks various researchers for feedback but does not disclose any funding sources, grants, or financial support."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: The University of Hong Kong, CMU, Salesforce Research, and University of Waterloo. These are prominently displayed under author names."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Two authors are from Salesforce Research, but no funding relationship or potential conflicts are discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. No disclosure of patents, equity, or financial interests is provided despite Salesforce Research affiliation."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While specific API versions are stated (e.g., 'gpt-4-0125-preview'), the training data cutoff dates for the evaluated models are not explicitly stated. The paper does not discuss when each model's training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper integrates 84 tasks from existing benchmarks (NL2Bash, Mind2Web, SheetCopilot, PPTC, GAIA) but does not discuss whether the evaluated models could have seen these tasks during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis is provided. While the majority of tasks are newly created (reducing contamination risk), the 84 integrated tasks from existing public benchmarks (22.8% of the dataset) could be in the training data of evaluated models, and this is never discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human performance study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human performance evaluation study involving college students."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Evaluators are described only as 'computer science major college students who possess basic software usage skills' (Section 3.4). No age, gender, geographic distribution, years of experience, or detailed characterization is provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "Only minimal criteria mentioned: CS major, basic software skills, no prior exposure to samples or software. No formal inclusion/exclusion criteria, screening process, or eligibility requirements are documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human performance study is observational — participants complete tasks without experimental conditions or group assignments. No randomization is needed or applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "The human performance study measures completion ability on tasks, not a comparative experiment with conditions. Blinding is not applicable to this observational measurement."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information about participant attrition or dropout from the human evaluation study is reported. The number of evaluators who started vs completed all tasks is not stated."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or cost per task are reported despite extensive use of commercial APIs (GPT-4V, Gemini, Claude-3) across 369 tasks in multiple settings."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Annotation effort is quantified (1800 man-hours) but computational budget for running experiments (GPU hours, API spend, total inference time) is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. All results appear to be from single runs with temperature=1.0 and top_p=0.9, meaning outputs are stochastic but no multi-seed analysis is conducted."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is not stated. Results appear to be single-run numbers, but this is never explicitly confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (temperature=1.0, top_p=0.9, max_tokens=1500) are reported but no search budget is given. The paper mentions trying few-shot prompting first (2.79%) but doesn't systematically report the search space explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper justifies its configuration: 'Our prior experiments following VisualWebArena adopt few-shot prompting... but this resulted in poor performance (success rate of 2.79%)... We attribute the result to a lack of history encoding and change in the prompting scheme' (Section 4.1). The switch to chat mode with trajectory history is motivated."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so correction for multiple comparisons is not applicable. All comparisons are based on raw percentage comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baseline agents themselves using the same framework but do not acknowledge self-implementation bias. No independent evaluation or discussion of potential bias from authors implementing their own baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is not reported as a function of compute budget. Different models have vastly different computational costs (e.g., GPT-4V vs CogAgent) but no compute-normalized comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "As the benchmark's creators, the authors extensively discuss what it measures: real-world computer task completion across diverse applications. Table 4 compares construct dimensions with existing benchmarks. Section 3 motivates task design from real user scenarios. Section 3.4 validates task difficulty via human performance comparison with WebArena."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same scaffolding framework (pyautogui action space, same prompt structure, same observation processing) is used across all model comparisons within each input setting. Table 5 is organized by input setting, so model comparisons are within controlled scaffolding conditions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The paper does not address whether task solutions or related tutorials could appear in models' training data, particularly for the 84 tasks integrated from existing benchmarks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The a11y tree provides structured element information that agents might exploit in ways not representative of real use — this is never discussed as a potential leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss independence of integrated benchmark tasks from training data. 84 tasks from NL2Bash, Mind2Web, SheetCopilot, PPTC, and GAIA are included without verifying they are not in training sets of evaluated models."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, temporal splits, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "The best model achieves only 12.24% success rate on OSWorld, compared to 72.36% human performance.",
       "evidence": "Table 5 shows GPT-4 with a11y tree input achieves 12.24% overall success rate. Human performance is 72.36% (Section 3.4, Figure 4). Consistent across detailed per-app breakdowns in Table 14.",
       "supported": "strong"
     },
     {
       "claim": "Current LLMs and VLMs are far from capable of serving as computer assistants.",
       "evidence": "Table 5 shows all models range from 0.99% to 12.24%, with workflow tasks reaching only 6.57% at best. The gap to human performance (72.36%) is consistently large across all categories.",
       "supported": "strong"
     },
     {
       "claim": "Higher screenshot resolution typically leads to improved performance for VLM agents.",
       "evidence": "Figure 5 shows performance increases with resolution for screenshot-only setting. However, this is tested on only 10% of examples (about 37 tasks), and the SoM setting shows a non-monotonic pattern.",
       "supported": "moderate"
     },
     {
       "claim": "Longer text-based trajectory history improves performance, but screenshot-based history does not.",
       "evidence": "Figure 7 shows SoM performance increases with more a11y tree history, while screenshot-only performance remains flat or decreases. Tested on 10% of examples.",
       "supported": "moderate"
     },
     {
       "claim": "Agent performance varies significantly more across task categories than human performance.",
       "evidence": "Table 5 shows agent performance ranges from 0% to 41.67% across categories while human performance stays within 70-75% range across all categories.",
       "supported": "strong"
     },
     {
       "claim": "OSWorld tasks are more challenging and time-consuming than WebArena tasks.",
       "evidence": "Figure 4 shows median completion time of 111.94s vs WebArena's 35.38s, and human accuracy of 72.36% vs WebArena's 88%. However, WebArena comparison uses only 100 sampled examples.",
       "supported": "moderate"
     },
     {
       "claim": "VLM agents are not robust to perturbations of position, size, and clutter of application windows.",
       "evidence": "Figure 8 shows performance drops from 50.79% to 36.5% (position), 15.04% (size), and 25.39% (clutter) on a subset of 28 well-performing tasks.",
       "supported": "moderate"
     },
     {
       "claim": "More than 75% of GPT-4V agent failures involve mouse click inaccuracies.",
       "evidence": "Section 5.4 states 'Among the 550 failed examples from different settings in our sample, more than 75% exist mouse click inaccuracies.' Qualitative examples are provided in Appendix D.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OSWorld introduces the first scalable, real computer environment for multimodal agents spanning 369 tasks across Ubuntu applications. The best model (GPT-4) achieves only 12.24% success compared to 72.36% human performance, with workflow tasks proving especially difficult (6.57% best). Key bottlenecks identified include GUI grounding accuracy (75%+ failures involve mouse click errors), lack of application-specific operational knowledge, and inability to handle environmental noise from unexpected windows. Higher screenshot resolution and longer text-based trajectory history improve performance, while Set-of-Mark augmentation shows inconsistent benefits across models.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results in Table 5 and throughout the paper are single-run point estimates with no error bars, confidence intervals, or variance across runs. Given stochastic sampling (temperature=1.0), results could vary substantially across runs."
     },
     {
       "flag": "Self-implementation bias unacknowledged",
       "detail": "All baseline agents are implemented by the paper's authors using their framework. Lucic et al. (2018) showed that authors' implementations of baselines systematically underperform. This bias is not discussed."
     },
     {
       "flag": "Contamination risk for integrated tasks",
       "detail": "84 tasks (22.8%) are integrated from existing public benchmarks (NL2Bash, Mind2Web, SheetCopilot, PPTC, GAIA). These could appear in the training data of evaluated models, yet contamination is never analyzed or discussed."
     },
     {
       "flag": "Ablation studies on very small subsets",
       "detail": "Resolution (Figure 5) and history (Figure 7) ablations use only '10% of examples' (~37 tasks). Window perturbation (Figure 8) uses 28 cherry-picked well-performing tasks. These small samples make the ablation conclusions fragile."
     }
   ],
   "cited_papers": [
     {
       "title": "Mind2Web: Towards a Generalist Agent for the Web",
       "authors": ["Xiang Deng", "Yu Gu", "Boyuan Zheng"],
       "year": 2023,
       "arxiv_id": "2306.06070",
       "relevance": "Web agent benchmark with 2,350 tasks used as a comparison point and source of integrated tasks for OSWorld."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["Shuyan Zhou", "Frank F Xu", "Hao Zhu"],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Realistic web agent benchmark with execution-based evaluation, directly compared to OSWorld for task difficulty and human performance."
     },
     {
       "title": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks",
       "authors": ["Jing Yu Koh", "Robert Lo"],
       "year": 2024,
       "arxiv_id": "2401.13649",
       "relevance": "Multimodal web agent benchmark extending WebArena with visual tasks; influenced OSWorld's SoM and prompting methodology."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Influential code agent benchmark for real-world software engineering tasks, related to OSWorld's goal of evaluating agents on real computer tasks."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Multi-environment benchmark for LLM agents across isolated environments, compared to OSWorld's unified computer environment approach."
     },
     {
       "title": "CogAgent: A Visual Language Model for GUI Agents",
       "authors": ["Wenyi Hong", "Weihan Wang"],
       "year": 2023,
       "arxiv_id": "2312.08914",
       "relevance": "Specialized VLM for GUI agent tasks, evaluated as an open-source baseline in OSWorld."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "General AI assistant benchmark with 466 tasks requiring real-world tool use; source of integrated tasks for OSWorld."
     },
     {
       "title": "WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?",
       "authors": ["Alexandre Drouin", "Maxime Gasse"],
       "year": 2024,
       "arxiv_id": "2403.07718",
       "relevance": "Web agent benchmark for knowledge work tasks with 23k instances, compared to OSWorld's approach in Table 4."
     },
     {
       "title": "InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback",
       "authors": ["John Yang", "Akshara Prabhakar"],
       "year": 2023,
       "arxiv_id": "2306.14898",
       "relevance": "Interactive coding environment benchmark with execution feedback, related to OSWorld's execution-based evaluation approach."
     },
     {
       "title": "GPT-4V(ision) is a Generalist Web Agent, if Grounded",
       "authors": ["Boyuan Zheng", "Boyu Gou"],
       "year": 2024,
       "arxiv_id": "2401.01614",
       "relevance": "Study demonstrating GPT-4V as a web agent when properly grounded; influenced OSWorld's SoM and grounding analysis."
     },
     {
       "title": "Set-of-Mark Prompting Unleashes Extraordinary Visual Grounding in GPT-4V",
       "authors": ["Jianwei Yang", "Hao Zhang"],
       "year": 2023,
       "arxiv_id": "2310.11441",
       "relevance": "Visual prompting technique evaluated as an input method in OSWorld; shown to have inconsistent effectiveness for GUI agent tasks."
     },
     {
       "title": "WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models",
       "authors": ["Hongliang He", "Wenlin Yao"],
       "year": 2024,
       "arxiv_id": "2401.13919",
       "relevance": "End-to-end multimodal web agent approach relevant to OSWorld's investigation of vision-based agent capabilities."
     }
   ]
 }