ai-research-survey

scan.json (28038B)

---

 {
   "paper": {
     "title": "BrowserArena: Evaluating LLM Agents on Real-World Web Navigation Tasks",
     "authors": [
       "Sagnik Anupam",
       "Davis Brown",
       "Shuo Li",
       "Eric Wong",
       "Hamed Hassani",
       "Osbert Bastani"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.02418"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We have open-sourced our codebase at https://github.com/sagnikanupam/browserarena' in Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes constructed datasets (220 Expedia tasks, 80 BBC tasks, 100 TriviaQA questions), there is no explicit link to download these datasets. TriviaQA itself is public, but the generated task datasets and collected user annotations do not appear to be released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using BrowserUse with Playwright and the OpenRouter API platform, but does not provide a requirements.txt, Dockerfile, or detailed environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the GitHub repository is linked, the paper itself does not include a 'Reproducing Results' section or describe how to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% confidence intervals for the bootstrapped ELO ratings (Figure 2c, Appendix B: 'with 95% confidence intervals being calculated by bootstrapping for 100 rounds')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about models (e.g., R1 has highest ELO, o4-mini uses more strategies) but does not perform statistical significance tests. Differences are reported as raw numbers and percentages without p-values or statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw percentages (e.g., strategy usage rates, agreement rates) but does not report standardized effect sizes like Cohen's d or odds ratios to quantify the magnitude of differences between models."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample sizes (109 tasks, 25 tasks for evaluator agreement, 220/80/100 tasks for failure mode studies) are not justified through power analysis or any formal reasoning about adequacy. No discussion of whether 109 battles are sufficient for reliable ELO estimation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports single-run results for each model on the failure mode datasets. No variance across runs, standard deviations, or spread measures are reported for the strategy usage percentages or model performance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares five different LLM models (DeepSeek R1, Claude 3.7 Sonnet, Llama-4-Maverick, o4-mini, Gemini 2.5-Pro) against each other using pairwise comparisons, and also compares VLM judges against human annotators."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All five evaluated models (DeepSeek R1, Claude 3.7 Sonnet, Llama-4-Maverick, o4-mini, Gemini 2.5-Pro) are contemporary frontier models as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 describes input ablations for VLM-as-a-judge: 'trace-only evaluation improves GPT-4o's agreement with the baseline annotations by 10 percentage points (79% vs. 68% with GIFs and traces), while GIF-only input collapses performance to 48%.' This ablation examines the contribution of different input modalities."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation approaches: average win rate, ELO ratings, pairwise win fractions, inter-annotator agreement, and per-failure-mode strategy usage rates."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is central to this work. Users vote on model outputs (Section 4.1), additional annotators evaluate a subset for inter-annotator agreement (Section 4.3), and step-level human annotations are collected for failure mode analysis (Section 5)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This paper is a benchmark/platform evaluation, not a model training study. There is no training phase that would necessitate a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-model breakdowns for all analyses: pairwise win fractions (Figure 2d), per-model strategy usage rates for captcha solving (Table 3), per-model pop-up banner scenarios (Table 4), and per-model direct navigation actions (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire Section 5 is dedicated to identifying and analyzing failure modes (captcha resolution, pop-up banner closure, direct navigation). Specific failure examples are discussed, such as R1 never detecting pop-up banners due to lacking multimodal capabilities."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: VLMs are unreliable judges (Section 4.4), multimodality hurts GPT-4o judge reliability, R1 cannot detect pop-up banners, and Gemini 2.5 fails 50% of TriviaQA tasks. DeepSeek-R1 'consistently misrepresents its ability to close pop-up banners.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about three failure modes (captcha, pop-ups, direct navigation) are supported by Sections 5.2-5.4. The claim about o4-mini deploying wider strategies is supported by Table 3. The claim about DeepSeek-R1 misleading users about pop-up closure is supported by Table 4 (0% banner detection but 53.75% marked as completed)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about R1's failure to detect banners being due to its lack of multimodal capabilities ('indicating that multi-modal reasoning ability is required for detecting the privacy policy pop-up'). This is justified by R1 being the only non-multimodal model and showing 0% banner detection. The VLM ablation study (Section 4.4) also supports causal claims about modality contributions through controlled removal of input components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 7) explicitly bounds generalization: results depend on the BrowserUse system, failure modes may be system-specific, and specific tasks that trigger failure modes 'may be different depending on the system configuration.' The paper does not overclaim beyond its tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its main findings. For example, R1's high completion rate despite not detecting banners could be due to factors other than lack of multimodal capabilities (e.g., different optimization pressures, instruction-following differences). The ELO ranking differences could be influenced by task distribution biases. No systematic consideration of confounds is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists models as 'DeepSeek R1, Anthropic Claude 3.7 Sonnet:Thinking, Meta Llama-4-Maverick, OpenAI o4-mini, and Google Gemini 2.5-Pro-Preview-03-25.' While Gemini includes a preview date, the others lack specific version strings, API snapshot dates, or model IDs. Marketing names like 'o4-mini' and 'Claude 3.7 Sonnet:Thinking' do not include snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in the appendices: task generation prompts for captcha (Appendix D) and pop-up banner tasks (Appendix E), the captcha analysis judge prompt (Appendix F), and the pop-up banner judge prompt (Appendix G). The system prompts and user prompts are given verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report temperature, top-p, or other sampling parameters for the LLMs used in the agent evaluations or for the o4-mini judge. The only hyperparameters mentioned are clustering parameters for failure mode discovery (Appendix I) and 'temperature = 0' for GPT-4o feature evaluation, but the main experiment settings are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The BrowserUse scaffolding is described in detail: Section 3 explains the agent receives task, previous steps, URL, open tabs, HTML elements with indices, and optionally screenshots; it outputs JSON with goal self-evaluation, memory, next goal, and actions. Table 1 in Appendix C lists all 22 permitted actions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents data collection and filtering: 213 valid responses approved from Prolific, filtered to 109 responses from 98 users 'due to system outages, logging issues, and invalid responses.' The collection was done in 3 batches with described compensation. Task generation for failure mode studies is documented in Appendices D and E."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is a dedicated 'Limitations' section that discusses two substantive limitations: dependence on the BrowserUse system and system-specificity of discovered failure modes."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'equipping models with different or more powerful capabilities may help improve agent capabilities' and 'the failure modes we discover may be system specific... it may be possible to reduce the likelihood of encountering captchas on a particular website by using rotating proxies.' These are study-specific threats, not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section discusses system-specific concerns, the paper does not explicitly state what its results do NOT show. For example, it does not explicitly state that the ELO rankings are not generalizable beyond BrowserUse, or that the failure modes do not represent an exhaustive taxonomy. The scope boundaries are implicit rather than explicit."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw user votes, step-level annotations, agent traces, and generated task datasets are not made available for independent verification. Only aggregate statistics are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection procedure in detail: Prolific recruitment with geographic criteria, approval rate filters, 3 batches, compensation rates, task submission and annotation workflow. Section 4.3 describes the evaluator agreement study collection. Appendices D-G describe the failure mode dataset generation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'We recruit users on Prolific from United Kingdom, United States, Australia, Canada, and New Zealand with response approval rates between 90-100%.' Compensation details are in Appendix H. The recruitment channels and selection criteria are clearly described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: 213 valid responses → 109 kept (after filtering for system outages, logging issues, invalid responses) from 98 users. The failure mode analysis pipeline is described in Section 5.1 with clustering methods. The judge evaluation pipeline for each failure mode is described in detail."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with University of Pennsylvania. Since no author is affiliated with any of the companies whose models are evaluated (OpenAI, Anthropic, Meta, Google, DeepSeek), there is no vendor conflict to disclose beyond the listed affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed, it cannot be determined whether the funder is independent of the outcome. The paper evaluates models from multiple major AI companies but does not state who funded the work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates five LLMs on various tasks including TriviaQA questions, but does not state the training data cutoff dates for any of the models. This is relevant because TriviaQA answers could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses TriviaQA (published 2017) questions to test agents but does not discuss whether these questions and answers appear in the training data of the tested models. Models trained after 2017 very likely have seen TriviaQA content."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TriviaQA was published in 2017 and is widely known; all tested models were likely trained on data including TriviaQA content. The paper does not address this contamination risk for the direct navigation experiment (Section 5.4)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the user study. No link to OSF, AsPredicted, or any pre-registration platform is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite the study collecting data from human participants via Prolific."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Participant demographics beyond geographic eligibility (UK, US, Australia, Canada, New Zealand) and approval rates (90-100%) are not reported. No information on age, gender, technical experience, or other demographic characteristics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 specifies: participants from UK, US, Australia, Canada, and New Zealand with 90-100% approval rates on Prolific. Specific task submission criteria are described (interactive tasks only, not search tasks). Examples of valid and invalid tasks are given in Appendix A."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes: 'two LLMs are chosen at random with uniform probability.' Users interact with randomly paired models. For the evaluator agreement study, 25 tasks are 'randomly selected' (Section 4.3)."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "The BrowserArena platform follows the Chatbot Arena design where users compare anonymous agents (Model A vs. Model B) without knowing which LLM powers each. This is implicit in the design (Figure 1 shows 'Model A' and 'Model B' labels), consistent with arena-style blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports attrition: 'We approved a total of 213 valid responses, ultimately keeping 109 responses from 98 users due to system outages, logging issues, and invalid responses.' The reasons for dropout are provided."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report API costs, token usage, or latency for running the BrowserUse agents across the evaluated models. This is relevant given that each task involves multiple API calls per agent."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget (API spend, GPU hours for local models, or infrastructure costs) is not stated. Only participant compensation costs are partially described in Appendix H."
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek R1 achieves the highest ELO rating among the five tested models on user-submitted web browsing tasks, despite being the only model without multimodal capabilities.",
       "evidence": "Figure 2 shows R1 with ELO rating 1168 (highest), average win rate 0.74, compared to Llama-4 (1117), o4-mini (969), Claude 3.7 (907), Gemini 2.5 (840). Based on 109 user-submitted tasks (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "There is a significant gap between VLM preferences and human preferences for evaluating web agent outputs.",
       "evidence": "Section 4.4 and Figure 3 show human-human agreement at 63.2%, GPT-4o agreement with baseline at 68%, o4-mini at 58%. When removing ties, human agreement rises to 100% and inter-annotator to 83%, but VLM results are not reported without ties.",
       "supported": "moderate"
     },
     {
       "claim": "Adding visual GIF inputs hurts GPT-4o's judge reliability compared to trace-only evaluation.",
       "evidence": "Section 4.4: 'trace-only evaluation improves GPT-4o's agreement with the baseline annotations by 10 percentage points (79% vs. 68% with GIFs and traces), while GIF-only input collapses performance to 48%.' Based on 25 randomly selected tasks.",
       "supported": "moderate"
     },
     {
       "claim": "o4-mini deploys a wider variety of captcha circumvention strategies than other models.",
       "evidence": "Table 3 in Appendix J shows o4-mini uses all 14 listed strategies at least once, including 3 strategies no other model uses (Text-only Rendering, Public Proxy, Internet Archive). Based on 220 Expedia tasks (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "DeepSeek R1 consistently misrepresents its ability to close pop-up banners, marking tasks as completed without detecting the banners.",
       "evidence": "Table 4: R1 shows 0% banner detection rate but 53.75% task completion rate, the highest of all models. This is attributed to R1's lack of multimodal capabilities (Section 5.3).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "qualitative"
   ],
   "key_findings": "BrowserArena introduces a live open-web evaluation platform using Arena-style pairwise comparisons and step-level human feedback to evaluate LLM web agents. DeepSeek R1 achieves the highest ELO rating despite lacking multimodal capabilities, while VLM judges show significant disagreement with human preferences. Three consistent failure modes are identified (captcha solving, pop-up banner closure, direct navigation), with notable differences in how models handle them -- o4-mini uses the widest variety of captcha avoidance strategies, while R1 cannot detect pop-up banners but falsely marks tasks as completed at the highest rate.",
   "red_flags": [
     {
       "flag": "Very small sample size for leaderboard",
       "detail": "The ELO leaderboard is based on only 109 battles across 5 models, yielding very few pairwise comparisons per pair (6-19 battles per pair as shown in Figure 2b). This is far fewer than the thousands of battles typically used in Chatbot Arena, making the rankings unreliable. The confidence intervals on ELO ratings are wide and overlapping."
     },
     {
       "flag": "LLM-as-judge for failure mode analysis",
       "detail": "The failure mode quantification in Sections 5.2-5.4 relies on o4-mini as a judge to classify agent traces. The reliability of o4-mini as a judge is not validated against human annotations for these specific classification tasks, despite the paper's own finding that VLMs are unreliable judges (Section 4.4)."
     },
     {
       "flag": "Synthetically generated evaluation datasets",
       "detail": "The failure mode evaluation datasets (220 Expedia tasks, 80 BBC tasks) are largely generated by GPT 4.1, not written by real users. Only 20 of 220 captcha tasks came from human templates. The ecological validity of LLM-generated tasks as proxies for real user behavior is not established."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims between models are based on raw percentage differences without any significance tests. With the small sample sizes, many observed differences could easily be due to random variation."
     },
     {
       "flag": "Contamination risk for TriviaQA experiment",
       "detail": "Section 5.4 uses 100 TriviaQA questions (published 2017) to study direct navigation behavior. All evaluated models were trained well after 2017 and likely have TriviaQA content in their training data. The paper does not acknowledge this, which affects the interpretation of 'direct answering' vs. 'search' behaviors."
     }
   ],
   "cited_papers": [
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng", "Ying Sheng"],
       "year": 2024,
       "relevance": "Foundational platform for pairwise human evaluation of LLMs, which BrowserArena builds upon and extends to web agent evaluation."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["Shuyan Zhou", "Frank F. Xu", "Hao Zhu"],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Key benchmark for web agent evaluation using self-hosted website clones, which BrowserArena aims to improve upon with open-web evaluation."
     },
     {
       "title": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks",
       "authors": ["Jing Yu Koh", "Robert Lo", "Lawrence Jang"],
       "year": 2024,
       "arxiv_id": "2401.13649",
       "relevance": "Extension of WebArena for visually-grounded web tasks, relevant to evaluation of multimodal web agents."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "relevance": "Open platform for AI agents in software engineering, demonstrating the broader trend of open-ended agent evaluation environments."
     },
     {
       "title": "Copilot Arena: A Platform for Code LLM Evaluation in the Wild",
       "authors": ["Wentao Chi", "Vivian Chen", "Anastasios N. Angelopoulos"],
       "year": 2025,
       "arxiv_id": "2502.09328",
       "relevance": "Arena-style evaluation platform for code LLMs, parallel methodology to BrowserArena applied to code generation."
     },
     {
       "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
       "authors": ["Tianbao Xie", "Danyang Zhang", "Jixuan Chen"],
       "year": 2024,
       "relevance": "Benchmark for evaluating multimodal agents in real computer environments, relevant to understanding agent capabilities beyond web browsing."
     },
     {
       "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents",
       "authors": ["Jason Wei", "Zhengdong Sun", "Sean Papay"],
       "year": 2025,
       "arxiv_id": "2504.12516",
       "relevance": "Recent challenging QA benchmark for web browsing agents that evaluates hard information retrieval tasks."
     },
     {
       "title": "Survey on Evaluation of LLM-Based Agents",
       "authors": ["Asaf Yehudai", "Lilach Eden", "Alan Li"],
       "year": 2025,
       "arxiv_id": "2503.16416",
       "relevance": "Survey covering evaluation methodologies for LLM-based agents, directly relevant to understanding the landscape of agent benchmarks."
     },
     {
       "title": "The BrowserGym Ecosystem for Web Agent Research",
       "authors": ["David Chezelles", "Thibault Le Sellier", "Maxime Gasse"],
       "year": 2024,
       "arxiv_id": "2412.05467",
       "relevance": "Open-web evaluation ecosystem for web agents that BrowserArena extends with human preference evaluation."
     },
     {
       "title": "WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models",
       "authors": ["Hongliang He", "Wenlin Yao", "Kaixin Ma"],
       "year": 2024,
       "arxiv_id": "2401.13919",
       "relevance": "End-to-end web agent using multimodal models, relevant benchmark for evaluating web navigation capabilities."
     },
     {
       "title": "Search Arena: Analyzing Search-Augmented LLMs",
       "authors": ["Martin Miroyan", "Ting-Hao Wu", "Luca King"],
       "year": 2025,
       "relevance": "Extension of arena-style evaluation to search-augmented LLMs, complementary to BrowserArena's web agent evaluation approach."
     },
     {
       "title": "Windows Agent Arena: Evaluating Multi-Modal OS Agents at Scale",
       "authors": ["Rogerio Bonatti", "Dan Zhao", "Francesco Bonacci"],
       "year": 2024,
       "arxiv_id": "2409.08264",
       "relevance": "Benchmark for multimodal OS-level agents, extending agent evaluation beyond web browsing to general computer use."
     }
   ]
 }