ai-research-survey

scan.json (33210B)

---

 {
   "paper": {
     "title": "Mind2Web: Towards a Generalist Agent for the Web",
     "authors": [
       "Xiang Deng",
       "Yu Gu",
       "Boyuan Zheng",
       "Shijie Chen",
       "Samuel Stevens",
       "Boshi Wang",
       "Huan Sun",
       "Yu Su"
     ],
     "year": 2023,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2306.06070",
     "doi": "10.48550/arXiv.2306.06070"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Mind2Web introduces the first large-scale dataset (2,350 tasks, 137 websites, 31 domains) for developing and evaluating generalist web agents on real-world websites. The proposed MINDACT framework, which uses a small LM (DeBERTa) for candidate element filtering followed by an LLM for action prediction via multi-choice QA, achieves 52.0% step success rate on seen websites but drops to ~39% on unseen websites/domains, revealing that generalization challenges stem from website design diversity rather than domain differences. GPT-4 with only 3-shot in-context learning matches fine-tuned Flan-T5 on cross-website/domain settings, suggesting promising LLM-based paths toward generalist web agents.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code released under MIT License at https://github.com/OSU-NLP-Group/Mind2Web, as documented in the artifacts table in the Supplementary (Section A)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Training data released on HuggingFace (CC BY 4.0) at https://huggingface.co/datasets/osunlp/Mind2Web, and test data available via a separate link with password. Both listed in the artifacts table."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (4×A100 80GB, A6000 48GB) and libraries (Sentence-Transformers, Transformers, Playwright) but does not provide a requirements.txt, Dockerfile, or environment specification listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper itself does not contain step-by-step reproduction instructions. Hyperparameters are listed in Table 4 and model links are provided, but no explicit 'how to reproduce' guide is given in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Table 2 report only point estimates with no confidence intervals or error bars. Table 5 in the appendix reports mean ± std over 5 seeds for an auxiliary experiment, but the primary results table has no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. All comparisons between methods (e.g., MINDACT vs. baselines, GPT-4 vs. Flan-T5) are based solely on comparing raw numbers without any hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance numbers with baselines for context (e.g., 'over 10% absolute gap in step SR between Cross-Task and Cross-Website settings'). Table 2 provides all raw numbers enabling direct comparison of magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of tasks (2,350), websites (137), or the GPT-4 evaluation subset (50 tasks per setting). No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Table 2) are single-run numbers without variance. Table 5 in the appendix reports std dev over 5 seeds for Flan-T5 models as an auxiliary experiment, but the primary reported results lack any spread measure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares MINDACT against a classification baseline (DeBERTa for element selection) and a generation baseline (Flan-T5 autoregressive generation), plus GPT-3.5 and GPT-4 via in-context learning."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DeBERTa-v3, Flan-T5 (2022), GPT-3.5-turbo, and GPT-4 — all state-of-the-art or near-state-of-the-art at the time of submission. The classification approach mirrors prior work [14, 35]."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The two-stage pipeline is effectively ablated: classification-only baseline tests candidate generation alone, generation baseline tests direct autoregressive prediction, and MINDACT combines both stages. Table 2 shows the contribution of each component. Multi-choice vs direct generation is also compared (Figure 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Element Accuracy, Operation F1, Step Success Rate, and (whole-task) Success Rate. These capture different aspects of performance (element selection, operation correctness, per-step, and end-to-end)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated by comparing predicted actions against ground-truth annotation. No human evaluation of the system's outputs is conducted. The paper acknowledges offline evaluation limitations in Section 6."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Three held-out test sets with clear separation: TestCross-Task (random 20% split, 252 tasks), TestCross-Website (held-out websites, 177 tasks), TestCross-Domain (held-out domains, 912 tasks). Section 4.1 describes the splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides per-website step success rate breakdowns across all three evaluation settings. Table 2 also reports results separately for each of the three generalization levels."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses failure modes: GPT-3.5's propensity to select the 'None' option, the overall low task success rate even for the best models, and the challenge of grounding commonsense knowledge into specific website actions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: GPT-3.5 underperforms fine-tuned smaller models (Table 2), the generation baseline underperforms classification despite larger model size, zero-shot Flan-T5XL 'fails to perform the task without fine-tuning' (Table 6, Appendix D.2), and overall task success rate is very low (≤7.1%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'first dataset' for generalist web agents (Table 1 comparison), 'diverse domains/websites/tasks' (137 websites, 31 domains, 2350 tasks), 'filtering with small LM improves effectiveness' (Table 2 MINDACT vs baselines), 'decent level of performance even on unseen websites/domains' (Table 2 cross-website/domain results), 'substantial room to improve' (low task success rates)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that small-LM filtering improves LLM effectiveness — is supported by controlled comparison: same LLM (Flan-T5) with and without candidate generation, with the only difference being the filtering stage. The ablation design adequately isolates the filtering component's contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately bounded. The abstract states 'a decent level of performance' rather than overstating, and explicitly notes 'substantial room to improve.' Section 6 discusses specific limitations including English-only websites, US-centric selection, and MTurk annotator bias."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for observed results. For instance, the similar performance on Cross-Website and Cross-Domain is attributed to 'challenges primarily stemming from website design diversity' without considering alternatives like dataset difficulty distribution or sample size effects."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's metrics (element accuracy, step success rate, task success rate) directly measure what is claimed — the ability to select correct web elements and complete multi-step tasks. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are specified precisely with HuggingFace links (deberta-v3-base, flan-t5-base/large/xl). However, GPT-3.5-turbo and GPT-4 are referenced only by marketing names without snapshot dates or API versions, despite model behavior changing across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 in the appendix provides the full prompt used for GPT in-context learning, including system message, three demonstration examples with expected outputs. Figures 4 and 5 show the input templates for candidate generation and action prediction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 in the appendix reports all hyperparameters: batch size (32), epochs (5), learning rates (3e-5 for candidate generation, 5e-5 for action prediction), and temperature (0) and number of demonstrations (3) for GPT models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "MINDACT is a two-stage model pipeline (candidate generation → action prediction), not an agentic scaffolding system. No tools, retry logic, memory, or feedback mechanisms are involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents preprocessing: 'We apply simple heuristics to clean the raw HTML documents, keeping only elements that are visible and carry substantial semantic meaning... This effectively reduces the average number of elements from 1,135 to 580, while still maintaining an overall recall of 94.7% for the target element.' Appendix C.1 describes element normalization for evaluation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Potential Societal Impact' provides extensive discussion covering data diversity, multimodal information, interaction dynamics, human-agent interaction, offline/online evaluation, and safety in deployment."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: 'predominantly comprises English-language websites primarily used in the U.S.', MTurk annotators 'might be biased towards a group that is more proficient in web use', offline evaluation 'potentially leading to false negatives due to the existence of multiple paths for completing the same task.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what was not tested: multimodal understanding not used, no interactive/conversational setting, no live website evaluation, no non-English websites, no users with accessibility challenges. Each limitation is framed as a specific boundary with a future direction."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset including MHTML files, DOM snapshots, HAR files, and trace files is released on HuggingFace (CC BY 4.0), enabling independent verification of the raw data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the four-stage collection process in detail: website selection (by popularity from similarweb.com, 3-5 per domain), task proposal (MTurk with ChatGPT seed tasks), task demonstration (Playwright-based tool), and task verification (by authors)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 describes annotator recruitment: Amazon MTurk platform, minimum 1,000 approved HITs with >98% approval rate, compensation at $10.10/hr estimated rate, $0.80 per approved task, consent form required, qualification assessment with questionnaire and test demonstrations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from collection to final dataset is documented with counts at each stage: 2,411 total collected tasks → 61 discarded → 2,350 retained, with 390 task descriptions refined and 187 instances with extraneous steps removed (Section 2.2). Preprocessing reduces elements from 1,135 to 580 average (Section 4.2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists funding: 'sponsored in part by NSF OAC 2112606, NSF CAREER #1942980, ARL W911NF2220144 and Ohio Supercomputer Center.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with The Ohio State University, clearly stated in the paper header. No commercial product is being evaluated, so no undisclosed product-company conflicts exist."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from NSF (National Science Foundation) and ARL (Army Research Laboratory) — government agencies with no financial stake in the benchmark results or any specific model's performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Absence of a declaration is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-3.5-turbo and GPT-4 on their benchmark but does not state the training data cutoff dates for either model. While the benchmark is new, the underlying websites existed prior to model training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the pre-trained models' training data and the test websites/tasks. While the task annotations are new, the website content itself may have appeared in GPT training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is newly created, making direct contamination of task annotations unlikely. However, the paper does not explicitly discuss this or address whether GPT models may have encountered the underlying website structures during pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The MTurk workers are data collection annotators, not study participants. The paper studies web agent performance, not human behavior. IRB exemption confirms this is not formal human subjects research."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "While the paper mentions IRB exemption for the annotation process (Appendix B.1), the MTurk workers are dataset annotators, not subjects of a study on human behavior."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. The MTurk annotators are data collectors, not subjects of investigation."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. Annotator qualification criteria (1,000+ HITs, 98%+ approval) are for data quality, not study design."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. This is a dataset and benchmark paper, not an experimental study of human behavior."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. Blinding is not applicable to a dataset construction effort."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. Task attrition (61/2,411 discarded) is dataset quality control, not participant dropout."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions GPT-4's 'high operational cost remains a concern' and that they could only evaluate 50 tasks per setting 'due to limited budget,' but no specific dollar amounts, token counts, or per-example costs are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (4×A100 80GB, A6000 48GB) but total training time, GPU hours, or API spend are not quantified. No total computational budget is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix D.1) reports results across 5 random seeds with mean and standard deviation, showing sensitivity is small (std < 1.0 for all settings)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1: 'we show the average and standard deviation of 5 runs with different random seeds.' However, it is unclear whether the main Table 2 results represent a single run or the mean."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Table 4 lists fixed hyperparameters without explaining how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the specific hyperparameters (learning rate 3e-5/5e-5, batch size 32, 5 epochs) were chosen. No mention of selection on a validation set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (DeBERTa classification, Flan-T5 generation) and compare against their own MINDACT framework without acknowledging the potential bias of evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 86M parameters (DeBERTa) to GPT-4 (unknown size). While Flan-T5 results across three sizes (base/large/xl) show scaling effects, the compute differences between methods are not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses offline evaluation limitations: 'the task will fail immediately if an action was not cached during data collection, potentially leading to false negatives.' The paper also acknowledges the gap between offline evaluation and live website interaction, and discusses missing multimodal signals."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "MINDACT uses the same pipeline architecture across all LLM comparisons (same candidate generation + same multi-choice format). No scaffolding confound exists because the evaluation framework is held constant."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether GPT models' training data includes information from the real-world websites used in the benchmark. While the task annotations are new, the website content existed before model training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (providing HTML snippets, candidate elements) leaks more information than would be available in a realistic deployment scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether tasks from the same website in training and test sets share structural patterns that could inflate generalization estimates. The Cross-Task setting explicitly includes same-website tasks in training and test."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Mind2Web is the first dataset for developing and evaluating generalist agents for the web, spanning 137 websites, 31 domains, and 2,350 tasks.",
       "evidence": "Table 1 comparison with existing datasets (MiniWoB++, WebShop, RUSS, PixelHelp, META-GUI, MoTIF) shows Mind2Web is unique in combining real-world websites, high diversity, and high-level task descriptions. Section 2 details the dataset composition.",
       "supported": "strong"
     },
     {
       "claim": "MINDACT with multi-choice QA formulation substantially outperforms direct classification and generation baselines.",
       "evidence": "Table 2 shows MINDACT with Flan-T5XL achieves 52.0% step SR on Cross-Task vs. 26.8% element accuracy for classification and 17.5% step SR for generation baseline.",
       "supported": "strong"
     },
     {
       "claim": "Small LM filtering as a first stage significantly improves LLM effectiveness for web action prediction.",
       "evidence": "Table 2: DeBERTa candidate generation achieves 88.9%/85.3%/85.7% Recall@50, and the two-stage MINDACT outperforms single-stage baselines by large margins (Section 4.3).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 achieves performance comparable to fine-tuned Flan-T5 on cross-website and cross-domain settings with only 3-shot in-context learning.",
       "evidence": "Table 2 shows GPT-4 at 30.1%/26.4% step SR vs Flan-T5XL at 38.9%/39.6% on Cross-Website/Cross-Domain. The claim of being 'on par' for element selection is supported, but step SR shows a gap. Only evaluated on 50 tasks per setting (Appendix D.3).",
       "supported": "moderate"
     },
     {
       "claim": "Generalization challenges primarily stem from website design diversity rather than domain specifics.",
       "evidence": "Table 2 and Figure 6 show Cross-Website (38.9% step SR) and Cross-Domain (39.6% step SR) are notably similar despite Cross-Domain being a harder setting. The paper argues this suggests domain knowledge is less important than website-specific grounding.",
       "supported": "moderate"
     },
     {
       "claim": "The data collection pipeline produces high-quality data, with only 61/2,411 tasks discarded and 94.7% target element recall after preprocessing.",
       "evidence": "Section 2.2 details the four-stage pipeline with verification by authors. Section 4.2 reports preprocessing recall. However, no inter-annotator agreement metrics are provided.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small GPT-4 evaluation set",
       "detail": "GPT-4 was evaluated on only 50 tasks per setting due to budget constraints. This is too small to draw reliable conclusions about comparative performance, yet the paper makes claims about GPT-4 being 'on par' with fine-tuned models based on these results."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All method comparisons in Table 2 rely on comparing raw numbers without any statistical testing. Given the variance shown in Table 5 (std up to 0.8), some differences between methods may not be statistically significant."
     },
     {
       "flag": "No inter-annotator agreement",
       "detail": "Despite being a dataset paper, no inter-annotator agreement metrics are reported. Tasks were verified by authors, but no systematic measurement of annotation consistency (e.g., Cohen's kappa for action correctness) is provided."
     },
     {
       "flag": "Main results lack variance reporting",
       "detail": "Table 2 presents single-run results for the main experiments. While Table 5 in the appendix shows multi-seed results for an auxiliary analysis, the primary results table lacks error bars or confidence intervals."
     }
   ],
   "cited_papers": [
     {
       "title": "Do as I can, not as I say: Grounding language in robotic affordances",
       "authors": ["Michael Ahn", "Anthony Brohan", "Noah Brown"],
       "year": 2022,
       "arxiv_id": "2204.01691",
       "relevance": "Foundational work on grounding language in robotic actions (SayCan), relevant to agentic AI grounding capabilities."
     },
     {
       "title": "Reinforcement learning on web interfaces using workflow-guided exploration",
       "authors": ["Evan Zheran Liu", "Kelvin Guu", "Panupong Pasupat", "Tianlin Shi", "Percy Liang"],
       "year": 2018,
       "relevance": "MiniWoB++ benchmark for web RL agents — directly comparable prior work on web agent evaluation."
     },
     {
       "title": "WebShop: Towards Scalable Real-World Web Interaction with Grounded Language Agents",
       "authors": ["Shunyu Yao", "Howard Chen", "John Yang", "Karthik Narasimhan"],
       "year": 2022,
       "arxiv_id": "2207.01206",
       "relevance": "Simulated web shopping environment with grounded language agents — key prior benchmark for web agents."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Influential agentic framework combining reasoning and acting with LLMs, foundational for web agent architectures."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "LLMs learning to use external tools autonomously — directly relevant to agentic tool-use capabilities."
     },
     {
       "title": "Augmented language models: a survey",
       "authors": ["Grégoire Mialon", "Roberto Dessì", "Maria Lomeli"],
       "year": 2023,
       "arxiv_id": "2302.07842",
       "relevance": "Survey of augmented LLMs covering tool use, retrieval, and reasoning — maps the landscape Mind2Web contributes to."
     },
     {
       "title": "Tool learning with foundation models",
       "authors": ["Yujia Qin", "Shengding Hu", "Yankai Lin"],
       "year": 2023,
       "arxiv_id": "2304.08354",
       "relevance": "Comprehensive framework for foundation model tool learning, directly relevant to web agents as tool users."
     },
     {
       "title": "ToolkenGPT: Augmenting frozen language models with massive tools via tool embeddings",
       "authors": ["Shibo Hao", "Tianyang Liu", "Zhen Wang", "Zhiting Hu"],
       "year": 2023,
       "arxiv_id": "2305.11554",
       "relevance": "Method for augmenting LLMs with tools via embeddings, relevant to practical agentic tool integration."
     },
     {
       "title": "LLM-Planner: Few-shot grounded planning for embodied agents with large language models",
       "authors": ["Chan Hee Song", "Jiaman Wu", "Clayton Washington", "Brian M. Sadler", "Wei-Lun Chao", "Yu Su"],
       "year": 2022,
       "arxiv_id": "2212.04088",
       "relevance": "Few-shot grounded planning with LLMs for embodied agents — directly relevant to LLM-based agent planning."
     },
     {
       "title": "Don't generate, discriminate: A proposal for grounding language models to real-world environments",
       "authors": ["Yu Gu", "Xiang Deng", "Yu Su"],
       "year": 2022,
       "arxiv_id": "2212.09736",
       "relevance": "Proposes discrimination over generation for grounding LMs in real environments — foundational idea behind MINDACT's multi-choice formulation."
     },
     {
       "title": "Understanding HTML with large language models",
       "authors": ["Izzeddin Gur", "Ofir Nachum", "Yingjie Miao"],
       "year": 2023,
       "relevance": "LLMs for HTML understanding — directly relevant prior work on the core technical challenge of web agent grounding."
     },
     {
       "title": "Scaling instruction-finetuned language models",
       "authors": ["Hyung Won Chung", "Le Hou", "Shayne Longpre"],
       "year": 2022,
       "arxiv_id": "2210.11416",
       "relevance": "Flan-T5 paper — the backbone model used in MINDACT experiments, relevant to instruction-tuned LLM capabilities."
     },
     {
       "title": "ALFRED: A benchmark for interpreting grounded instructions for everyday tasks",
       "authors": ["Mohit Shridhar", "Jesse Thomason", "Daniel Gordon"],
       "year": 2020,
       "relevance": "Benchmark for grounded instruction following in embodied settings — related evaluation paradigm for agent grounding."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Dataset and model code are released and directly usable by researchers building web agents, though not yet a deployable end-user tool."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Fills a clear gap in web agent benchmarks but doesn't challenge conventional wisdom — results confirming LLMs struggle with real-world web tasks are expected."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Section 6 briefly raises safety concerns about autonomous web agents (financial transactions, CAPTCHA bypassing, malicious use) but this is not the paper's focus."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or conflicts — a straightforward dataset and benchmark contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code, data, and trained models are open-sourced on GitHub and HuggingFace, enabling researchers to download and experiment immediately."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From the OSU NLP Group, a respected academic lab but not a household-name organization like OpenAI or Google DeepMind."
     }
   }
 }