scan.json (17948B)
1 { 2 "paper": { 3 "title": "Measuring and Mitigating Overreliance is Necessary for Building Human-Compatible AI", 4 "authors": [ 5 "Lujain Ibrahim", 6 "Katherine M. Collins", 7 "Sunnie S. Y. Kim", 8 "Anka Reuel", 9 "Max Lamparth", 10 "Kevin Feng", 11 "Lama Ahmad", 12 "Prajna Soni", 13 "Alia El Kattan", 14 "Merlin Stein", 15 "Siddharth Swaroop", 16 "Ilia Sucholutsky", 17 "Andrew Strait", 18 "Q. Vera Liao", 19 "Umang Bhatt" 20 ], 21 "year": 2025, 22 "venue": "arXiv preprint", 23 "arxiv_id": "2509.08010" 24 }, 25 "scan_version": 2, 26 "active_modules": [], 27 "methodology_tags": ["theoretical"], 28 "key_findings": "This position paper argues that measuring and mitigating overreliance must become central to LLM research and deployment. It consolidates risks at individual and societal levels (high-stakes errors, deskilling, governance challenges, cultural homogenization), identifies three gaps in existing overreliance measurement approaches (interactive use breaks single-point measures, complex outputs defy binary metrics, general-purpose nature undermines ground truth), and proposes mitigation strategies at model, system, and user levels. The paper distinguishes overreliance from trust and overdependence, grounding the discussion in the judge-advisor framework tradition.", 29 "claims": [ 30 { 31 "claim": "Existing measurement approaches for overreliance break down for LLMs due to interactive use, complex text-based outputs, and general-purpose nature.", 32 "evidence": "Section 5.1 argues that LLMs undermine discrete decision points (judge-advisor framework), binary correctness assumptions, and stable notions of 'appropriate reliance' across diverse tasks.", 33 "supported": "moderate" 34 }, 35 { 36 "claim": "Overreliance on LLMs poses both individual risks (high-stakes errors, deskilling) and societal risks (governance challenges, cultural homogenization).", 37 "evidence": "Section 3 provides examples including hallucinated legal citations (Mata v. Avianca), AI misdiagnosis, and LLM-assisted writers producing less diverse ideas (ref [5]), but these are drawn from existing literature rather than new empirical work.", 38 "supported": "moderate" 39 }, 40 { 41 "claim": "As AI becomes more capable, detecting errors becomes more difficult, making overreliance measurement more essential rather than less.", 42 "evidence": "Section 7 (Alternative views) argues this point conceptually, citing [10] (Bender et al.) on convincing outputs, but provides no new empirical evidence.", 43 "supported": "weak" 44 } 45 ], 46 "checklist": { 47 "artifacts": { 48 "code_released": { 49 "applies": false, 50 "answer": false, 51 "justification": "This is a theoretical position paper with no code, experiments, or computational artifacts to release." 52 }, 53 "data_released": { 54 "applies": false, 55 "answer": false, 56 "justification": "No data was collected or analyzed. This is a position paper synthesizing existing literature." 57 }, 58 "environment_specified": { 59 "applies": false, 60 "answer": false, 61 "justification": "No computational experiments were conducted." 62 }, 63 "reproduction_instructions": { 64 "applies": false, 65 "answer": false, 66 "justification": "No experiments to reproduce. This is a position paper." 67 } 68 }, 69 "statistical_methodology": { 70 "confidence_intervals_or_error_bars": { 71 "applies": false, 72 "answer": false, 73 "justification": "No quantitative experiments are conducted; this is a theoretical position paper." 74 }, 75 "significance_tests": { 76 "applies": false, 77 "answer": false, 78 "justification": "No statistical comparisons are made." 79 }, 80 "effect_sizes_reported": { 81 "applies": false, 82 "answer": false, 83 "justification": "No quantitative results are reported." 84 }, 85 "sample_size_justified": { 86 "applies": false, 87 "answer": false, 88 "justification": "No samples are collected." 89 }, 90 "variance_reported": { 91 "applies": false, 92 "answer": false, 93 "justification": "No experimental runs are conducted." 94 } 95 }, 96 "evaluation_design": { 97 "baselines_included": { 98 "applies": false, 99 "answer": false, 100 "justification": "No evaluation is performed; this is a position paper proposing directions, not testing a method." 101 }, 102 "baselines_contemporary": { 103 "applies": false, 104 "answer": false, 105 "justification": "No evaluation conducted." 106 }, 107 "ablation_study": { 108 "applies": false, 109 "answer": false, 110 "justification": "No system or method to ablate." 111 }, 112 "multiple_metrics": { 113 "applies": false, 114 "answer": false, 115 "justification": "No evaluation conducted." 116 }, 117 "human_evaluation": { 118 "applies": false, 119 "answer": false, 120 "justification": "No system outputs to evaluate." 121 }, 122 "held_out_test_set": { 123 "applies": false, 124 "answer": false, 125 "justification": "No evaluation conducted." 126 }, 127 "per_category_breakdown": { 128 "applies": false, 129 "answer": false, 130 "justification": "No evaluation conducted." 131 }, 132 "failure_cases_discussed": { 133 "applies": false, 134 "answer": false, 135 "justification": "No system to produce failure cases." 136 }, 137 "negative_results_reported": { 138 "applies": false, 139 "answer": false, 140 "justification": "No experiments conducted." 141 } 142 }, 143 "claims_and_evidence": { 144 "abstract_claims_supported": { 145 "applies": true, 146 "answer": true, 147 "justification": "The abstract claims are appropriately scoped as positions rather than empirical findings: 'argues that measuring and mitigating overreliance must become central,' 'consolidate risks,' 'explore LLM characteristics,' 'propose mitigation strategies.' The paper body delivers on each of these." 148 }, 149 "causal_claims_justified": { 150 "applies": false, 151 "answer": false, 152 "justification": "The paper makes no causal claims from its own data. It discusses causal mechanisms from the literature (e.g., RLHF amplifying sycophancy) but does not claim to demonstrate them." 153 }, 154 "generalization_bounded": { 155 "applies": true, 156 "answer": true, 157 "justification": "The paper is explicit about its scope: it focuses on LLMs as 'thought partners' and acknowledges the discussion is forward-looking. Section 7 (Alternative views) directly addresses objections, including that evidence is limited." 158 }, 159 "alternative_explanations_discussed": { 160 "applies": true, 161 "answer": true, 162 "justification": "Section 7 presents and responds to three alternative views: that overreliance is not prevalent, that it will diminish as AI improves, and that it should be addressed through societal resilience rather than technical means." 163 }, 164 "proxy_outcome_distinction": { 165 "applies": false, 166 "answer": false, 167 "justification": "No measurements are taken; this is a theoretical paper." 168 } 169 }, 170 "setup_transparency": { 171 "model_versions_specified": { 172 "applies": false, 173 "answer": false, 174 "justification": "No models are used in experiments." 175 }, 176 "prompts_provided": { 177 "applies": false, 178 "answer": false, 179 "justification": "No prompting is used." 180 }, 181 "hyperparameters_reported": { 182 "applies": false, 183 "answer": false, 184 "justification": "No experiments conducted." 185 }, 186 "scaffolding_described": { 187 "applies": false, 188 "answer": false, 189 "justification": "No agentic scaffolding used." 190 }, 191 "data_preprocessing_documented": { 192 "applies": false, 193 "answer": false, 194 "justification": "No data processing conducted." 195 } 196 }, 197 "limitations_and_scope": { 198 "limitations_section_present": { 199 "applies": true, 200 "answer": false, 201 "justification": "There is no dedicated limitations section. Section 7 ('Alternative views') addresses counterarguments but does not discuss limitations of the paper's own analysis or framework." 202 }, 203 "threats_to_validity_specific": { 204 "applies": true, 205 "answer": false, 206 "justification": "No specific threats to validity of the paper's own arguments are discussed. The Alternative Views section addresses broad objections but not specific weaknesses of the paper's framing." 207 }, 208 "scope_boundaries_stated": { 209 "applies": true, 210 "answer": true, 211 "justification": "The paper states its scope clearly in Section 1: it focuses on overreliance on LLMs specifically, and its contributions are framed as consolidating risks, identifying measurement gaps, and proposing directions. The Alternative Views section also bounds the scope of the argument." 212 } 213 }, 214 "data_integrity": { 215 "raw_data_available": { 216 "applies": false, 217 "answer": false, 218 "justification": "No data collected; this is a position paper." 219 }, 220 "data_collection_described": { 221 "applies": false, 222 "answer": false, 223 "justification": "No data collection performed." 224 }, 225 "recruitment_methods_described": { 226 "applies": false, 227 "answer": false, 228 "justification": "No participants or data samples recruited." 229 }, 230 "data_pipeline_documented": { 231 "applies": false, 232 "answer": false, 233 "justification": "No data pipeline exists." 234 } 235 }, 236 "conflicts_of_interest": { 237 "funding_disclosed": { 238 "applies": true, 239 "answer": false, 240 "justification": "No funding disclosure or acknowledgments section is present in the paper." 241 }, 242 "affiliations_disclosed": { 243 "applies": true, 244 "answer": true, 245 "justification": "Author affiliations are clearly listed, including OpenAI (Lama Ahmad) and UK AI Security Institute (Andrew Strait, Merlin Stein). These affiliations are relevant given the paper discusses LLM deployment practices." 246 }, 247 "funder_independent_of_outcome": { 248 "applies": true, 249 "answer": false, 250 "justification": "No funding is disclosed, so independence cannot be assessed. One author is from OpenAI, a major LLM provider, which creates a potential conflict of interest in framing overreliance narratives." 251 }, 252 "financial_interests_declared": { 253 "applies": true, 254 "answer": false, 255 "justification": "No competing interests statement is present. Multiple authors are affiliated with organizations (OpenAI, UK AISI) that have direct stakes in LLM deployment and policy." 256 } 257 }, 258 "contamination": { 259 "training_cutoff_stated": { 260 "applies": false, 261 "answer": false, 262 "justification": "No pre-trained model is evaluated on any benchmark." 263 }, 264 "train_test_overlap_discussed": { 265 "applies": false, 266 "answer": false, 267 "justification": "No model evaluation is performed." 268 }, 269 "benchmark_contamination_addressed": { 270 "applies": false, 271 "answer": false, 272 "justification": "No benchmark evaluation is conducted." 273 } 274 }, 275 "human_studies": { 276 "pre_registered": { 277 "applies": false, 278 "answer": false, 279 "justification": "No human participants in this study." 280 }, 281 "irb_or_ethics_approval": { 282 "applies": false, 283 "answer": false, 284 "justification": "No human participants." 285 }, 286 "demographics_reported": { 287 "applies": false, 288 "answer": false, 289 "justification": "No human participants." 290 }, 291 "inclusion_exclusion_criteria": { 292 "applies": false, 293 "answer": false, 294 "justification": "No human participants." 295 }, 296 "randomization_described": { 297 "applies": false, 298 "answer": false, 299 "justification": "No human participants." 300 }, 301 "blinding_described": { 302 "applies": false, 303 "answer": false, 304 "justification": "No human participants." 305 }, 306 "attrition_reported": { 307 "applies": false, 308 "answer": false, 309 "justification": "No human participants." 310 } 311 }, 312 "cost_and_practicality": { 313 "inference_cost_reported": { 314 "applies": false, 315 "answer": false, 316 "justification": "Theoretical position paper; no computational method proposed." 317 }, 318 "compute_budget_stated": { 319 "applies": false, 320 "answer": false, 321 "justification": "No computation performed." 322 } 323 } 324 }, 325 "red_flags": [ 326 { 327 "flag": "No systematic literature review methodology", 328 "detail": "The paper synthesizes a large body of literature on overreliance but does not follow PRISMA or any structured review protocol. Paper selection appears ad-hoc, which could introduce selection bias toward supporting the authors' position." 329 }, 330 { 331 "flag": "OpenAI author affiliation not addressed as conflict", 332 "detail": "One author (Lama Ahmad) is from OpenAI. The paper argues for more investment in measuring and mitigating overreliance, which could serve commercial interests (framing overreliance as a solvable technical problem rather than a reason to restrict deployment). No conflict of interest statement is provided." 333 } 334 ], 335 "cited_papers": [ 336 { 337 "title": "To trust or to think: Cognitive forcing functions can reduce overreliance on AI in AI-assisted decision-making", 338 "authors": ["Zana Buçinca", "Maja Barbara Malaya", "Krzysztof Z Gajos"], 339 "year": 2021, 340 "relevance": "Empirical study on cognitive forcing functions to reduce overreliance in AI-assisted decision-making." 341 }, 342 { 343 "title": "\"I'm Not Sure, But...\": Examining the Impact of Large Language Models' Uncertainty Expression on User Reliance and Trust", 344 "authors": ["Sunnie S. Y. Kim", "Q Vera Liao", "Mihaela Vorvoreanu", "Stephanie Ballard", "Jennifer Wortman Vaughan"], 345 "year": 2024, 346 "relevance": "Studies how LLM uncertainty expressions affect user reliance, directly relevant to overreliance measurement." 347 }, 348 { 349 "title": "Towards understanding sycophancy in language models", 350 "authors": ["Mrinank Sharma", "Meg Tong", "Tomasz Korbak"], 351 "year": 2023, 352 "arxiv_id": "2310.13548", 353 "relevance": "Investigates sycophantic behavior in LLMs, a key driver of overreliance identified in this paper." 354 }, 355 { 356 "title": "Cybersecurity risks of AI-generated code", 357 "authors": ["Baram Nathan", "Ryan Fedasiuk"], 358 "year": 2024, 359 "relevance": "Documents security vulnerabilities in AI-generated code, relevant to overreliance risks in software engineering." 360 }, 361 { 362 "title": "Explanations can reduce overreliance on AI systems during decision-making", 363 "authors": ["Helena Vasconcelos", "Matthew Jörke", "Madeleine Grunde-McLaughlin"], 364 "year": 2023, 365 "relevance": "Empirical study on explanation-based interventions for reducing overreliance." 366 }, 367 { 368 "title": "Building machines that learn and think with people", 369 "authors": ["Katherine M Collins", "Ilia Sucholutsky", "Umang Bhatt"], 370 "year": 2024, 371 "relevance": "Foundational framework for human-AI collaboration as cognitive partnership, central framing of this paper." 372 }, 373 { 374 "title": "Generation probabilities are not enough: Uncertainty highlighting in AI code completions", 375 "authors": ["Helena Vasconcelos", "Gagan Bansal", "Adam Fourney", "Q. Vera Liao", "Jennifer Wortman Vaughan"], 376 "year": 2024, 377 "relevance": "Studies uncertainty communication in AI code completions, relevant to overreliance in programming contexts." 378 }, 379 { 380 "title": "Large language models help humans verify truthfulness–except when they are convincingly wrong", 381 "authors": ["Chenglei Si", "Navita Goyal", "Sherry Tongshuang Wu"], 382 "year": 2023, 383 "arxiv_id": "2310.12558", 384 "relevance": "Empirical evidence on LLM-assisted truth verification and overreliance on convincingly wrong outputs." 385 }, 386 { 387 "title": "Appropriate reliance on AI advice: Conceptualization and the effect of explanations", 388 "authors": ["Max Schemmer", "Niklas Kuehl", "Carina Benz"], 389 "year": 2023, 390 "relevance": "Defines appropriate reliance framework central to overreliance measurement discussion." 391 }, 392 { 393 "title": "The impact of generative AI on critical thinking: Self-reported reductions in cognitive effort and confidence effects from a survey of knowledge workers", 394 "authors": ["Hao-Ping Lee", "Advait Sarkar", "Lev Tankelevitch"], 395 "year": 2025, 396 "relevance": "Empirical survey on how generative AI reduces critical thinking in knowledge workers, directly relevant to deskilling concerns." 397 }, 398 { 399 "title": "Modulating language model experiences through frictions", 400 "authors": ["Katherine M Collins", "Valerie Chen", "Ilia Sucholutsky"], 401 "year": 2024, 402 "arxiv_id": "2407.12804", 403 "relevance": "Proposes friction-based interventions for LLM interfaces, a key mitigation strategy discussed in this paper." 404 }, 405 { 406 "title": "Relying on the unreliable: The impact of language models' reluctance to express uncertainty", 407 "authors": ["Kaitlyn Zhou", "Jena D Hwang", "Xiang Ren", "Maarten Sap"], 408 "year": 2024, 409 "arxiv_id": "2401.06730", 410 "relevance": "Studies how LLMs' failure to express uncertainty contributes to overreliance." 411 } 412 ] 413 }