scan-v5.json (26536B)
1 { 2 "scan_version": 5, 3 "paper_type": "empirical", 4 "paper": { 5 "title": "FloodBrain: Flood Disaster Reporting by Web-based Retrieval Augmented Generation with an LLM", 6 "authors": [ 7 "Grace Colverd", 8 "Paul Darm", 9 "Leonard Silverberg", 10 "Noah Kasmanoff" 11 ], 12 "year": 2023, 13 "venue": "arXiv.org / NeurIPS 2023 Workshop", 14 "arxiv_id": "2311.02597", 15 "doi": "10.48550/arXiv.2311.02597" 16 }, 17 "checklist": { 18 "claims_and_evidence": { 19 "abstract_claims_supported": { 20 "applies": true, 21 "answer": true, 22 "justification": "All major claims in abstract (LLM hallucination risks, pipeline design, GPT-4/human correlation, ablation study) are addressed in the paper's content and results.", 23 "source": "haiku" 24 }, 25 "causal_claims_justified": { 26 "applies": true, 27 "answer": true, 28 "justification": "Ablation study (Table 3) tests causal claims about pipeline components (enhanced search, source confirmation) by measuring ROUGE impact of removing each component.", 29 "source": "haiku" 30 }, 31 "generalization_bounded": { 32 "applies": true, 33 "answer": false, 34 "justification": "Paper claims to 'advance the use of LLMs for disaster impact reporting' and 'humanitarian assistance' broadly, but evaluation limited to 10-26 flood events from ReliefWeb. No discussion of generalization to other disaster types or geographic regions.", 35 "source": "haiku" 36 }, 37 "alternative_explanations_discussed": { 38 "applies": true, 39 "answer": true, 40 "justification": "Ablation results discussion offers alternative explanations: 'could be attributed to...erroneously rejected sources...or closer phrase alignment to ReliefWeb reports' (Section 3).", 41 "source": "haiku" 42 }, 43 "proxy_outcome_distinction": { 44 "applies": true, 45 "answer": true, 46 "justification": "Paper acknowledges ROUGE 'will not capture issues related to style, reports that capture more context, synonyms, or other hallucinations' and uses multiple metrics (ROUGE, G-EVAL, human judgment) to address this limitation.", 47 "source": "haiku" 48 } 49 }, 50 "limitations_and_scope": { 51 "limitations_section_present": { 52 "applies": true, 53 "answer": true, 54 "justification": "Dedicated 'Limitations and Future Work' section in conclusion discusses scope constraints, hallucination risks, and ethical concerns.", 55 "source": "haiku" 56 }, 57 "threats_to_validity_specific": { 58 "applies": true, 59 "answer": true, 60 "justification": "Specific threats identified: FloodBrain 'reports only on externally recognized disaster-classified flooding events, risking oversight in less monitored regions' and hallucination risks despite human-in-the-loop approach.", 61 "source": "haiku" 62 }, 63 "scope_boundaries_stated": { 64 "applies": true, 65 "answer": true, 66 "justification": "Paper states tool is 'designed for collaborative report writing between human and LLM' and limited to events recognized by EMSR/GDACS/ReliefWeb, but does not fully bound evaluation scope (only 10-26 reports).", 67 "source": "haiku" 68 } 69 }, 70 "conflicts_of_interest": { 71 "funding_disclosed": { 72 "applies": true, 73 "answer": true, 74 "justification": "Funding explicitly disclosed: 'Frontier Development Lab Europe...public/private partnership between European Space Agency (ESA), Trillium Technologies...supported by Google Cloud and NVIDIA Corporation.'", 75 "source": "haiku" 76 }, 77 "affiliations_disclosed": { 78 "applies": true, 79 "answer": true, 80 "justification": "All authors have institutional affiliations listed. One author (Silverberg) affiliated with Trillium Technologies, which is named as a funder/partner.", 81 "source": "haiku" 82 }, 83 "funder_independent_of_outcome": { 84 "applies": true, 85 "answer": false, 86 "justification": "Google Cloud is a named funder and the paper evaluates Google's PaLM-Text-Bison model as one of three LLM backbones, creating potential conflict of interest.", 87 "source": "haiku" 88 }, 89 "financial_interests_declared": { 90 "applies": true, 91 "answer": false, 92 "justification": "No explicit competing interests statement, patent declarations, equity stakes, or consulting relationships disclosed beyond employment/affiliation.", 93 "source": "haiku" 94 } 95 }, 96 "scope_and_framing": { 97 "key_terms_defined": { 98 "applies": true, 99 "answer": true, 100 "justification": "'Flood report' defined as 'situational report that highlights the cause, impact, context, and future work needed for recovery' (Section 1.2). LLM hallucination and RAG concepts referenced but not formally defined.", 101 "source": "haiku" 102 }, 103 "intended_contribution_clear": { 104 "applies": true, 105 "answer": true, 106 "justification": "Contribution explicitly stated: 'we introduce a sophisticated pipeline embodied in our tool FloodBrain, specialized in generating flood disaster impact reports by extracting and curating information from the web.'", 107 "source": "haiku" 108 }, 109 "engagement_with_prior_work": { 110 "applies": true, 111 "answer": true, 112 "justification": "Engages with RAG work (Lewis et al. 2020), LLM hallucination issues (Berglund et al., Kaddour et al.), and G-EVAL methodology (Liu et al. 2023), though no dedicated related work section comparing alternative disaster reporting systems.", 113 "source": "haiku" 114 } 115 } 116 }, 117 "type_checklist": { 118 "empirical": { 119 "artifacts": { 120 "code_released": { 121 "applies": true, 122 "answer": false, 123 "justification": "Web tool FloodBrain exists at floodbrain.com with UI and demo, but no source code repository or downloadable implementation provided for reproduction.", 124 "source": "haiku" 125 }, 126 "data_released": { 127 "applies": true, 128 "answer": false, 129 "justification": "No generated reports, extracted sources, search queries, or annotations released. Only uses public ReliefWeb data as reference baseline but contributes no new datasets.", 130 "source": "haiku" 131 }, 132 "environment_specified": { 133 "applies": true, 134 "answer": false, 135 "justification": "No requirements.txt, Docker file, Python version, dependency list, or environment specifications provided for reproducing the system.", 136 "source": "haiku" 137 }, 138 "reproduction_instructions": { 139 "applies": true, 140 "answer": false, 141 "justification": "No step-by-step instructions for reproducing results. Paper describes pipeline design but not how to run experiments or generate reports from raw data.", 142 "source": "haiku" 143 } 144 }, 145 "statistical_methodology": { 146 "confidence_intervals_or_error_bars": { 147 "applies": true, 148 "answer": false, 149 "justification": "Tables 1-3 report raw scores with no confidence intervals, error bars, or uncertainty estimates across model runs or samples.", 150 "source": "haiku" 151 }, 152 "significance_tests": { 153 "applies": true, 154 "answer": false, 155 "justification": "No statistical significance testing (t-tests, ANOVA, permutation tests) performed to determine if differences between GPT-4 (3.23), GPT-3.5 (2.78), and PaLM (2.76) are significant.", 156 "source": "haiku" 157 }, 158 "effect_sizes_reported": { 159 "applies": true, 160 "answer": true, 161 "justification": "Ablation study reports effect sizes as percentages: 'Removing LLM-assisted search decreases report quality across all ROUGE metrics: 6.3% for ROUGE-1, 6.2% for ROUGE-2, and 7.2% for ROUGE-L.'", 162 "source": "haiku" 163 }, 164 "sample_size_justified": { 165 "applies": true, 166 "answer": false, 167 "justification": "Human evaluation uses 10 ReliefWeb reports, ablation uses 26 reports. No justification for sample sizes or power analysis provided.", 168 "source": "haiku" 169 }, 170 "variance_reported": { 171 "applies": true, 172 "answer": false, 173 "justification": "No standard deviations, ranges, or variance metrics reported across multiple runs or sampling. Only point estimates in tables.", 174 "source": "haiku" 175 } 176 }, 177 "evaluation_design": { 178 "baselines_included": { 179 "applies": true, 180 "answer": true, 181 "justification": "Three LLM models compared (GPT-4, GPT-3.5, PaLM-Text-Bison) as baselines. No comparison to non-LLM approaches or human report generation time/effort.", 182 "source": "haiku" 183 }, 184 "baselines_contemporary": { 185 "applies": true, 186 "answer": true, 187 "justification": "All three models (GPT-4, GPT-3.5, PaLM-Text-Bison) are contemporary with 2023 publication date.", 188 "source": "haiku" 189 }, 190 "ablation_study": { 191 "applies": true, 192 "answer": true, 193 "justification": "Ablation study isolates two pipeline components: 'No enhanced search' and 'No source confirmation', testing their individual impact on ROUGE scores (Table 3).", 194 "source": "haiku" 195 }, 196 "multiple_metrics": { 197 "applies": true, 198 "answer": true, 199 "justification": "Three evaluation metrics used: ROUGE (overlap), G-EVAL (LLM-based scoring), and human judgment. Pearson correlation computed between G-EVAL and human scores.", 200 "source": "haiku" 201 }, 202 "human_evaluation": { 203 "applies": true, 204 "answer": true, 205 "justification": "Four human annotators evaluated generated vs. reference reports using the same G-EVAL framework (consistency/comprehensiveness/coherence), then Pearson correlation computed against human scores.", 206 "source": "haiku" 207 }, 208 "held_out_test_set": { 209 "applies": true, 210 "answer": false, 211 "justification": "No explicit held-out test set. ReliefWeb reports are evaluated retrospectively but no discussion of whether they overlapped with GPT-4/GPT-3.5 training data or formal train/test split.", 212 "source": "haiku" 213 }, 214 "per_category_breakdown": { 215 "applies": true, 216 "answer": false, 217 "justification": "No per-category or per-question breakdown of how well the system answers each of the 6 flood report questions (affected population, regions, causes, timeline, knock-on effects).", 218 "source": "haiku" 219 }, 220 "failure_cases_discussed": { 221 "applies": true, 222 "answer": false, 223 "justification": "No error analysis, failure cases, or examples of incorrect/hallucinated information in generated reports presented or discussed.", 224 "source": "haiku" 225 }, 226 "negative_results_reported": { 227 "applies": true, 228 "answer": true, 229 "justification": "Ablation shows mixed results: removing source confirmation actually increases ROUGE-2 and ROUGE-L (5.8% and 1% respectively), interpreted as negative signal.", 230 "source": "haiku" 231 } 232 }, 233 "setup_transparency": { 234 "model_versions_specified": { 235 "applies": true, 236 "answer": false, 237 "justification": "Models named generically as 'GPT-4', 'GPT-3.5', 'PaLM-Text-Bison' with no version numbers, snapshot dates, or API endpoint versions specified.", 238 "source": "haiku" 239 }, 240 "prompts_provided": { 241 "applies": true, 242 "answer": false, 243 "justification": "Prompts listed in Table 4 (Appendix A.2) show only 6 high-level questions but not actual system prompts, instructions given to LLMs, or full prompt templates used in pipeline.", 244 "source": "haiku" 245 }, 246 "hyperparameters_reported": { 247 "applies": true, 248 "answer": true, 249 "justification": "Temperature = 1 specified for G-EVAL evaluation. No other hyperparameters (top-p, frequency_penalty, max_tokens) reported for report generation or search expansion steps.", 250 "source": "haiku" 251 }, 252 "scaffolding_described": { 253 "applies": true, 254 "answer": true, 255 "justification": "Pipeline steps described in detail (Figure 1, Section 2.1): key phrase → web search → LLM query expansion → source relevance evaluation → Q&A extraction → final summarization. Agentic scaffolding (ReAct-based chatbot) mentioned in Section A.5.", 256 "source": "haiku" 257 }, 258 "data_preprocessing_documented": { 259 "applies": true, 260 "answer": true, 261 "justification": "Text extraction from web sources documented ('Textual data is extracted from each website'). Source relevance filtering documented. Data pipeline visualization in Figure 1. Limited detail on cleaning/normalization steps.", 262 "source": "haiku" 263 } 264 }, 265 "data_integrity": { 266 "raw_data_available": { 267 "applies": true, 268 "answer": false, 269 "justification": "No raw data (web search results, extracted sources, source snippets, annotations) made available. Only ReliefWeb reference reports are public.", 270 "source": "haiku" 271 }, 272 "data_collection_described": { 273 "applies": true, 274 "answer": true, 275 "justification": "Data collection procedure described: 'key phrase used to perform web search...textual data extracted from each website...evaluated by LLM for relevancy' (Section 2.1). Limited detail on web scraping/text extraction specifics.", 276 "source": "haiku" 277 }, 278 "recruitment_methods_described": { 279 "applies": true, 280 "answer": false, 281 "justification": "Human evaluation involves 4 annotators but no recruitment method, selection criteria, or annotator background/expertise described.", 282 "source": "haiku" 283 }, 284 "data_pipeline_documented": { 285 "applies": true, 286 "answer": true, 287 "justification": "Full pipeline from query to report described in Figure 1 and Figure 5 (including mapping and chatbot). Curation/filtering steps documented but not raw data collection/storage details.", 288 "source": "haiku" 289 } 290 }, 291 "contamination": { 292 "training_cutoff_stated": { 293 "applies": true, 294 "answer": false, 295 "justification": "No training data cutoff date stated for GPT-4 or GPT-3.5. Critical for determining if ReliefWeb reports (published pre-2023) overlapped with model training data.", 296 "source": "haiku" 297 }, 298 "train_test_overlap_discussed": { 299 "applies": true, 300 "answer": false, 301 "justification": "No discussion of potential overlap between ReliefWeb report data and GPT-4/GPT-3.5 training data. No contamination mitigation discussed.", 302 "source": "haiku" 303 }, 304 "benchmark_contamination_addressed": { 305 "applies": true, 306 "answer": false, 307 "justification": "ReliefWeb reports are used for evaluation but published before 2023. No discussion of whether these were in GPT-4 training cutoff (April 2023) or prior to model releases.", 308 "source": "haiku" 309 } 310 }, 311 "human_studies": { 312 "pre_registered": { 313 "applies": false, 314 "answer": false, 315 "justification": "Not a human participant study; evaluation uses 4 annotators as raters, not research subjects. No pre-registration needed.", 316 "source": "haiku" 317 }, 318 "irb_or_ethics_approval": { 319 "applies": false, 320 "answer": false, 321 "justification": "Not a human subjects research study. Annotators are evaluating outputs, not research participants. No IRB approval mentioned or needed.", 322 "source": "haiku" 323 }, 324 "demographics_reported": { 325 "applies": false, 326 "answer": false, 327 "justification": "Four annotators used but no demographic information provided. Not applicable as this is not human subjects research.", 328 "source": "haiku" 329 }, 330 "inclusion_exclusion_criteria": { 331 "applies": false, 332 "answer": false, 333 "justification": "Not applicable; no human participants enrolled in study.", 334 "source": "haiku" 335 }, 336 "randomization_described": { 337 "applies": false, 338 "answer": false, 339 "justification": "Not applicable; no human participant randomization.", 340 "source": "haiku" 341 }, 342 "blinding_described": { 343 "applies": false, 344 "answer": false, 345 "justification": "Not applicable; annotators knew they were evaluating LLM-generated vs. human-written reports.", 346 "source": "haiku" 347 }, 348 "attrition_reported": { 349 "applies": false, 350 "answer": false, 351 "justification": "Not applicable; no human participant attrition in annotator evaluation.", 352 "source": "haiku" 353 } 354 }, 355 "cost_and_practicality": { 356 "inference_cost_reported": { 357 "applies": true, 358 "answer": true, 359 "justification": "Source relevance filtering reduces computational cost by 59% (1,795 fewer API calls) but no absolute cost figures ($/report or total budget) provided.", 360 "source": "haiku" 361 }, 362 "compute_budget_stated": { 363 "applies": true, 364 "answer": false, 365 "justification": "No total computational budget stated. No figures on total API calls, cost per report, or infrastructure expenses for generating 10-26 evaluations.", 366 "source": "haiku" 367 } 368 } 369 } 370 }, 371 "claims": [ 372 { 373 "claim": "GPT-4 generates reports with highest overlap to human-written ReliefWeb reports compared to GPT-3.5 and PaLM-Text-Bison", 374 "evidence": "Table 1: G-EVAL scores 3.23 (GPT-4) vs 2.78 (GPT-3.5) vs 2.76 (PaLM). ROUGE Recall: 52.53 vs 51.02 vs 41.43", 375 "supported": "strong" 376 }, 377 { 378 "claim": "GPT-4 evaluation scores correlate highly with human annotator judgments", 379 "evidence": "Table 2: Pearson correlation 0.78 between G-EVAL (GPT-4) and human mean scores", 380 "supported": "strong" 381 }, 382 { 383 "claim": "LLM-assisted search expansion improves report quality", 384 "evidence": "Table 3 ablation: Removing enhanced search decreases ROUGE-1 by 6.3%, ROUGE-2 by 6.2%, ROUGE-L by 7.2%", 385 "supported": "moderate" 386 }, 387 { 388 "claim": "Source relevance filtering reduces computational cost by 59%", 389 "evidence": "Ablation section: filtering eliminates 359 failed sources, avoiding 1,795 API calls out of 2,047 total (59% reduction)", 390 "supported": "strong" 391 }, 392 { 393 "claim": "FloodBrain pipeline design with multiple components improves disaster reporting automation", 394 "evidence": "Ablation study and pipeline description (Figures 1, 5). No user study validating actual improvement for humanitarian agencies", 395 "supported": "weak" 396 }, 397 { 398 "claim": "ROUGE metric correlates well with human evaluation quality (high-level proxy claim)", 399 "evidence": "ROUGE-L shows 0.59 correlation with human scores (Table 2), lower than G-EVAL (0.78), suggesting ROUGE is imperfect proxy", 400 "supported": "moderate" 401 } 402 ], 403 "methodology_tags": [ 404 "benchmark-eval", 405 "case-study" 406 ], 407 "key_findings": "FloodBrain implements an end-to-end LLM+RAG pipeline for automated flood disaster reporting. Evaluation on 10 ReliefWeb reports shows GPT-4 achieves the highest G-EVAL scores (3.23/5.0) with strong correlation (r=0.78) to human judgments, outperforming GPT-3.5 (2.78) and PaLM (2.76). Ablation on 26 reports demonstrates LLM-assisted search expansion improves ROUGE by ~6%, while source-relevance filtering reduces API calls by 59% with mixed ROUGE effects. However, evaluation is limited to retrospective ReliefWeb data with no actual user testing or deployment metrics in real disaster scenarios.", 408 "red_flags": [ 409 { 410 "flag": "Proxy outcome problem", 411 "detail": "ROUGE (word overlap) used as primary evaluation metric for disaster reports, but ROUGE doesn't measure factual accuracy, actionability, or humanitarian utility—the paper acknowledges it 'will not capture...hallucinations'" 412 }, 413 { 414 "flag": "No statistical significance testing", 415 "detail": "Differences between LLMs (GPT-4 3.23 vs GPT-3.5 2.78) not tested for significance; unclear if variation exceeds noise" 416 }, 417 { 418 "flag": "Very small evaluation sample", 419 "detail": "Human evaluation on only 10 report pairs (4 annotators), ablation on 26 pairs—insufficient for robust conclusions about real-world performance" 420 }, 421 { 422 "flag": "Potential training data contamination", 423 "detail": "No discussion of whether ReliefWeb reports (published before 2023) overlapped with GPT-4 training cutoff; could inflate apparent performance" 424 }, 425 { 426 "flag": "No actual user testing", 427 "detail": "Claims to 'advance humanitarian assistance' but never evaluates with actual humanitarian agencies or validates time-to-report improvements" 428 }, 429 { 430 "flag": "Irreproducible", 431 "detail": "Code not released, data not released, full prompts not provided. Only web tool accessible but no source code for reproduction" 432 }, 433 { 434 "flag": "Funding conflict of interest", 435 "detail": "Google Cloud is named funder; paper evaluates Google's PaLM model alongside OpenAI models, raising potential bias" 436 }, 437 { 438 "flag": "Ablation interpretation selective", 439 "detail": "Claim source-confirmation 'not valuable' cherry-picks ROUGE-1 results while ROUGE-2/L actually degrade (5.8%, 1% respectively) without filtering" 440 } 441 ], 442 "cited_papers": [ 443 { 444 "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks", 445 "authors": "Lewis et al.", 446 "year": 2020, 447 "relevance": "Foundational RAG methodology used in FloodBrain pipeline" 448 }, 449 { 450 "title": "The Reversal Curse: LLMs trained on 'A is B' fail to learn 'B is A'", 451 "authors": "Berglund et al.", 452 "year": 2023, 453 "relevance": "Demonstrates LLM knowledge representation gaps and hallucination risks motivating pipeline design" 454 }, 455 { 456 "title": "G-EVAL: NLG Evaluation Using GPT-4 With Better Human Alignment", 457 "authors": "Liu et al.", 458 "year": 2023, 459 "relevance": "LLM-based evaluation methodology (G-EVAL) used for quality assessment instead of traditional metrics" 460 }, 461 { 462 "title": "ROUGE: A Package for Automatic Evaluation of Summaries", 463 "authors": "Lin", 464 "year": 2004, 465 "relevance": "Primary automated metric (ROUGE) used for benchmarking report quality" 466 }, 467 { 468 "title": "ReAct: Synergizing Reasoning and Acting in Language Models", 469 "authors": "Yao et al.", 470 "year": 2023, 471 "relevance": "Scaffolding approach used in FloodBrain chatbot component for interactive report refinement" 472 }, 473 { 474 "title": "Challenges and applications of large language models", 475 "authors": "Kaddour et al.", 476 "year": 2023, 477 "relevance": "Survey of LLM limitations and practical deployment challenges relevant to humanitarian context" 478 } 479 ], 480 "engagement_factors": { 481 "practical_relevance": { 482 "score": 2, 483 "justification": "Direct application to humanitarian disaster reporting with live web tool, but evaluation limited to retrospective analysis of past events with no actual deployment or user testing" 484 }, 485 "surprise_contrarian": { 486 "score": 0, 487 "justification": "Uses standard RAG + GPT-4 evaluation methodology; no novel findings or methods that challenge conventional wisdom about LLM capabilities" 488 }, 489 "fear_safety": { 490 "score": 1, 491 "justification": "Acknowledges hallucination risks and ethical concerns (environmental impact, bias) but focused on practical mitigation, not raising new safety concerns" 492 }, 493 "drama_conflict": { 494 "score": 2, 495 "justification": "Humanitarian disaster context is compelling and timely; ethical concerns about closed-source AI in aid noted, but no controversial findings or heated debates" 496 }, 497 "demo_ability": { 498 "score": 2, 499 "justification": "Live web demo at floodbrain.com and YouTube demo video available; source code not released so cannot reproduce locally" 500 }, 501 "brand_recognition": { 502 "score": 2, 503 "justification": "Frontier Development Lab and ESA partnership notable; Google Cloud/NVIDIA support provides credibility, but authors not from top-tier ML labs" 504 } 505 }, 506 "hn_data": { 507 "threads": [ 508 { 509 "hn_id": "42675299", 510 "title": "Diffusion Models Generalize via Geometry-Adaptive Harmonic Representations", 511 "points": 3, 512 "comments": 0, 513 "url": "https://news.ycombinator.com/item?id=42675299" 514 }, 515 { 516 "hn_id": "45839286", 517 "title": "Evaluating Control Protocols for Untrusted AI Agents", 518 "points": 1, 519 "comments": 1, 520 "url": "https://news.ycombinator.com/item?id=45839286" 521 }, 522 { 523 "hn_id": "40350736", 524 "title": "Generalization in diffusion models arises from geometry-adaptive harmonic reps", 525 "points": 1, 526 "comments": 0, 527 "url": "https://news.ycombinator.com/item?id=40350736" 528 }, 529 { 530 "hn_id": "40291318", 531 "title": "Generalization in Diffusion Models", 532 "points": 1, 533 "comments": 0, 534 "url": "https://news.ycombinator.com/item?id=40291318" 535 }, 536 { 537 "hn_id": "34591657", 538 "title": "Blip-2", 539 "points": 1, 540 "comments": 0, 541 "url": "https://news.ycombinator.com/item?id=34591657" 542 }, 543 { 544 "hn_id": "23842224", 545 "title": "Application of Cybernetic and Control Theory for a New Paradigm in Cybersecurity", 546 "points": 1, 547 "comments": 0, 548 "url": "https://news.ycombinator.com/item?id=23842224" 549 } 550 ], 551 "top_points": 3, 552 "total_points": 8, 553 "total_comments": 1 554 } 555 }