ai-research-survey

scan.json (28718B)

---

 {
   "paper": {
     "title": "FloodBrain: Flood Disaster Reporting by Web-based Retrieval Augmented Generation with an LLM",
     "authors": [
       "Grace Colverd",
       "Paul Darm",
       "Leonard Silverberg",
       "Noah Kasmanoff"
     ],
     "year": 2023,
     "venue": "6th Workshop on Artificial Intelligence for Humanitarian Assistance and Disaster Response (NeurIPS 2023)",
     "arxiv_id": "2311.02597",
     "doi": "10.48550/arXiv.2311.02597"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "FloodBrain is a RAG-based pipeline for generating flood disaster impact reports by extracting and curating web information. GPT-4 as backbone produced reports with highest overlap with human-written ReliefWeb reports across G-EVAL and ROUGE metrics. G-EVAL showed a 0.78 Pearson correlation with human annotators, higher than ROUGE metrics. LLM-assisted search query expansion improved ROUGE scores by 6-7%, while source relevancy filtering reduced computational cost by 59% with mixed impact on report quality.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a web UI at floodbrain.com but does not release any source code. No GitHub repository, Zenodo archive, or code link is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The dataset of 10 (and expanded 26) FloodBrain/ReliefWeb report pairs used for evaluation is not released. ReliefWeb reports are publicly available in general, but the specific curated evaluation dataset is not provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or software versions are provided anywhere in the paper or appendix."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The pipeline is described at a high level but there are no step-by-step instructions or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., G-EVAL scores, ROUGE scores, Pearson correlations) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 'exhibited superior performance' compared to GPT-3.5 and PaLM-Text-Bison based solely on comparing point estimates (3.27 vs 2.96 vs 2.34) without any significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports percentage changes with context: 'Removing LLM-assisted search decreases report quality across all ROUGE metrics: 6.3% for ROUGE-1, 6.2% for ROUGE-2, and 7.2% for ROUGE-L.' Baseline scores are also provided in Table 3."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The main evaluation uses only 10 report pairs and 4 human annotators, and the ablation uses 26 pairs. No justification is given for these small sample sizes, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "G-EVAL is described as prompting GPT-4 ten times with temperature 1, but no variance or standard deviation across these runs is reported. Report generation results also have no variance measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three LLMs are compared: GPT-4, GPT-3.5, and PaLM-Text-Bison (Table 1). The ablation study also compares the full pipeline against ablated versions (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4, GPT-3.5, and PaLM-Text-Bison were all current state-of-the-art models at the time of publication (2023)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents an ablation study removing pipeline components: LLM-assisted search expansion and source relevancy confirmation, individually and together."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ROUGE-1, ROUGE-2, ROUGE-L, G-EVAL scores, and human evaluation scores — five distinct evaluation approaches."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3: 'a manual annotation round was conducted, involving four annotators assigned to perform an equivalent evaluation to G-EVAL checking for consistency between the reference and generated report.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The prompts were 'developed with feedback from domain experts' but there is no discussion of whether any of the 10 evaluation events were used during prompt development. No explicit dev/test separation is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as averages across all report pairs. No per-event, per-region, or per-disaster-type breakdowns are provided to show whether performance varies across different flood events."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis is presented. The paper does not show examples of generated reports that were particularly poor or discuss specific failure modes of the pipeline."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports mixed results for source relevancy removal: 'ROUGE-1 declines by 4.2%, while ROUGE-2 and ROUGE-L increase by 5.8% and 1% respectively.' The paper openly discusses this counter-intuitive finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'notable correlation' between GPT-4 and human evaluator scores, supported by Table 2 (Pearson r=0.78). The ablation study claim is supported by Table 3. The comparison of LLMs is supported by Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims 'augmenting our web search with LLM-generated additional queries further improves the agreement.' The ablation study uses controlled single-variable manipulation (removing one component at a time), which is adequate for this causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title and scope are specific to 'Flood Disaster Reporting.' The paper does not overclaim to all disasters or all NLP tasks. The conclusion appropriately scopes to 'flood disaster reporting' and 'the field of humanitarian assistance.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for why GPT-4 outperforms the other models, or whether the evaluation metrics (ROUGE, G-EVAL) could be confounded. For instance, GPT-4 is used both as the G-EVAL evaluator and as one of the systems being evaluated."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures ROUGE overlap and G-EVAL consistency with reference reports but frames this as 'report quality.' Similarity to a ReliefWeb reference report is not the same as actual utility for humanitarian response — a generated report could be useful even if dissimilar to the reference, or high-ROUGE but practically useless. This gap is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4', 'GPT-3.5', and 'PaLM-Text-Bison' without specifying API versions, snapshot dates, or model version identifiers (e.g., 'gpt-4-0613')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 provides the five questions used for report extraction, and the paper references 'Appendix A.2' for prompt information, but the actual system prompts for source relevancy evaluation, query expansion, and final report summarization are described in natural language rather than provided as full prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "For G-EVAL, temperature=1 is stated. However, no temperature, top-p, max tokens, or other sampling parameters are reported for the actual report generation pipeline across any of the three LLMs."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 and Figures 1 and 5 describe the pipeline: key phrase → web search → source extraction → LLM relevancy evaluation → question/answer extraction per source → final report summarization. The ReAct-based chatbot component is also described (Appendix A.5)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the high-level pipeline (web search → text extraction → relevancy filtering) but does not document how text was extracted from websites, how extraction failures were handled, or what cleaning/preprocessing was applied to source text before passing to the LLM."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations and Future Work' subsection and an 'Ethical and Practical Considerations' subsection are included at the end of Section 4, discussing scope of events, environmental costs, and hallucination risks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies a specific threat: 'The FloodBrain UI currently reports only on externally recognized disaster-classified flooding events, risking oversight in less monitored regions due to unclear classification criteria of external agencies.' Also notes that skipping human verification 'may lead to incorrect information in FloodBrain reports.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states explicit boundaries: limited to 'externally recognized disaster-classified flooding events,' the tool is 'designed for collaborative report writing between human and LLM, to be used cautiously,' and 'Verification before official sharing is advised.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 10 and 26 report pairs, the individual source documents, the LLM responses, and the human annotator ratings are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states evaluation uses 'a dataset comprising 10 ReliefWeb reports' but does not specify which 10 events were selected, what selection criteria were used, or what time period they cover."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Four human annotators conducted the evaluation but no information is provided about who they were, their expertise level, how they were recruited, or whether they had prior experience with disaster reporting."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The report generation pipeline is described at a high level (Section 2.1, Figure 1), but the evaluation data pipeline — how 10 events were selected, how ReliefWeb reports were matched to generated reports, what quality control was applied to annotations — is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding: 'enabled by Frontier Development Lab Europe a public/private partnership between the European Space Agency (ESA), Trillium Technologies, the University of Oxford' with support from 'Google Cloud and NVIDIA Corporation.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Cambridge, University of Strathclyde, Trillium Technologies, and New York University. One author (Silverberg) works at Trillium Technologies, which manages FDL Europe."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google Cloud provided computational resources and Google's PaLM-Text-Bison is one of the three models evaluated. While GPT-4 ultimately performed best, the funder has a stake in their model's evaluation. Trillium Technologies manages FDL Europe and employs one author."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. One author works at Trillium Technologies (the FDL Europe organizer) and this potential conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-4, GPT-3.5, or PaLM-Text-Bison. This matters because ReliefWeb reports used as references may be in the models' training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the ReliefWeb reference reports (used as ground truth) appeared in the training data of GPT-4, GPT-3.5, or PaLM-Text-Bison. ReliefWeb is a major public humanitarian data source likely included in web-scraped training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ReliefWeb reports are publicly available and were published before the models' training cutoffs. If a model memorized these reports, ROUGE overlap scores would be inflated. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human subjects research. The 4 human annotators are evaluators of system output, not research participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects research is conducted. Annotators evaluated system outputs; they were not studied as participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The 4 annotators are part of the evaluation methodology, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotators were recruited for evaluation, not as research participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experimental study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experimental study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The 4 annotators are evaluators, not study participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports relative cost savings (59% reduction in API calls from source filtering, and '1,795 LLM API calls' avoided) but no actual monetary costs, latency figures, or per-report generation time are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgments thank Google Cloud for 'extensive computational resources' but no specific compute budget (GPU hours, total API spend, hardware specifications) is quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "G-EVAL uses temperature 1 with 10 prompts per evaluation, but no seed sensitivity analysis is reported for the report generation pipeline itself. No variance across runs is shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "For G-EVAL, 10 prompts per evaluation are stated. However, it is not stated whether reports were generated once or multiple times per event. The main pipeline results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The pipeline design appears fixed with no discussion of alternative configurations tested during development."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline configuration (number of search queries, number of questions, etc.) is presented without justification for why these specific choices were made or what alternatives were considered."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors built FloodBrain and evaluate it. No discussion of the bias inherent in evaluating one's own system, and no independent evaluation is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The three models likely have very different compute costs (GPT-4 >> GPT-3.5 >> PaLM-Text-Bison), but no performance-per-dollar or performance-per-token analysis is provided. The ablation mentions cost reduction but not the cost-quality tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "ROUGE measures n-gram overlap with reference reports and G-EVAL measures LLM-judged consistency, but neither is validated as a measure of actual report utility for humanitarian responders. The paper does not discuss whether these metrics capture what matters for disaster response."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All three LLMs are compared within the same FloodBrain pipeline, so the scaffolding is held constant across model comparisons. The ablation also modifies pipeline components while holding the model constant (Bison)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' training data includes ReliefWeb reports from the evaluated flood events. If GPT-4 was trained on data including these reports, its generated reports would naturally overlap more with the references."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline performs web searches which could retrieve the actual ReliefWeb reference reports used as ground truth. If the system retrieves and summarizes the same reference report it's being compared against, ROUGE scores would be artificially inflated. This critical potential confound is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the web sources retrieved by FloodBrain overlap with or are derived from the ReliefWeb reports used as ground truth references."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. The paper does not check whether retrieved web sources include the reference reports or content derived from them."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 as pipeline backbone produces flood reports with highest overlap with human-written ReliefWeb reports",
       "evidence": "Table 1: GPT-4 achieves G-EVAL 3.27 (human: 3.23), ROUGE-1 52.53, ROUGE-2 15.76, ROUGE-L 41.83, compared to GPT-3.5 (2.96, 51.02, 13.62, 40.10) and PaLM-Text-Bison (2.34, 41.43, 10.20, 32.08).",
       "supported": "moderate"
     },
     {
       "claim": "G-EVAL scores show high correlation with human annotator scores (Pearson r=0.78)",
       "evidence": "Table 2: Pearson correlation between G-EVAL mean and human mean is 0.78, compared to ROUGE-1 (0.54), ROUGE-2 (0.62), ROUGE-L (0.59). Section 3.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-assisted search query expansion improves report quality by 6-7% on ROUGE metrics",
       "evidence": "Table 3 and Section 3: 'Removing LLM-assisted search decreases report quality across all ROUGE metrics: 6.3% for ROUGE-1, 6.2% for ROUGE-2, and 7.2% for ROUGE-L.' Based on 26 report pairs with Bison backbone.",
       "supported": "moderate"
     },
     {
       "claim": "Source relevancy filtering reduces computational cost by 59%",
       "evidence": "Section 3: '41% of sources pass the relevancy check (252 out of 611 for 26 reports), averting an additional 1,795 LLM API calls.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny sample sizes",
       "detail": "The main evaluation uses only 10 report pairs with 4 human annotators. The ablation study expands to 26 pairs. These are very small samples for the comparative claims being made, with no error bars, significance tests, or power analysis."
     },
     {
       "flag": "Potential feature leakage via web search",
       "detail": "The pipeline performs live web searches that could retrieve the same ReliefWeb reports used as ground truth references. If FloodBrain retrieves and summarizes content from the reference report, ROUGE scores are artificially inflated. This confound is never discussed."
     },
     {
       "flag": "GPT-4 used as both system and evaluator",
       "detail": "GPT-4 is one of the three LLM backbones being evaluated, and GPT-4 is also the evaluator in the G-EVAL methodology. This creates a potential self-preference bias where GPT-4 may rate GPT-4-generated text more favorably."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are point estimates without confidence intervals, error bars, or standard deviations. G-EVAL is run 10 times with temperature 1 but variance across these runs is not reported."
     },
     {
       "flag": "Missing model versions",
       "detail": "Models are referenced only as 'GPT-4', 'GPT-3.5', and 'PaLM-Text-Bison' without API versions or snapshot dates, making exact reproduction impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "GPTs are GPTs: An Early Look at the Labor Market Impact Potential of Large Language Models",
       "authors": ["Tyna Eloundou", "Sam Manning", "Pamela Mishkin", "Daniel Rock"],
       "year": 2023,
       "arxiv_id": "2303.10130",
       "relevance": "Early assessment of LLM impact on labor markets, relevant to understanding AI capability claims and workforce automation."
     },
     {
       "title": "The Reversal Curse: LLMs Trained on 'A is B' Fail to Learn 'B is A'",
       "authors": ["Lukas Berglund", "Meg Tong", "Max Kaufmann"],
       "year": 2023,
       "relevance": "Identifies gaps in LLM knowledge representation, relevant to understanding LLM limitations and hallucination."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper used as the core methodology for FloodBrain's approach to grounding LLM outputs in retrieved information."
     },
     {
       "title": "Challenges and Applications of Large Language Models",
       "authors": ["Jean Kaddour", "Joshua Harris", "Maximilian Mozes"],
       "year": 2023,
       "arxiv_id": "2307.10169",
       "relevance": "Survey of LLM challenges including hallucination, directly relevant to understanding limitations of LLM-based systems."
     },
     {
       "title": "G-EVAL: NLG Evaluation Using GPT-4 with Better Human Alignment",
       "authors": ["Yang Liu", "Dan Iter", "Yichong Xu"],
       "year": 2023,
       "relevance": "Introduces the G-EVAL methodology for LLM-as-evaluator, directly used in this paper's evaluation framework."
     },
     {
       "title": "PaLM 2 Technical Report",
       "authors": ["Rohan Anil", "Andrew M. Dai", "Orhan Firat"],
       "year": 2023,
       "relevance": "Technical report for PaLM-Text-Bison, one of the three LLMs evaluated in the FloodBrain pipeline."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "ReAct framework used as the reasoning system for FloodBrain's chatbot component, relevant to agentic LLM workflows."
     },
     {
       "title": "Estimating the Carbon Footprint of BLOOM, a 176B Parameter Language Model",
       "authors": ["Alexandra Sasha Luccioni", "Sylvain Viguier", "Anne-Laure Ligozat"],
       "year": 2022,
       "relevance": "Referenced for environmental cost of LLMs, relevant to understanding practical implications of LLM deployment."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "FloodBrain is a deployed tool (floodbrain.com) for a real humanitarian use case, though it targets a niche domain of flood disaster reporting."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm expected patterns: GPT-4 outperforms smaller models, RAG improves factual grounding, and LLM evaluators correlate with humans."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper mentions hallucination risk in humanitarian reporting but does not raise novel AI safety or security concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to existing work, and no provocative claims."
     },
     "demo_ability": {
       "score": 2,
       "justification": "A live demo at floodbrain.com and a YouTube demo video are provided, though source code is not available for local use."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-4 and PaLM but comes from academic/nonprofit collaborations (Cambridge, NYU, Trillium, ESA), not from a major AI lab."
     }
   }
 }