ai-research-survey

scan.json (27212B)

---

 {
   "paper": {
     "title": "Building Understandable Messaging for Policy and Evidence Review (BUMPER) with AI",
     "authors": [
       "Katherine A. Rosenfeld",
       "Maike Sonnewald",
       "Sonia J. Jindal",
       "Kevin A. McCarthy",
       "Joshua L. Proctor"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2407.12812",
     "doi": "10.48550/arXiv.2407.12812"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released on GitHub at https://github.com/krosenfeld/bumper_paper (Section 2, line 178). A second repository for the measles seasonality code is at https://github.com/NThakkar-IDM/seasonality (Section 4, footnote 5)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available data: the PyMC rugby tutorial dataset and publicly available measles case data. The code repository enables reproduction. The underlying measles analysis data is from published sources (Thakkar et al., 2024b)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 specifies the environment: 'The environment was maintained using pixi' with instructions 'Install pixi', 'Install dependencies: pixi install', 'Run: pixi run start'. Snakemake is used for workflow. This provides a reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 provides step-by-step instructions: 1) Install pixi, 2) Install dependencies: pixi install, 3) Run: pixi run start. Individual scripts can be run via 'pixi run python figure_3.py'. An OpenAI API key is required."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows distributions of compliance scores (Figures 4, 5) but does not report confidence intervals or error bars for any quantitative results. The distributions are shown as histograms but no summary statistics with uncertainty are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about stability across models and prompt configurations (e.g., 'adding the explanation adds stability to the assessment') but provides no statistical significance tests. Comparisons are based on visual inspection of distributions."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Claims about differences between models and configurations are qualitative (e.g., 'gpt-4o-2024-05-13 passes answers more consistently') without quantifying the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are stated (N=25 synthesized answers, N=3 compliance checks per answer, N=1100 for UMAP analysis) but no justification is given for why these particular numbers were chosen or whether they are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While distributions are shown visually in Figures 4-6, no numerical variance, standard deviation, or interquartile range statistics are reported. The reader cannot assess result stability from the information provided beyond visual inspection of histograms."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 compares BUMPER against ChatGPT4 with the tutorial as context (Figure 2). This serves as a baseline comparison showing BUMPER provides more specific answers tied to evidence."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is ChatGPT4 (GPT-4, as of 5/20/2024), which was a contemporary model at the time of writing. This is a reasonable and competitive baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper effectively ablates the compliance scoring mechanism: comparing whole-guideline vs. per-element assessment (Figure 5a), and with/without explanation prompts (Figure 5b). It also compares three different models (gpt-3.5-turbo-0125, gpt-4-0125-preview, gpt-4o-2024-05-13) in Figure 4."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation dimensions: compliance score (S = P0), pass/fail flags, Jaccard similarity between answer clusters (Figure 6), and UMAP embedding visualization. However, these are all related to the compliance checking mechanism rather than independent evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs is conducted. The paper claims BUMPER produces trustworthy and useful answers for policymakers but does not involve any policymakers or domain experts in evaluating the outputs. All evaluation is automated via compliance scores."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a framework demonstration paper with case studies, not a benchmark evaluation. There is no training/test split or held-out evaluation paradigm."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (gpt-3.5-turbo-0125, gpt-4-0125-preview, gpt-4o-2024-05-13), per query variation, per scoring method (whole vs. per-element), and per cluster in the UMAP analysis (clusters C0-C4 in Appendix A.5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 explicitly shows failure cases: Error (Antarctica query), Out-of-scope (cost comparison query), Check flag (low compliance score), and Check fail (comparative question that violates guidelines). The bi-modality in Figure 4 for gpt-4-0125-preview is discussed as a failure mode."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the compliance score distributions show 'clear bi-modality from gpt-4-0125-preview and gpt-4o-2024-05-13' revealing 'inconsistencies that undermine BUMPER's reliability.' The UMAP analysis reveals failure modes (cluster C4 with mixed pass/fail). These are honestly reported negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are largely descriptive of what the framework does and are supported by the case studies. The abstract's claim about 'a worked example in health policy' is demonstrated in Section 4. Claims about 'trustworthiness through transparency, scope-limiting, explicit-checks, and uncertainty measures' are demonstrated through the compliance scoring mechanism."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'adding the explanation adds stability to the assessment' (Section 4) based on comparing compliance score distributions. However, no controlled experiment isolates this factor — other differences between prompt configurations could contribute to the observed differences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims BUMPER 'can facilitate accessibility of and confidence in scientific evidence for policymakers' and 'increase and accelerate the impact of scientific knowledge used for policy decisions' — broad claims from two demonstration case studies (a toy rugby model and one measles example). The paper does not explicitly bound these generalizations to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its observed results. For example, the bi-modality in compliance scores could be due to prompt sensitivity, model stochasticity, or guideline ambiguity, but these alternatives are not systematically explored. The discussion section mentions challenges but not alternative explanations for the specific empirical observations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified: 'gpt-4-0125-preview' (Section 2), 'gpt-3.5-turbo-0125' and 'gpt-4o-2024-05-13' (Figure 4 caption). These are specific API snapshot identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 provides the full prompt templates used for the guidelines check, including both no-explanation and with-explanation variants for whole guideline, individual criteria, and topic evaluations. Few-shot examples are included. The fill values (guidelines G and evidence E) are also specified in Appendix A.3."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported. The paper uses the OpenAI API for multiple models but does not state what temperature or sampling settings were used, which significantly affects the compliance score distributions being analyzed."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The framework architecture is described in detail in Section 2: action identification via OpenAI's assistants API, knowledge retrieval and aggregation, evidence scoring via separate LLM instance, compliance score computation. The workflow is illustrated in Figure 1. However, the paper acknowledges the assistants API is 'a black box regarding how the matching is done.'"
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The knowledge base components are documented in Tables 2 and 3 (Appendix A.2), describing each data source, access method, purpose, and description. The rugby example uses PyMC tutorial data; the measles example uses published seasonality/susceptibility code."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section (Section 5) discusses several limitations: the potential loss of human-to-human interaction, the additional work burden on scientists, validation challenges, and the limitation to proprietary models due to computational constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discussed are largely generic and forward-looking rather than specific threats to the validity of the results presented. For example, 'the task we approach... is not well-suited for assessment against established bench-marking datasets' is stated but not analyzed as a specific threat. No specific threats regarding the compliance score experiments (e.g., sensitivity to prompt wording, limited query variety, single-domain testing) are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. It does not bound its claims to the two specific case studies tested or acknowledge that the compliance score behavior observed may not generalize to other domains, models, or knowledge base configurations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the code is released, the raw experimental data (the 1100 synthesized answers for the UMAP analysis, the compliance score distributions, the individual LLM responses) does not appear to be released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: N=25 synthesized answers generated per query, N=3 compliance checks per answer, N=2 fixed queries (Figures 4-5), and N=1100 gpt-4-0125-preview answers for UMAP analysis (Figure 6). The process of generating answers via the BUMPER framework is described in Section 2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study. The data consists of LLM-generated outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from user query to compliance score is documented in Section 2 (steps 1-5) and illustrated in Figure 1. The compliance score computation is formalized in Equation 1. The experimental setup for generating distributions is described in the figure captions."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states 'KR, SJ, KM, and JP are employees of the Bill and Melinda Gates Foundation.' The affiliation footnotes also indicate the Institute for Disease Modeling, Bill & Melinda Gates Foundation."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Institute for Disease Modeling, Bill & Melinda Gates Foundation (KR, SJ, KM, JP) and Department of Computer Science, University of California, Davis (MS). The Gates Foundation affiliation is prominent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'this study does not necessarily represent the views of the Bill and Melinda Gates Foundation.' The Gates Foundation funds disease modeling and has an interest in measles control, but the BUMPER framework itself is a research tool, not a product. The funder's interest is in improving health outcomes, not in a specific outcome of the framework evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The paper does not include a formal conflicts-of-interest declaration beyond the affiliation disclosure."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It uses LLMs as components of a framework for evidence synthesis and compliance checking. The evaluation is about the framework's behavior, not the model's knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. The paper evaluates a framework's compliance scoring mechanism, not a model's knowledge of test data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No standard benchmarks are used. The evaluation consists of case studies with domain-specific queries, not benchmark tasks that could have appeared in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported. The paper runs substantial LLM experiments (1100 generations for UMAP, 25x3x2 for distribution analysis, across three models) but does not report API costs, token counts, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper mentions 'due to computational limitations, we have been limited to a proprietary model and associated API' (Section 5) but does not quantify the actual compute used."
       }
     }
   },
   "claims": [
     {
       "claim": "BUMPER provides more specific and evidence-grounded answers compared to ChatGPT4 with context.",
       "evidence": "Section 3, Figure 2: Comparison on 'Which team has the second worst attack?' query shows BUMPER provides a specific answer using model parameters while 'ChatGPT4 incorrectly hedges its answer.'",
       "supported": "weak"
     },
     {
       "claim": "Adding explanation prompts increases the stability of the compliance score assessment.",
       "evidence": "Section 4, Figure 5b: Comparison of compliance score distributions with and without explanation prompts shows 'adding the explanation adds stability to the assessment.'",
       "supported": "moderate"
     },
     {
       "claim": "Checking guidelines by individual element (rather than the whole set) reduces compliance scores and increases spread.",
       "evidence": "Section 4, Figure 5a: Per-element assessment 'reduces the compliance score and increases the spread of S.'",
       "supported": "moderate"
     },
     {
       "claim": "Token-level probabilities (P0) from the compliance check reveal inconsistencies across models that undermine reliability.",
       "evidence": "Section 4, Figure 4: Shows 'clear bi-modality from gpt-4-0125-preview and gpt-4o-2024-05-13' while gpt-3.5-turbo-0125 shows consistent results. The paper states 'These distributions reveal inconsistencies that undermine BUMPER's reliability.'",
       "supported": "moderate"
     },
     {
       "claim": "UMAP embedding and clustering of synthesized answers reveals distinct answer clusters with generally consistent compliance scores.",
       "evidence": "Section 4, Figure 6: K-means clustering of UMAP embeddings for N=1100 answers shows distinct clusters. Jaccard similarity scores within clusters are reported. Cluster C4 shows mixed pass/fail behavior.",
       "supported": "moderate"
     },
     {
       "claim": "The BUMPER framework can facilitate accessibility of and confidence in scientific evidence for policymakers.",
       "evidence": "Abstract and Discussion (Section 5). This is argued based on the framework design and two case studies, but no policymaker evaluation or user study is conducted.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "case-study"
   ],
   "key_findings": "The paper introduces BUMPER, a framework for using LLMs to translate scientific evidence for policymakers, featuring a novel compliance score based on token probabilities. Through case studies on rugby prediction and measles health policy, the authors demonstrate that compliance score distributions vary significantly across models (gpt-3.5-turbo, gpt-4, gpt-4o), with bi-modality revealing reliability concerns. Adding explanation prompts and per-element guideline assessment improves stability but reduces compliance scores. UMAP clustering of 1100 synthesized answers reveals distinct response clusters with varying compliance behaviors, including failure modes where identical answers receive contradictory pass/fail verdicts.",
   "red_flags": [
     {
       "flag": "No human evaluation",
       "detail": "The paper claims BUMPER is useful for policymakers to access scientific evidence, but no policymakers, domain experts, or users were involved in evaluating the system. All evaluation is automated via the compliance score mechanism, which is itself part of the system being evaluated."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature and sampling parameters are not reported for any of the three models used, despite the paper analyzing token probability distributions that are directly affected by these settings. This is a critical omission when the main evaluation metric (compliance score) is derived from token probabilities."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The paper argues BUMPER 'can facilitate accessibility of and confidence in scientific evidence for policymakers' and 'increase and accelerate the impact of scientific knowledge used for policy decisions' based on two case studies (one toy, one domain-specific) with no user evaluation. The reported results actually reveal significant reliability problems (bi-modal scores, inconsistent pass/fail)."
     },
     {
       "flag": "Circular evaluation",
       "detail": "The compliance score mechanism is both a component of BUMPER and the primary evaluation metric. The paper evaluates BUMPER largely by examining how its own compliance scoring behaves, rather than independently assessing whether the framework produces correct or useful outputs."
     },
     {
       "flag": "Very limited query diversity",
       "detail": "The measles case study results are based on essentially two queries ('When should the next SIA in Cameroon be planned?' and 'When should Cameroon run SIAs?'). The rugby example uses a single query. The generalizability of compliance score behavior to diverse queries is untested."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection",
       "authors": ["A. Asai", "Z. Wu", "Y. Wang", "A. Sil", "H. Hajishirzi"],
       "year": 2023,
       "arxiv_id": "2310.11511",
       "relevance": "Self-reflective RAG approach relevant to survey scope on LLM architectures for reliable evidence synthesis."
     },
     {
       "title": "On the Opportunities and Risks of Foundation Models",
       "authors": ["R. Bommasani", "D. A. Hudson"],
       "year": 2022,
       "arxiv_id": "2108.07258",
       "relevance": "Major survey on foundation model risks and opportunities, foundational reference for AI safety and reliability research."
     },
     {
       "title": "Building Guardrails for Large Language Models",
       "authors": ["Y. Dong", "R. Mu", "G. Jin"],
       "year": 2024,
       "arxiv_id": "2402.01822",
       "relevance": "Survey on guardrail architectures for LLMs, directly related to safety and alignment in LLM-based systems."
     },
     {
       "title": "Llama Guard: LLM-based Input-Output Safeguard for Human-AI Conversations",
       "authors": ["H. Inan", "K. Upasani"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "LLM safety guardrail system relevant to the survey's scope on AI safety and responsible deployment."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agentic LLM framework combining reasoning and action, directly relevant to agentic AI survey scope."
     },
     {
       "title": "PAL: Program-aided Language Models",
       "authors": ["L. Gao", "A. Madaan", "S. Zhou"],
       "year": 2023,
       "arxiv_id": "2211.10435",
       "relevance": "Tool-augmented LLM approach using code generation, relevant to agentic AI and LLM programming research."
     },
     {
       "title": "TrustLLM: Trustworthiness in Large Language Models",
       "authors": ["L. Sun", "Y. Huang", "H. Wang"],
       "year": 2024,
       "arxiv_id": "2401.05561",
       "relevance": "Benchmark for LLM trustworthiness evaluation, directly relevant to survey scope on LLM safety and reliability."
     },
     {
       "title": "The Rise and Potential of Large Language Model Based Agents: A Survey",
       "authors": ["Z. Xi", "W. Chen", "X. Guo"],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Comprehensive survey on LLM-based agents, core reference for agentic AI research."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique relevant to LLM capability and programming survey scope."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2021,
       "arxiv_id": "2005.11401",
       "relevance": "Foundational RAG paper relevant to LLM architectures for evidence synthesis and knowledge-intensive applications."
     },
     {
       "title": "Can LLMs Express Their Uncertainty? An Empirical Evaluation of Confidence Elicitation in LLMs",
       "authors": ["M. Xiong", "Z. Hu", "X. Lu"],
       "year": 2024,
       "arxiv_id": "2306.13063",
       "relevance": "Directly relevant to BUMPER's use of token probabilities as confidence measures in LLM outputs."
     },
     {
       "title": "NeMo Guardrails: A Toolkit for Controllable and Safe LLM Applications with Programmable Rails",
       "authors": ["T. Rebedea", "R. Dinu", "M. Sreedhar"],
       "year": 2023,
       "arxiv_id": "2310.10501",
       "relevance": "LLM guardrail framework relevant to safety and controlled deployment of LLM-based systems."
     }
   ]
 }