ai-research-survey

scan.json (29748B)

---

 {
   "paper": {
     "title": "FacTool: Factuality Detection in Generative AI — A Tool Augmented Framework for Multi-Task and Multi-Domain Scenarios",
     "authors": [
       "I-Chun Chern",
       "Steffi Chern",
       "Shiqi Chen",
       "Weizhe Yuan",
       "Kehua Feng",
       "Chunting Zhou",
       "Junxian He",
       "Graham Neubig",
       "Pengfei Liu"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2307.13528",
     "doi": "10.48550/arXiv.2307.13528"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "FacTool, a tool-augmented factuality detection framework, outperforms LLM self-check baselines across knowledge-based QA, code generation, math problem solving, and scientific literature review. FacTool powered by GPT-4 achieves the highest performance across all scenarios, with especially large gains in scientific literature review (95.24 vs 36.71 claim-level F1 against the best self-check baseline). Self-check methods are prone to false positives and less sensitive at detecting factual errors. Among evaluated chatbots (GPT-4, ChatGPT, Claude-v1, Bard, Vicuna-13B), GPT-4 has the best factual accuracy.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'We release the code of FACTOOL associated with ChatGPT plugin interface at https://github.com/GAIR-NLP/factool.' A concrete URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "While evaluation uses standard public benchmarks (RoSE, HumanEval, GSM-Hard), the paper also constructs custom datasets — FactPrompts (50 KB-QA prompts+responses) and 100 scientific literature review prompts+responses — along with ground-truth annotations for all tasks. These custom datasets and annotations are not explicitly stated as released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or dependency specifications are mentioned in the paper. Only the models used (gpt-3.5-turbo, gpt-4) are named."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Code is released but the paper itself contains no 'Reproducing Results' section or equivalent."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4, 5, and 6 are reported as point estimates (e.g., '89.09 claim-level F1') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'FACTOOL powered by GPT-4 outperforms all other baselines across all scenarios' based solely on comparing point estimates in Tab. 5 without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance is reported with baseline context throughout (e.g., '71.79 v.s. 57.14 response-level F1 on KB-based QA', '95.24 v.s. 36.71 claim-level F1' on scientific review), allowing the reader to assess the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes range from 50 to 164 samples per task (Tab. 3) with no justification for why these sizes are adequate, no power analysis, and no acknowledgment that small sample sizes may limit conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are single-run point estimates with no standard deviation, variance, or any spread measure reported across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: Self-Check with zero-shot CoT and Self-Check with 3-shot CoT, both using ChatGPT and GPT-4 (Section 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Self-Check baselines reference Madaan et al. (2023) and Chen et al. (2023), which are contemporary works from the same year using the same models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The framework has five components (claim extraction, query generation, tool querying, evidence collection, agreement verification) but no ablation study is conducted to measure the contribution of individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported using accuracy, recall, precision, and F1-score at both claim-level and response-level (Tab. 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human annotation is used to create ground-truth labels (Section 5.3: 'the authors collectively annotate the extracted claims'), but this is dataset construction, not human evaluation of the system's outputs. The schema specifies 'manual classification of the benchmark or dataset itself does not count.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit dev/test split is described. The system's prompts were likely developed and refined using the same data that evaluation is reported on. No statement about separating data used for prompt engineering from test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (KB-QA, Code, Math, Scientific) in Tab. 5, by chatbot in Tab. 6, and per-scenario in Figures 4 and 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2.3 'Failure Analysis' provides detailed discussion of failure cases across all four tasks, including reasoning errors, conflicting evidence, limited test case variety, round-off errors, and title/author matching issues, with specific examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 'Self-check models powered by ChatGPT outperform FACTOOL powered by ChatGPT on KB-QA' (Section 6.2.1), which is a negative result for their own method. They also analyze why (reasoning errors in ChatGPT's agreement verification)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'Experiments on four different tasks show the efficacy of the proposed method,' which is supported by Tab. 5 showing superior performance across all four tasks when powered by GPT-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The core claim is that tool augmentation improves factuality detection. This is tested by comparing FacTool (with tools) against Self-Check (same LLM, without tools), which is a controlled comparison isolating the tool-use variable. The study design is adequate for this comparative claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to be a 'task and domain agnostic framework' (abstract and Section 1) but only evaluates on four specific tasks in English. The title and framing significantly overclaim relative to the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the paper does not consider whether improvements come from the specific prompting strategy rather than tool use, or whether the baselines are suboptimally configured."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines factuality for each task (Section 3.2, Tab. 2) and measures it directly against those definitions. Claims match the granularity of measurements — no proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 specifies 'ChatGPT (gpt-3.5-turbo-0301) and GPT-4 (gpt-4-0314)' with exact API model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix A (Figures 6, 7, 8) for claim extraction, query generation, and agreement verification across all four tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for the LLM API calls, despite these significantly affecting output quality."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The five-stage pipeline (claim extraction → query generation → tool querying → evidence collection → agreement verification) is described in detail in Section 4, with each component elaborated in dedicated subsections and illustrated in Figures 1-3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 documents dataset construction procedures for each task: sampling 100 summaries from RoSE, 50 responses from FactPrompts, processing HumanEval prompts per Chen et al. (2022), sampling 100 positive-target GSM-Hard prompts, and creating 100 scientific prompts via self-instruct."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2.3 'Failure Analysis' is a dedicated subsection providing substantive discussion of where FacTool fails across all four tasks, including specific failure modes with examples."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The failure analysis (Section 6.2.3) discusses system-level failure cases (reasoning errors, conflicting evidence, round-off errors) but does not discuss threats to the study's validity, such as evaluation methodology limitations, small sample sizes, or potential biases in author annotation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to be 'task and domain agnostic' without explicitly stating what it does NOT show — e.g., that it was only tested on four specific tasks, in English, using only OpenAI models."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw annotations, intermediate outputs (extracted claims, generated queries, collected evidence per example), and annotated ground truth labels are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes data collection procedures for each task including source datasets, sampling criteria (e.g., 'positive target solution value' for GSM-Hard), and response generation methods."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data comes from standard benchmarks (RoSE, HumanEval, GSM-Hard) and model-generated responses. Author annotation does not constitute human subject participation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While dataset construction is described at a high level (Section 5), there are gaps: how exactly FactPrompts were selected from Quora/TruthfulQA is unclear, the scientific prompt generation process lacks detail, and the annotation procedure ('the authors collectively annotate') provides no inter-annotator agreement or annotation protocol."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgements section thanks three people for discussions but lists no funding sources, grants, or corporate sponsors despite authors being affiliated with multiple universities and Meta AI."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Shanghai Jiao Tong University, CMU, City University of Hong Kong, NYU, Meta AI, HKUST, and Shanghai AI Laboratory."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot be determined since funding is not disclosed. One author (Chunting Zhou) is from Meta AI while the paper evaluates OpenAI products, but without funding disclosure, independence cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declarations are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for ChatGPT (gpt-3.5-turbo-0301) or GPT-4 (gpt-4-0314), which is necessary to assess whether benchmark data could be in the training set."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, GSM-Hard, RoSE, or TruthfulQA data appeared in the training data of ChatGPT or GPT-4."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), GSM8K/GSM-Hard (2021), and TruthfulQA (2022) were all published before the GPT-4 training cutoff and could be in the training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Data is from benchmarks and model-generated responses; authors serve as annotators but not as study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; IRB review is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The framework makes extensive API calls to ChatGPT and GPT-4 for claim extraction, query generation, and agreement verification, plus external tool queries (Google Search, Google Scholar, code execution), but no inference cost, latency, or token consumption is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, computational budget, or wall-clock time is reported despite the framework requiring multiple LLM calls and tool queries per example."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No analysis of sensitivity to random seeds or LLM sampling stochasticity is provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Design choices such as the number of generated search queries (2), test cases (3), and potential solutions (3) appear tuned but no search budget or selection process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of specific configurations (e.g., 2 queries per claim, 3 test cases, 3 potential solutions) is not justified. No comparison of different configurations is shown."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 4 tasks × 6 methods × 2 levels (claim/response) without any correction for multiple comparisons, though no formal statistical tests are conducted either."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed and implemented both FacTool and the self-check baselines, then compared them, without acknowledging the potential bias of evaluating their own system. Baseline implementations may be suboptimal."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "FacTool uses significantly more compute than self-check baselines (multiple LLM calls plus external tool queries vs. a single LLM call), but this compute difference is never discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its benchmarks adequately measure factuality as defined. For example, whether code execution on synthetic test cases truly captures code factuality, or whether author/title matching captures scientific citation accuracy."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "FacTool uses a complex multi-stage pipeline with external tools while self-check baselines use a simple prompting approach. The confound between the tool/scaffold and the underlying LLM reasoning is not addressed — performance gains could come from the scaffolding rather than tool use per se."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Benchmarks like HumanEval (2021) and GSM-Hard (derived from GSM8K, 2021) existed before GPT-4's training, meaning the model may have seen solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, ChatGPT generates both the responses being evaluated and serves as the claim extractor and verifier."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data. The same model that generated responses is used in FacTool's pipeline for verification."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods (canary strings, membership inference, temporal splits, decontamination) are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "FacTool powered by GPT-4 outperforms all other baselines across all four scenarios at both claim and response levels.",
       "evidence": "Tab. 5 shows FacTool+GPT-4 achieves highest F1 in all tasks: 89.09/71.79 (KB-QA), 92.11/92.11 (Code), 98.97/80.36 (Math), 95.24/94.74 (Scientific) claim/response-level F1.",
       "supported": "moderate"
     },
     {
       "claim": "FacTool significantly outperforms self-check baselines in scientific literature review, with 95.24 vs 36.71 claim-level F1.",
       "evidence": "Tab. 5 shows FacTool+GPT-4 at 95.24 vs Self-Check(3)+GPT-4 at 36.71 claim-level F1, and 94.74 vs 21.54 response-level F1 for scientific literature review.",
       "supported": "moderate"
     },
     {
       "claim": "Self-check models are prone to false positives and less sensitive in detecting factual errors compared to FacTool.",
       "evidence": "Tab. 5 shows consistently lower precision for self-check vs FacTool: 63.16 vs 87.50 (KB-QA), 76.43 vs 89.74 (Code), 58.33 vs 69.23 (Math), 12.73 vs 100.00 (Scientific) response-level precision.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 has the best factual accuracy among evaluated chatbots (GPT-4, ChatGPT, Claude-v1, Bard, Vicuna-13B).",
       "evidence": "Tab. 6 and Exp-III (Section 6.3) show GPT-4 with 75.60% weighted claim-level accuracy and 43.33% response-level accuracy, highest among all chatbots, evaluated on only 60 prompts.",
       "supported": "weak"
     },
     {
       "claim": "ChatGPT's claim extraction closely matches human-annotated atomic content units, with ROUGE-1 F1 of 0.7836.",
       "evidence": "Tab. 4 reports ChatGPT achieves 0.7836 ROUGE-1 F1, 0.6610 ROUGE-2 F1, 0.7655 ROUGE-L F1, and 0.7174 BERTScore F1 compared to golden ACUs on RoSE (Exp-I, Section 6.1).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All 'outperforms' claims are based on comparing point estimates without any significance tests. With sample sizes of 50-164, observed differences could be due to chance."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "Single-run results with no error bars, confidence intervals, or variance across runs. LLM outputs are stochastic, so results could vary substantially across runs."
     },
     {
       "flag": "Authors annotated their own benchmark",
       "detail": "Section 5.3 states 'the authors collectively annotate the extracted claims' for KB-QA and scientific literature review. No inter-annotator agreement is reported, and author bias toward their own system's outputs is not addressed."
     },
     {
       "flag": "Overclaiming generality",
       "detail": "The paper claims 'task and domain agnostic' in the abstract and title but only tests four specific tasks in English using OpenAI models. The framework requires task-specific claim definitions, query strategies, and tools for each new domain."
     },
     {
       "flag": "Unfair compute comparison",
       "detail": "FacTool uses multiple LLM calls plus external tool queries (Google Search, Google Scholar, code execution) per example, while self-check baselines use a single LLM call. The significant compute advantage is never discussed."
     },
     {
       "flag": "Small evaluation datasets",
       "detail": "Evaluation uses 50-164 samples per task (Tab. 3), with heavily imbalanced class distributions in some cases (e.g., 10:90 positive:negative in scientific literature review). No justification for these sizes."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Benchmarks like HumanEval and GSM-Hard predate GPT-4's training and could be in its training data, but this contamination risk is never discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark for code generation evaluation, used as one of the four evaluation tasks in this paper."
     },
     {
       "title": "RARR: Researching and revising what language models say, using language models",
       "authors": ["Luyu Gao", "Zhuyun Dai", "Panupong Pasupat"],
       "year": 2022,
       "relevance": "Prior work on using LLMs to verify and revise their own outputs through retrieval, directly relevant to factuality detection in LLM outputs."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "relevance": "Key work on tool use in LLMs, foundational motivation for FacTool's tool-augmented approach to factuality detection."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Self-check baselines in FacTool are based on this work's approach of using LLMs to identify and correct their own errors."
     },
     {
       "title": "Teaching large language models to self-debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli"],
       "year": 2023,
       "relevance": "Related work on LLM self-debugging for code generation, one of the baseline approaches for self-checking."
     },
     {
       "title": "Survey of hallucination in natural language generation",
       "authors": ["Ziwei Ji", "Nayeon Lee", "Rita Frieske"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLM hallucination problems that motivates factuality detection frameworks like FacTool."
     },
     {
       "title": "PAL: Program-aided language models",
       "authors": ["Luyu Gao", "Aman Madaan", "Shuyan Zhou"],
       "year": 2022,
       "arxiv_id": "2211.10435",
       "relevance": "Demonstrates using code execution to augment LLM reasoning, relevant to FacTool's tool-augmented verification approach for math problems."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Chain-of-thought prompting is used in FacTool's self-check baselines and agreement verification step."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduces GSM8K dataset from which GSM-Hard (used in FacTool evaluation) is derived."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "GPT-4 is one of the two primary LLMs used to power FacTool and is evaluated as a chatbot in the factuality assessment."
     },
     {
       "title": "CodeT: Code generation with generated tests",
       "authors": ["Bei Chen", "Fengji Zhang", "Anh Nguyen"],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "FacTool's code verification approach of generating test cases and potential solutions builds on this work's methodology."
     },
     {
       "title": "LIMA: Less is more for alignment",
       "authors": ["Chunting Zhou", "Pengfei Liu", "Puxin Xu"],
       "year": 2023,
       "relevance": "Source of KB-QA prompts used in FacTool's chatbot evaluation experiment (Exp-III)."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly usable tool for fact-checking LLM outputs across multiple domains, released as open-source code with a ChatGPT plugin interface."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expected finding that external tools help verify LLM-generated facts rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses LLM hallucination concerns but does not demonstrate novel attacks or raise new safety issues."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward framework proposal and evaluation."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Code released on GitHub with ChatGPT plugin interface, immediately tryable by developers."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Authors from CMU, NYU, and Meta AI; evaluates ChatGPT and GPT-4 which are high-profile, but paper is not from OpenAI/Anthropic/Google directly."
     }
   }
 }