ai-research-survey

scan.json (26516B)

---

 {
   "paper": {
     "title": "Attacks by Content: Automated Fact-checking is an AI Security Issue",
     "authors": ["Michael Schlichtkrull"],
     "year": 2025,
     "venue": "Conference on Empirical Methods in Natural Language Processing (EMNLP 2025)",
     "arxiv_id": "2510.11238",
     "doi": "10.18653/v1/2025.emnlp-main.431"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/MichSchli/AgentCogSec (footnote 1, Section 1). The paper also states the attack dataset is available at that repository (Appendix B)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset of 60 attack scenarios is released via the GitHub repository at https://github.com/MichSchli/AgentCogSec/attack_dataset.json (Appendix B)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned in the paper. The paper does not describe how to set up the experimental environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the prompts are provided in Appendix A and the data is released, there are no step-by-step reproduction instructions, no README with commands to run, and no scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., '78.3%', '90.0%') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'the clear stand-out is Llama 3.1 8b', 'vulnerability rates are significantly different across the three conditions' in Appendix D) but no statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Vulnerability rates are reported as percentages with baseline context across conditions (baseline, fact-check, source warning, both), allowing readers to compute the magnitude of defense effectiveness. E.g., Claude Sonnet 4 goes from 61.7% baseline to 15.0% with fact-check to 0.0% with both defenses (Table 1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size is 60 scenarios (10 per area of concern), but no justification for why 60 scenarios or 10 per category is sufficient is provided. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Each scenario appears to be run once per model. No repeated runs are mentioned."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes a 'Baseline' column (no protection) compared against three defense conditions (fact-check, source warning, both). Multiple models serve as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models tested are highly contemporary: GPT-4.1, o4-mini-high, Claude Sonnet 4, Claude Opus 4, Gemini 2.5 Pro/Flash, Llama 3.3, Qwen 3, DeepSeek R1, Grok 3. These represent state-of-the-art models as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The experimental design includes ablation-like comparisons: baseline (no defense), fact-check only, source warning only, and both combined. Appendix D further ablates the Grok system prompt with and without epistemological warning lines."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only one metric is used: vulnerability rate (percentage of attacks the model passes on to users). No other metrics such as false positive rate (rejecting legitimate information) are measured."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates whether LLMs are vulnerable to content attacks, which involves subjective quality judgments about agent behavior. No human evaluation of whether the model's decisions were correct or reasonable is included. The scenarios were manually created/edited but there is no human evaluation of model outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning training/testing study. The 60 scenarios are evaluated directly with no tuning/selection process requiring train/test separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-category breakdowns across all 6 areas of concern (Charity, Finance, Healthcare, Law, Politics, Useless products) for each model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure patterns: charity fraud is identified as the most problematic category where almost all attacks succeeded against all models (Section 5, Table 2). The finding that larger models are not necessarily less vulnerable is also a failure-case discussion."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: defenses that reduce vulnerability significantly still leave many models vulnerable; fact-checking is not sufficient for all cases (Limitations section notes 60-65% accuracy of current systems); charity fraud remains extremely difficult to defend against."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are: (1) injection of instructions is not necessary to manipulate agents (supported by Table 1 baseline column), (2) existing defenses are ineffective against attacks by content (supported by the conceptual argument in Section 2), (3) fact-checking is proposed as a defense (supported by Table 1 showing reduced vulnerability with fact-checks). Claims are appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about defenses reducing vulnerability (e.g., 'Fact-checking and source warnings were both highly effective defense strategies'). These are supported by controlled experimental manipulation — identical scenarios tested with and without each defense, holding all else constant. This is adequate for causal inference about the defense's effect."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper generalizes broadly from a narrow experimental setup. The title claims 'Automated Fact-checking is an AI Security Issue' broadly, but the experiment tests only a simulated research agent on 60 hand-crafted scenarios using a simple yes/no prompt. The paper does not adequately bound generalizations to this specific setting — it uses language like 'agents must critically evaluate retrieved information' as a general claim based on this limited evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed vulnerability rates. For instance, the vulnerability might be an artifact of the specific prompt framing (asking 'should you pass this on' may bias toward YES), the synthetic nature of the scenarios, or the simplicity of the agent setup. The Limitations section acknowledges fact-checking limitations but does not discuss alternative explanations for the experimental results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists specific model identifiers: meta-llama/llama-3.1-8b-instruct, meta-llama/llama-3.1-70b-instruct, openai/gpt-4.1, openai/o4-mini-high, anthropic/claude-sonnet-4, anthropic/claude-opus-4, google/gemini-2.5-flash, google/gemini-2.5-pro, deepseek/deepseek-r1-0528, etc. These are API-level model identifiers sufficient for reproduction."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt templates are provided in Appendix A (Figures 5 and 6), including the exact text with placeholders for task, attack_text, fact_check, and source_warning. An example data point filling these placeholders is provided in Appendix B (Figure 7). The full dataset is also released on GitHub."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — temperature, top-p, max tokens, and other sampling settings are not mentioned despite using multiple LLM APIs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The experimental setup uses a simple single-prompt, single-response design (Figures 5 and 6). There is no agentic scaffolding — the LLM is given a prompt and asked for a yes/no response."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data creation process: 'Each scenario was created by initially generating a fictional scenario using Claude 4 Opus, which we then manually edited for plausibility.' The 60 scenarios are distributed as 10 per area of concern (Section 5). The full dataset is released."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is a dedicated 'Limitations' section discussing that automated fact-checking does not guarantee protection, current systems correctly verify only 60-65% of claims, and attackers might circumvent protections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section raises specific threats: (1) current fact-checking systems only verify 60-65% of real-world claims, (2) attackers might inject adversarial data into evidence sources, (3) claims might be crafted to be adversarial against fact-checking systems. The Ethics section (Section 9) also raises specific concerns about bias in fact-checking tools."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what results do NOT show. It does not bound the findings to the specific experimental setup (60 synthetic scenarios, simple yes/no agent, specific prompt design). The Limitations section discusses fact-checking limitations but not the scope boundaries of the empirical findings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The attack dataset is released on GitHub (https://github.com/MichSchli/AgentCogSec/attack_dataset.json, Appendix B). However, the raw model outputs (YES/NO responses and reasoning) are not mentioned as being available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data collection: 60 scenarios across 6 areas of concern, 10 per area, initially generated by Claude 4 Opus then manually edited for plausibility."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study. The study evaluates LLM responses to synthetic scenarios."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes scenario creation but does not document the full data pipeline: how model outputs were collected, how vulnerability rates were computed from raw outputs, whether any responses were ambiguous or excluded. The step from model response to vulnerability rate percentage is not fully documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work was supported by the Engineering and Physical Sciences Research Council [grant number EP/Y009800/1], through funding from Responsible AI UK (KP0016).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation is clearly stated: School of Electronic Engineering and Computer Science, Queen Mary University of London. The author is not affiliated with any of the model providers being tested."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (EPSRC / Responsible AI UK) is a UK government research council with no financial stake in the vulnerability of any specific model or the effectiveness of fact-checking as a defense."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The experiment does not evaluate pre-trained models on a benchmark testing their knowledge or capabilities in a way that could be contaminated by training data. The scenarios are synthetic and test agent behavior (whether to pass information to users), not model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The experimental setup tests agent decision-making on novel synthetic scenarios, not model knowledge on existing benchmarks. Contamination is structurally irrelevant."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The 60 scenarios were newly created for this paper using Claude 4 Opus and then manually edited. There is no pre-existing benchmark that could be contaminated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests 20 models across 60 scenarios with 4 conditions each (4,800 API calls minimum) but reports no inference cost, token consumption, or API spending."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API costs, or hardware used are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-based agents are highly vulnerable to attacks by content (malicious data rather than malicious instructions), with baseline vulnerability rates ranging from 61.7% to 98.3% across 20 models.",
       "evidence": "Table 1 shows baseline vulnerability rates for 20 models on 60 attack scenarios. The lowest is Claude Sonnet 4 at 61.7% and the highest are Cohere Command R Plus and Command A at 98.3%.",
       "supported": "moderate"
     },
     {
       "claim": "Fact-checking and source warnings are effective defenses against attacks by content, with the combination reducing vulnerability rates drastically.",
       "evidence": "Table 1 shows that combining fact-checks and source warnings reduces vulnerability for most models (e.g., Llama 3.1 8b from 78.3% to 0.0%, GPT-4.1 from 90.0% to 6.7%). However, no statistical tests are provided.",
       "supported": "moderate"
     },
     {
       "claim": "Charity fraud is the most problematic category, where almost all attacks succeeded against all models.",
       "evidence": "Table 2 shows the Charity column averaging 99.5% vulnerability across all models, with most models at 100%.",
       "supported": "strong"
     },
     {
       "claim": "Media literacy skills in LLMs do not correlate with model size — smaller models within a family are often less vulnerable.",
       "evidence": "Section 4 discusses this finding with reference to Table 1: Llama 3.1 8b (78.3%) is less vulnerable than Llama 3.1 405b (95.0%); Claude Sonnet 4 (61.7%) is less vulnerable than Claude Opus 4 (65.0%).",
       "supported": "weak"
     },
     {
       "claim": "Epistemological guidance in system prompts improves model robustness against attacks by content.",
       "evidence": "Appendix D (Table 3) shows Grok 3 baseline vulnerability drops from 91.7% with a generic prompt to 65.0% with the official Grok prompt containing epistemological warnings. However, the word 'significantly' is used without statistical testing, and this is tested on only one model.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "This position paper introduces 'attacks by content' — indirect data attacks where adversaries subvert AI agents through biased, misleading, or false information rather than injected instructions. Experiments across 20 contemporary LLMs show high vulnerability rates (61.7%-98.3% baseline), with charity fraud being the most effective attack category (99.5% average success). Automated fact-checking and source warnings are proposed as defenses, showing significant reductions when combined (e.g., reducing vulnerability from 90% to 6.7% for GPT-4.1). The paper also finds that vulnerability does not decrease with model scale within model families.",
   "red_flags": [
     {
       "flag": "Tiny evaluation dataset",
       "detail": "Only 60 synthetic scenarios are used (10 per area of concern). With such small sample sizes per category, individual scenario design choices could heavily influence category-level and overall vulnerability rates. No justification for why 60 scenarios suffice."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are point estimates with no confidence intervals, error bars, or variance measures. The paper uses the word 'significantly' (Appendix D) without any statistical test. With 10 scenarios per category, even small differences in vulnerability rates may not be statistically meaningful."
     },
     {
       "flag": "Synthetic scenarios only",
       "detail": "All 60 attack scenarios were generated by Claude 4 Opus and manually edited. No real-world attacks or naturalistic misinformation were tested. The generalizability of vulnerability rates to real attack content is unknown."
     },
     {
       "flag": "Simplistic agent design",
       "detail": "The experimental 'agent' is a single-prompt, single-response system asked to make a yes/no decision (Figures 5 and 6). Real-world agents have multi-step reasoning, tool use, and memory. The vulnerability rates may not transfer to more sophisticated agent architectures."
     },
     {
       "flag": "No false positive measurement",
       "detail": "The paper measures only vulnerability rate (passing on malicious content) but does not measure whether the defenses also cause the agent to reject legitimate information. The utility cost of the proposed defenses is not assessed."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the 20 models tested. Different temperature settings could substantially change vulnerability rates."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM-based applications, directly related to the survey's scope on AI security."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunovic", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramèr"],
       "year": 2024,
       "relevance": "Benchmark for evaluating prompt injection attacks and defenses in LLM agents, directly relevant to agent security evaluation methodology."
     },
     {
       "title": "Defeating prompt injections by design",
       "authors": ["Edoardo Debenedetti", "Ilia Shumailov", "Tianqi Fan", "Jamie Hayes", "Nicholas Carlini"],
       "year": 2025,
       "arxiv_id": "2503.18813",
       "relevance": "Proposes architectural defenses against prompt injection, relevant to the survey's coverage of LLM safety and defense mechanisms."
     },
     {
       "title": "Can llms be scammed? a baseline measurement study",
       "authors": ["Udari Madhushani Sehwag", "Kelly Patel", "Francesca Mosca", "Vineeth Ravi", "Jessica Staddon"],
       "year": 2024,
       "arxiv_id": "2410.13893",
       "relevance": "Baseline study on LLM vulnerability to scams, directly relevant to measuring AI agent security and robustness."
     },
     {
       "title": "How johnny can persuade LLMs to jailbreak them: Rethinking persuasion to challenge AI safety by humanizing LLMs",
       "authors": ["Yi Zeng", "Hongpeng Lin", "Jingwen Zhang", "Diyi Yang", "Ruoxi Jia", "Weiyan Shi"],
       "year": 2024,
       "relevance": "Studies LLM susceptibility to persuasion-based jailbreaks, relevant to understanding data-based (rather than instruction-based) attacks on AI systems."
     },
     {
       "title": "Teaching models to balance resisting and accepting persuasion",
       "authors": ["Elias Stengel-Eskin", "Peter Hase", "Mohit Bansal"],
       "year": 2025,
       "arxiv_id": "2410.14596",
       "relevance": "Addresses the challenge of models needing to accept beneficial persuasion while resisting harmful persuasion, directly relevant to agent defense."
     },
     {
       "title": "Aligning llms to be robust against prompt injection",
       "authors": ["Sizhe Chen", "Arman Zharmagambetov", "Saeed Mahloujifar", "Kamalika Chaudhuri", "Chuan Guo"],
       "year": 2024,
       "arxiv_id": "2410.05451",
       "relevance": "Proposes alignment-based defenses against prompt injection, relevant to AI safety and robustness research."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R Narasimhan", "Yuan Cao"],
       "year": 2023,
       "relevance": "Foundational work on reasoning and acting agents that are the subject of the attacks studied in this paper."
     },
     {
       "title": "The earth is flat because...: Investigating LLMs' belief towards misinformation via persuasive conversation",
       "authors": ["Rongwu Xu", "Brian Lin", "Shujian Yang"],
       "year": 2024,
       "relevance": "Studies LLM susceptibility to misinformation through persuasive conversation, directly relevant to understanding content-based attacks."
     },
     {
       "title": "Astute RAG: Overcoming imperfect retrieval augmentation and knowledge conflicts for large language models",
       "authors": ["Fei Wang", "Xingchen Wan", "Ruoxi Sun", "Jiefeng Chen", "Sercan Ö. Arık"],
       "year": 2024,
       "arxiv_id": "2410.07176",
       "relevance": "Addresses knowledge conflicts in retrieval-augmented generation, relevant to defending against content manipulation in RAG systems."
     },
     {
       "title": "Certifiably robust RAG against retrieval corruption",
       "authors": ["Chong Xiang", "Tong Wu", "Zexuan Zhong", "David Wagner", "Danqi Chen", "Prateek Mittal"],
       "year": 2024,
       "arxiv_id": "2405.15556",
       "relevance": "Proposes certifiable robustness for RAG systems against retrieval corruption, directly relevant to defending agents against content manipulation."
     },
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "relevance": "Early work on prompt injection attacks, provides foundational taxonomy that the current paper extends with the content-attack dimension."
     }
   ]
 }