ai-research-survey

scan-v4.json (33066B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "DRIFT: Dynamic Rule-Based Defense with Injection Isolation for Securing LLM Agents",
     "authors": [
       "Hao Li",
       "Xiaogeng Liu",
       "Hung-Chun Chiu",
       "Dianqi Li",
       "Ning Zhang"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2506.12104",
     "doi": "10.48550/arXiv.2506.12104"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (ASR reduced from 30.7% to 1.3%, utility outperforms CaMeL) are supported by Figure 3 and detailed results in Section 3.2. Cross-model generalization claims are supported by Figure 5.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about each component's contribution are supported by the ablation study (Table 1), which uses controlled single-variable manipulation. Each component is added incrementally, isolating its effect.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies evaluation on 'AgentDojo and ASB benchmark' with specific models. The limitations section (Appendix A) explicitly states 'the benchmark domains are limited and do not fully cover the diverse tasks and attack scenarios encountered in real-world agentic systems.'",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, the utility improvements could partly stem from additional LLM reasoning calls rather than the security mechanism itself. The paper doesn't consider confounding factors.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Metrics (Benign Utility, Utility Under Attack, ASR) are clearly defined and match the granularity of claims. The paper doesn't inflate 'success on AgentDojo tasks' into broader claims about real-world security beyond what's measured.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A contains a dedicated 'Limitations' section discussing benchmark domain limitations.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section states only that 'the benchmark domains are limited and do not fully cover the diverse tasks and attack scenarios encountered in real-world agentic systems.' This is close to boilerplate — it doesn't identify specific threats like benchmark overfitting, GPT-4o-mini's weaknesses affecting generalization, or whether the training data generation pipeline introduces biases.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section does not enumerate specific untested scenarios, attack types, deployment contexts, or populations. 'Diverse tasks and attack scenarios' is vague. No equivalent of 'What the evidence does not show.'",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This project is partially supported by Schmidt Science AI2050 Early Career Fellow and Open philanthropy.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Washington University in St. Louis, Johns Hopkins University, and Independent Researchers. No conflicts with evaluated products.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Schmidt Science and Open Philanthropy are independent foundations with no financial stake in the evaluated defense mechanism or the models used.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Absence of disclosure is not absence of conflict.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'prompt injection attack' is explained with an example, 'Secure Planner/Dynamic Validator/Injection Isolator' are defined in Section 2, and privilege categories (Read/Write/Execute) are explicitly defined in Section 2.2.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Contributions are explicitly bulleted at the end of the introduction: a new framework (DRIFT) and extensive empirical validation of its effectiveness and adaptability.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B and Section 1 engage substantively with prior defenses (IsolateGPT, CaMeL, Progent, StruQ), explaining how DRIFT addresses their specific limitations rather than merely listing them.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Code is released at https://github.com/SaFoLab-WISC/DRIFT, stated in the abstract and confirmed in the NeurIPS checklist.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "They use publicly available benchmarks (AgentDojo, ASB) and state 'We will release our DRIFT training dataset' in the NeurIPS checklist. The benchmarks themselves are open-sourced.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. Training details (batch size, optimizer, learning rate) are mentioned in Section 3.1 but no software environment or dependency specifications.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes implementation details in Section 3.1 but does not provide step-by-step reproduction instructions. Code is released in supplementary materials but the paper lacks a 'Reproducing Results' section.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are point estimates without confidence intervals or error bars. The NeurIPS checklist item 7 explicitly answers 'No' for error bars.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are used. Claims like 'DRIFT outperforms CaMeL by 20.1%' are based on comparing point estimates without any tests.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are given as absolute and relative differences with baseline context: e.g., 'ASR is successfully reduced from 30.7% to 1.3%', 'utility outperforms CaMeL by 20.1% under no attack' (Section 3.2). Both baseline and DRIFT numbers are provided throughout.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification for the sample sizes. AgentDojo has 97 user tasks and 629 injection tasks; ASB has 10 scenarios. No power analysis or discussion of whether these are sufficient.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results appear to be single-run with no standard deviations, variance, or multi-run analysis. The NeurIPS checklist confirms no error bars are reported.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Multiple baselines compared: repeat_user_prompt, spotlighting_with_delimiting, tool_filter, transformers_pi_detector, CaMeL, Progent on AgentDojo (Figure 3); delimiters_defense, ob_sandwich_defense, instructional_prevention, Progent on ASB (Figure 4).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include CaMeL (2025) and Progent (2025), both concurrent system-level defenses. Other baselines span 2023-2025.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 presents a thorough ablation: Native Agent → +Planner → +Planner+Validator → +Planner+Validator+Isolator, plus Isolator alone. Each component's contribution is measured.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Three metrics are used throughout: Benign Utility (task completion without attacks), Utility Under Attack (task completion with attacks), and Attack Success Rate (ASR).",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "All evaluation is fully automated using benchmark success criteria. No human evaluation of agent outputs, defense decisions, or utility quality.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Training data is collected from ToolBench (Section 2.5.1) while evaluation uses AgentDojo and ASB, which are completely separate benchmarks.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D (Tables 6-8) provides per-scenario breakdowns across Banking, Slack, Travel, and Workspace for all models. Figure 6a also shows per-session comparison.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section C.1 analyzes failure on open-ended tasks (Table 4 shows DRIFT achieves 17.6% vs base agent's 25.7%). Section C.3 provides a detailed case study. Section 3.5 shows where static policies fail (trajectory length ≥3).",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 shows the strict static policy (Planner only) causes severe utility drops (63.55% → 37.71%). Table 4 shows DRIFT reduces open-ended task completion. Claude-3-haiku shows limited utility improvement (Table 6). These negative aspects are reported transparently.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "GPT-4o-mini is specified as 'GPT-4o-mini-2024-07-18' (Section 3.1), but GPT-4o has no snapshot date, Claude-3.5-sonnet and Claude-3-haiku lack version identifiers. Per schema, 'GPT-4o' without a snapshot date does not count.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full prompt texts are provided in Appendix E (Figures 8-13) for the Secure Planner, Privilege Assignment, Intent Alignment Validation, Injection Detection, Planning Sampling, and Injection Sampling.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Training hyperparameters are reported (batch size 4, 3 epochs, Adam optimizer, lr 2e-5 in Section 3.1). However, LLM API inference settings (temperature, top-p, max tokens) for the online models are not reported.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The agentic scaffolding is described in detail across Sections 2.1-2.4, with workflow diagrams (Figures 1, 2), including the Secure Planner, Dynamic Validator, Injection Isolator, and their interaction with the agent and tools.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.5.1 describes the full data construction pipeline: ToolBench conversations are modified using GPT-4o-mini for both Planner and Isolator training data. Tool environment reconstruction adds 0-25 extra tools. 1,000 samples collected for each component.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No raw experimental data (per-task results, logs, model outputs) is released. Only aggregate metrics are presented in figures and tables.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.5.1 describes how training data was collected from ToolBench: assistant messages rewritten by GPT-4o-mini, injection content simulated and placed by GPT-4o-mini. Evaluation uses standard benchmarks with described setups.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data comes from standard benchmarks (AgentDojo, ASB) and ToolBench.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.5.1 documents the pipeline: ToolBench conversations → GPT-4o-mini rewrites assistant messages → 1,000 Planner samples and 1,000 Isolator samples collected → tool environment augmented with 0-25 extra tools from 10,000+ unique tools across 5,000 samples.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "This paper tests defense mechanisms against prompt injection attacks, not model knowledge or capability on benchmarks. Contamination of benchmark tasks in training data is not the primary concern.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Paper evaluates a defense system, not pre-trained model capabilities. Any contamination would affect all compared methods equally.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Paper evaluates defense effectiveness, not model knowledge. Benchmark contamination is not structurally relevant to the defense comparison.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study. Evaluation is entirely automated via benchmarks.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 reports total token usage for DRIFT (2.37M tokens) and all baselines on AgentDojo, plus an efficiency metric (Utility−ASR)/Total Tokens. DRIFT uses ~1.89× more tokens than the undefended agent.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Token usage for inference is reported in Table 3, but no GPU hours, training time, or total API spend for the LoRA fine-tuning of Qwen2.5-7B-Instruct is provided.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results appear to be single-run. No seed sensitivity analysis or multi-seed results are reported.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of experimental runs is never stated. Results are presented as point estimates without indication of how many runs produced them.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameter search budget is reported. Training hyperparameters (lr, batch size, epochs) are stated but not how they were selected.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "No explanation of how the final configuration was selected or whether alternative configurations were tried.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "Multiple comparisons across 5 models, 2 benchmarks, 4 scenarios, and multiple baselines are made without any correction for multiple testing.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Authors implement and evaluate their own system against baselines without acknowledging potential self-comparison bias. No independent evaluation.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 compares DRIFT's performance against token cost for all methods, and computes an efficiency metric. DRIFT's cost relative to performance is explicitly analyzed.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether AgentDojo or ASB actually measure real-world agent security. The limitations section notes benchmark domains are limited but does not question construct validity of the benchmarks themselves.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Model comparisons in Figure 5 use ReAct as the consistent baseline scaffold across all models. DRIFT is applied as a system-level addition on top of the same base scaffold, controlling for scaffold confounds.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether evaluated models may have seen AgentDojo or ASB benchmark tasks during training, or whether the benchmarks were created before model training cutoffs.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the evaluation setup leaks information about attack presence or correct task execution.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of independence between training data (ToolBench) and evaluation data (AgentDojo), or structural similarities that might bias results.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No leakage detection or prevention method is applied.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DRIFT reduces attack success rate from 30.7% to 1.3% on GPT-4o-mini in AgentDojo while outperforming CaMeL in utility by 21.8% (no attack) and 10.9% (under attack).",
       "evidence": "Figure 3 shows the direct comparison across all defenses. Table 1 ablation confirms each component's contribution.",
       "supported": "strong"
     },
     {
       "claim": "DRIFT generalizes across five LLMs (GPT-4o, GPT-4o-mini, Claude-3.5-sonnet, Claude-3-haiku, Qwen2.5-7B), reducing ASR to single digits for all models.",
       "evidence": "Figure 5 shows per-model results; ASR ranges from 0.0% to 4.4% with DRIFT across all five models.",
       "supported": "strong"
     },
     {
       "claim": "Policy-tuned Qwen2.5-7B-Instruct achieves 0.0% ASR while improving utility by 5.6% over the base DRIFT configuration.",
       "evidence": "Figure 5 shows Qwen +DRIFT at 0.0% ASR vs. 15.1% for +ReAct, with utility increasing from 26.6% to 32.2%.",
       "supported": "strong"
     },
     {
       "claim": "Dynamic policy is essential for complex tasks: static policy utility drops sharply for trajectory length ≥3, while dynamic policy remains stable.",
       "evidence": "Figure 6b shows the trajectory-length scaling law; static policy drops below 30% success rate for length ≥4, dynamic policy maintains ~45%.",
       "supported": "strong"
     },
     {
       "claim": "The Injection Isolator fills a defense gap not covered by control/data constraints, catching attacks that manipulate final output without deviating the tool trajectory.",
       "evidence": "Table 1 shows Isolator alone reduces ASR from 30.67% to 7.95%; Figure 7 demonstrates a case where trajectory is correct but injected output appears.",
       "supported": "strong"
     },
     {
       "claim": "DRIFT is robust against adaptive attacks, with only a 0.81% ASR increase under combined isolator+validator adaptive attacks and 0.31% under PAIR.",
       "evidence": "Table 2 shows full adaptive attack results; the increases are minor relative to the defended baseline.",
       "supported": "moderate"
     },
     {
       "claim": "DRIFT operates at lower token cost than CaMeL (2.37M vs. 6.09M tokens) while achieving better utility-security tradeoff.",
       "evidence": "Table 3 reports total tokens and efficiency metric; DRIFT efficiency score is 19.7 vs. CaMeL's 5.8.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DRIFT achieves a substantially better security-utility tradeoff than prior system-level defenses against prompt injection: it reduces attack success rate from 30.7% to 1.3% on AgentDojo while improving utility by 21.8% over the next-best secure method (CaMeL). The dynamic constraint mechanism proves critical for multi-step tasks — static policies degrade sharply at trajectory length ≥3, explaining why prior static defenses sacrifice task completion. A complementary Injection Isolator addresses a distinct attack vector (output manipulation without tool-call deviation) that control/data constraints alone cannot block. Policy fine-tuning on Qwen2.5-7B achieves 0.0% ASR, suggesting that simpler dynamic subtasks (privilege classification, intent alignment) are within reach of smaller models.",
   "red_flags": [
     {
       "flag": "No variance across runs",
       "detail": "All results are single-run point estimates. The NeurIPS checklist explicitly answers 'No' to error bars. For stochastic LLM systems, single-run results can vary substantially; without variance no claim about margins is reliable."
     },
     {
       "flag": "Most model versions unpinned",
       "detail": "Only GPT-4o-mini has a specific version (2024-07-18). GPT-4o, Claude-3.5-sonnet, and Claude-3-haiku are listed without snapshot dates, making exact reproduction impossible as these models are updated."
     },
     {
       "flag": "Inference hyperparameters unreported",
       "detail": "Temperature, top-p, and max-token settings for all LLM calls (Secure Planner, Dynamic Validator, Injection Isolator, agent) are not reported, affecting reproducibility."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "AgentDojo (NeurIPS 2024) is publicly available and may be in the training corpora of GPT-4o and Claude models evaluated here; no discussion of this potential confound."
     },
     {
       "flag": "Training data not released",
       "detail": "The 1,000-sample Planner training dataset and 1,000-sample Isolator dataset are 'will release' at submission time, not yet available; the fine-tuned Qwen model results cannot be reproduced."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents",
       "relevance": "Primary evaluation benchmark; provides the AgentDojo scenarios (banking, Slack, travel, workspace) used throughout the paper."
     },
     {
       "title": "Defeating Prompt Injections by Design (CaMeL)",
       "relevance": "Key static policy-based defense baseline that DRIFT directly improves upon in both security and utility."
     },
     {
       "title": "Progent: Programmable Privilege Control for LLM Agents",
       "relevance": "Concurrent dynamic policy defense; extensively compared against in main results and Appendix C.2."
     },
     {
       "title": "IsolateGPT: An Execution Isolation Architecture for LLM-Based Agentic Systems",
       "relevance": "Prior isolation-based defense that motivates the Injection Isolator design."
     },
     {
       "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated LLM Agents",
       "relevance": "Characterizes the prompt injection attack surface that DRIFT defends against."
     },
     {
       "title": "ToolBench: Facilitating Large Language Models to Master 16000+ Real-World APIs",
       "relevance": "Source dataset used to construct DRIFT's policy training data for the fine-tuned Qwen model."
     },
     {
       "title": "Agent Security Bench (ASB): Formalizing and Benchmarking Attacks and Defenses in LLM-Based Agents",
       "relevance": "Second evaluation benchmark used to validate DRIFT's generalization beyond AgentDojo."
     },
     {
       "title": "Jailbreaking Black Box Large Language Models in Twenty Queries (PAIR)",
       "relevance": "Advanced adaptive attack used in the stress test (Section 3.6) to evaluate DRIFT's robustness."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "relevance": "Baseline agent framework compared against in all multi-model experiments."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "DRIFT is a deployable defense framework with released code applicable to existing LLM agents, but requires non-trivial integration into agentic systems."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Shows dynamic policies substantially outperform static ones and that memory isolation matters — useful but not deeply surprising given the motivation."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that even GPT-4o has 51.7% ASR when undefended against prompt injection, highlighting real security risks in LLM agentic systems."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy — straightforward defense paper comparing against other methods without contentious claims."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub (SaFoLab-WISC/DRIFT) and benchmarks are publicly available, but not pip-installable or live demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Washington University in St. Louis and Johns Hopkins University — reputable universities but not major AI labs. Uses well-known models (GPT-4o, Claude) in experiments."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44770561",
         "title": "B-Splines and Fourier-Best Friends for Spatial-Temporal Video Super-Resolution",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44770561"
       },
       {
         "hn_id": "47002668",
         "title": "LLMs exceed physicians on complex text-based differential diagnosis",
         "points": 3,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=47002668"
       },
       {
         "hn_id": "45534337",
         "title": "Advancing medical artificial intelligence using a century of cases",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45534337"
       },
       {
         "hn_id": "43401539",
         "title": "CriteoPrivateAd: RealWorld Bidding Dataset to Design Private Advertising Systems",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43401539"
       },
       {
         "hn_id": "31894669",
         "title": "Protecting President Zelenskyy Against Deep Fakes",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=31894669"
       },
       {
         "hn_id": "27612994",
         "title": "LegoFormer: Transformers for Block-by-Block Multi-View 3D Reconstruction",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=27612994"
       },
       {
         "hn_id": "44971660",
         "title": "Scaling laws found in large generative medical event models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44971660"
       },
       {
         "hn_id": "41227450",
         "title": "Τ-Bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41227450"
       },
       {
         "hn_id": "40782080",
         "title": "Should AI optimize your code? A studio",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40782080"
       },
       {
         "hn_id": "28895006",
         "title": "IQ-Learn: Inverse Soft-Q Learning for Imitation",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=28895006"
       }
     ],
     "top_points": 4,
     "total_points": 20,
     "total_comments": 4
   }
 }