ai-research-survey

scan.json (29090B)

---

 {
   "paper": {
     "title": "RTBAS: Defending LLM Agents Against Prompt Injection and Privacy Leakage",
     "authors": [
       "Peter Yong Zhong",
       "Siyuan Chen",
       "Ruiqi Wang",
       "McKenna McCall",
       "Ben L. Titzer",
       "Heather Miller",
       "Phillip B. Gibbons"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.08966",
     "doi": "10.48550/arXiv.2502.08966"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RTBAS adapts information flow control to LLM agent systems, using dependency screening (LM-Judge or attention-based) to selectively propagate security labels and mask irrelevant history regions. On the AgentDojo benchmark, RTBAS achieves 100% integrity against prompt injection attacks with <2% utility degradation. On a synthesized privacy leakage benchmark (37 tasks), RTBAS achieves 8.1% FPR and 10.8% FNR with the LM-Judge screener, significantly outperforming both always-confirm and never-confirm baselines.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "AgentDojo is publicly available, but the authors' augmented labels and the synthesized privacy leakage benchmark (37 test cases across 3 domains) are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using GPT-4o, OPT-125m, and Phi-3-Mini-128K but provides no requirements.txt, Dockerfile, or detailed environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Figures 5, 6, 7 and Tables 2, 4 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Comparative claims between RTBAS and baselines (e.g., 'outperforming state-of-the-art defenses') are based solely on comparing raw numbers with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: '10% and 7.4% degradation in utility' (§8.1.2), '<2% degradation', FPR reduced from 29.7% (GPTs) to 8.1% (LM-Judge). Raw utility values and percentage differences are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "79 user tasks for prompt injection (629 test cases) and 37 test cases for privacy leakage are used without any justification for why these sample sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: Tool Filter by AgentDojo (SOTA), Naive Tainting, Redact All for prompt injection; Confirm Never, Confirm Every Time (GPTs), and Oracle for privacy leakage (§8.1.1, §8.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Tool Filter from AgentDojo (2024) is cited as existing SOTA. They also reference and dismiss PI Detector, Delimiting, and Prompt Sandwiching from recent work as strictly worse than Tool Filter."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Two screener variants (LM-Judge and Attention-Based) are compared against each other and against Naive Tainting and Redact All, which serve as ablated versions of the framework. §8.3.2 directly compares the two screener approaches."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Prompt injection evaluation uses both Utility and Integrity metrics (§8.1.1). Privacy leakage uses FPR, FNR, and Utility (§8.2). Taint tracking accuracy is also reported (§8.3.1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated through benchmark suites. No human evaluation of the system's defense outputs is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For the attention-based classifier, §7.2 reports '85% train accuracy and 81% test accuracy', indicating a train/test split. AgentDojo provides fixed test cases not used for model development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task suite (banking, travel, workspace, slack) in Figures 5 and 6, and by domain (Amazon, flight booking, Venmo) in Figure 7."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Slack suite is discussed as an outlier with detailed explanation of why performance drops (§8.1.2). A workspace test case failure is explained where user-labeled high-integrity content contained injection. Over-tainting of flight booking history is noted (§8.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Slack suite's poor performance (33% and 22% utility for Attention and LM-Judge) is reported and explained. The paper also reports an over-tainting failure case in the flight booking domain."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '100% of targeted attacks' prevented are supported by Figure 5 (with one acknowledged exception in workspace involving user-labeled content). '<2% loss of task utility' is supported by the weighted average results. 'Near-oracle performance' on privacy leaks is supported by Table 2 and Figure 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that RTBAS 'prevents' attacks. The controlled benchmark evaluation design, where the defense mechanism is the only variable changed while model and tasks remain constant, provides adequate basis for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Defending LLM Agents' broadly, but evaluation uses only GPT-4o as the backend LM and only 4 task suites from AgentDojo plus 37 synthesized tasks. No testing with other LMs, other attack types, or real-world deployments. The paper does not explicitly bound these generalization limits."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Discussion section (§9) addresses labeling burden and cost as limitations, but does not discuss alternative explanations for the observed results. For example, it does not consider whether performance gains could be due to AgentDojo's specific attack patterns or whether the synthesized benchmark's design inherently favors IFC-based approaches."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Utility (task completion) and Integrity (attack prevention) directly. These metrics are well-defined by AgentDojo and map directly to the claims being made. The proxy gap is explicitly addressed: Integrity is defined as 'whether the attacker succeeds in their attacks' (§8.1.1)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'GPT-4o' without a snapshot date or API version. 'OPT-125m' and 'Phi-3-Mini-128K' are more specific but GPT-4o is the primary evaluation model and its version is not pinned."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The LM-Judge approach is described conceptually (region tagging with «REGION_N» markers, prompt sandwiching) but the actual prompt text used for the judge is not provided. Only the tagging format is shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or sampling settings are reported for GPT-4o. For the LSTM classifier, they mention 'two-layer LSTM' but report no learning rate, batch size, or training hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RTBAS framework is described in extensive detail through Algorithms 1-4, with formal definitions of the screener, redactor, and runtime behavior. The full information flow pipeline from screening to redaction to tool call verification is documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data labeling for AgentDojo is described in §8.1.1 with key labeling principles (low-integrity for external sources, high-integrity for side-effect tools). The privacy leakage benchmark creation is described in §8.2 with stated principles. Attention data collection from 3,424 pairs is described in §3.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 (Discussion) serves as a limitations section, discussing labeling burden ('a open problem in IFC') and computational cost ('currently resource-intensive'). The Slack suite failure analysis in §8.1.2 also functions as limitation discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: (1) 'the security guarantees provided are only as good as the labels provided' (§8.1.2), (2) Slack tasks are 'inherently unsafe' due to dependence on untrusted web content, (3) labeling burden on developers (§9), (4) resource-intensive screener operation (§9)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that security guarantees depend on label quality and policy correctness (§8.1.2). The attacker model (§5) explicitly bounds what attackers can and cannot do. The Discussion notes that labeling is 'an open problem in IFC' and cost is a limitation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, logs, or detailed per-test-case results are released. Only aggregate results in figures and tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "AgentDojo benchmark is referenced with details in Table 1 (79 tasks, 4 suites, tools, messages per test case). Privacy leakage benchmark creation is described in §8.2 and Table 3. Attention data collection described in §3.3 with 3,424 pairs and 2,416 labeled data points."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (AgentDojo) and author-synthesized test cases."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While data labeling principles are described, the full pipeline from raw benchmark data to final evaluation results has gaps. For the attention classifier, dataset collection from '40 well-labeled test cases' (§7.2) is mentioned but the split into train/test and exact dataset size are unclear."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work supported in part by the Parallel Data Lab, the WebAssembly Research Center and Cylab at Carnegie Mellon University, and the National Science Foundation under grant 2211882.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (Carnegie Mellon University, Two Sigma Investments) are clearly listed. The evaluated systems (GPT-4o, Claude) are from third parties (OpenAI, Anthropic), so no affiliation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF and CMU research centers are the funders. Heather Miller's Two Sigma affiliation is disclosed but Two Sigma has no direct financial interest in the prompt injection defense results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests a defense mechanism (RTBAS) against prompt injection attacks, not a pre-trained model's knowledge on a benchmark. The evaluation measures defense effectiveness, not model capability."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates defense frameworks, not model knowledge. Contamination of the LM's training data is not relevant to whether the defense mechanism correctly detects and blocks attacks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper tests defenses rather than model knowledge, making benchmark contamination structurally inapplicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated benchmark testing."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. §11 Ethics Considerations confirms 'No real-world user data or personally identifiable information (PII) was involved.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports average price ($), time (seconds), and token counts per test case for the banking suite. RTBAS (Attn) costs $0.028 vs $0.015 for vanilla, with runtime of 8.7s vs 4.4s."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 provides per-test-case averages for the banking suite only. Total compute budget across all experiments (all 4 suites, privacy leakage benchmark, attention model training) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from a single run or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for the LSTM classifier or any other component of the system."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents results for LM-Judge and Attention-based screeners but does not explain how design choices (e.g., two-layer LSTM, specific attention features) were selected."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is moot. However, multiple comparisons across 4 suites and multiple baselines are made without any correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own RTBAS system against baselines they re-implemented (Naive Tainting, Redact All) without acknowledging self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 directly compares runtime cost (price, time, tokens) across methods alongside performance. §8.3.3 explicitly discusses the overhead introduced by the detectors, noting that RTBAS costs roughly 2x compared to vanilla."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether AgentDojo adequately represents real-world prompt injection threats or whether the synthesized privacy benchmark captures the diversity of real privacy leakage scenarios."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "RTBAS is itself the scaffold/defense being tested. The same underlying model (GPT-4o) is used across all conditions, and the defense mechanism IS the variable under evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o's training data may include information about AgentDojo tasks or attack patterns."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information that would not be available in real deployments."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential non-independence between test cases (e.g., same task suite sharing tools and system prompts)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied to the evaluation data."
       }
     }
   },
   "claims": [
     {
       "claim": "RTBAS prevents 100% of attacks that violate the security policy on the AgentDojo benchmark.",
       "evidence": "Figure 5 shows integrity scores of 1.00 for both RTBAS variants across banking, slack, and travel suites, and 0.99 for workspace. §8.1.2 acknowledges one workspace exception where user-labeled high-integrity content contained an injection.",
       "supported": "strong"
     },
     {
       "claim": "RTBAS incurs less than 2% utility degradation under prompt injection attack compared to no defense.",
       "evidence": "Figure 5(e) weighted average shows RTBAS (LM-Judge) at 0.53 utility and RTBAS (Attn) at 0.58 utility vs baseline 0.58 without defense. Under attack, the LM-Judge loses <1% and Attention-based loses 3% (§8.1.2).",
       "supported": "moderate"
     },
     {
       "claim": "RTBAS achieves near-oracle performance on detecting privacy leaks, detecting and executing without user confirmation the same set of tool calls as the oracle for all but one task.",
       "evidence": "Table 2 shows LM-Judge FPR of 8.1% and FNR of 10.8%. Figure 7 shows utility closely matching oracle. §8.2 states 'we are losing utility only in 1 out of 15 test cases.'",
       "supported": "moderate"
     },
     {
       "claim": "Attention scores from small open-source LMs effectively capture dependency relationships in TBAS backed by large closed-source models.",
       "evidence": "Figures 2 and 3 in §3.3 show attention score distributions separating dependent from non-dependent data using OPT-125m for TBAS backed by GPT-4o and Claude. 74-86% of non-dependent attention mass is below 0.2.",
       "supported": "moderate"
     },
     {
       "claim": "The selective propagation approach arrives at exact ground truth privacy labels more than 70% (LM-Judge) and 57% (Attention) of the time.",
       "evidence": "§8.3.1 reports taint tracking accuracy: 70.0% for LM-Judge and 57.3% for Attention-based, compared to 22.3% for Redact All and 56.7% for GPTs.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small synthesized benchmark",
       "detail": "The privacy leakage benchmark contains only 37 test cases across 3 domains, all manually created by the authors. This is too small to draw robust conclusions about real-world privacy leakage detection, and author-created benchmarks risk favoring the proposed approach."
     },
     {
       "flag": "No reproducibility artifacts",
       "detail": "No code, synthesized benchmark data, augmented labels, or reproduction instructions are released. The system relies on multiple components (LM-Judge prompts, LSTM classifier, attention feature extraction) that cannot be recreated from the paper alone."
     },
     {
       "flag": "Single-run results without variance",
       "detail": "All results appear to be from single experimental runs with no error bars, confidence intervals, or variance across seeds. LLM outputs are stochastic, so single-run results may not be stable."
     },
     {
       "flag": "Model version not pinned",
       "detail": "GPT-4o is used without specifying an API version or snapshot date. Model behavior changes across versions, making results potentially unreproducible."
     },
     {
       "flag": "User confirmation modeled as rejection",
       "detail": "In the prompt injection benchmark, the paper does not model user confirmations — all flagged calls are simply skipped. This means the reported utility may underestimate actual RTBAS utility if users would confirm legitimate-but-flagged calls."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentDojo: A dynamic environment to evaluate attacks and defenses for LLM agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunović", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramèr"],
       "year": 2024,
       "arxiv_id": "2406.13352",
       "relevance": "Primary benchmark used for evaluating prompt injection attacks and defenses in LLM agent systems."
     },
     {
       "title": "Prompt injection attack against LLM-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang"],
       "year": 2024,
       "relevance": "Foundational work on prompt injection attacks against LLM-integrated applications."
     },
     {
       "title": "Formalizing and benchmarking prompt injection attacks and defenses",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng", "Jinyuan Jia", "Neil Zhenqiang Gong"],
       "year": 2024,
       "relevance": "Formalizes prompt injection attacks and defenses with benchmarking framework."
     },
     {
       "title": "StruQ: Defending against prompt injection with structured queries",
       "authors": ["Sizhe Chen", "Julien Piet", "Chawin Sitawarin", "David Wagner"],
       "year": 2024,
       "relevance": "Prompt injection defense using structured queries and data/instruction separation."
     },
     {
       "title": "Jatmo: Prompt injection defense by task-specific finetuning",
       "authors": ["Julien Piet", "Maha Alrashed", "Chawin Sitawarin", "Sizhe Chen"],
       "year": 2024,
       "relevance": "Defense approach using task-specific finetuning on non-instruction-tuned models."
     },
     {
       "title": "The instruction hierarchy: Training LLMs to prioritize privileged instructions",
       "authors": ["Eric Wallace", "Kai Xiao", "Reimar Leike", "Lilian Weng"],
       "year": 2024,
       "relevance": "Pre-training approach to enforce instruction hierarchies for prompt injection resistance."
     },
     {
       "title": "Permissive information-flow analysis for large language models",
       "authors": ["Shoaib Ahmed Siddiqui", "Radhika Gaonkar", "Boris Köpf", "David Krueger"],
       "year": 2024,
       "relevance": "Most closely related work on information flow control for LLMs, which RTBAS improves upon by reducing exponential to constant overhead."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "relevance": "Benchmark for indirect prompt injection attacks in tool-integrated LLM agents."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du"],
       "year": 2023,
       "relevance": "Foundational agent framework for tool-based LLM interaction patterns."
     },
     {
       "title": "Attention tracker: Detecting prompt injection attacks in LLMs",
       "authors": ["Kuo-Han Hung", "Ching-Yun Ko", "Ambrish Rawat"],
       "year": 2024,
       "relevance": "Uses attention mechanisms for prompt injection detection, related to RTBAS's attention-based screener."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "authors": ["Keegan Hines", "Gary Lopez", "Matthew Hall", "Federico Zarfati"],
       "year": 2024,
       "relevance": "Defense against indirect prompt injection using input transformation techniques."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr"],
       "year": 2023,
       "relevance": "Demonstrates universal adversarial attacks on aligned LMs, motivating defense research like RTBAS."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The RTBAS framework addresses a real problem (prompt injection in LLM agents) but no code is released, requiring practitioners to reimplement from the paper."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Adapting information flow control to LLMs is a novel framing but does not strongly challenge conventional wisdom about prompt injection defenses."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates concrete prompt injection and privacy leakage risks in LLM agent systems with financial and messaging examples."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or adversarial framing; straightforward defense paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Carnegie Mellon is well-respected in security and systems research but not a household name like OpenAI or Google."
     }
   }
 }