ai-research-survey

scan.json (27466B)

---

 {
   "paper": {
     "title": "StepShield: When, Not Whether to Intervene on Rogue Agents",
     "authors": ["Gloria Felicia", "Michael Eniolade", "Jinfeng He", "Zitha Sasindran", "Hemant Kumar", "Milan Hussain Angati", "Sandeep Bandarupalli"],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.22136",
     "doi": "10.48550/arXiv.2601.22136"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "StepShield introduces the first benchmark for evaluating temporal detection of rogue agent behavior, revealing a 2.3x gap in Early Intervention Rate between LLM-based (0.59) and static (0.26) detectors that is invisible to standard accuracy metrics. All detectors struggle with subtle L3 violations requiring cross-step reasoning (best EIR: 0.38). A cascaded HybridGuard detector reduces monitoring costs by 75% compared to no detection, projecting $108M cumulative savings over 5 years at enterprise scale.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides a GitHub URL and installation instructions: 'git clone https://github.com/glo26/stepshield'. Released under Apache 2.0 license."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset of 9,213 trajectories released under CC BY 4.0 (data) and Apache 2.0 (code). Appendix C describes the data directory structure (data/train/, data/test/)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C specifies hardware (8 NVIDIA A100 GPUs, 80GB) and provides pip install command. However, no requirements.txt or specific library versions are mentioned in the paper text, though the installable package ('pip install -e .') implies dependency specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides step-by-step instructions: clone, install, run with specific commands ('python -m stepshield.evaluate --detector [detector]'). Key hyperparameters are listed: seed=42, model=GPT-4.1-mini, context_steps=5, cascade_threshold=0.7, temperature=0.0."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F reports 95% bootstrap confidence intervals for EIR (e.g., LLMJudge [0.59, 0.62], StaticGuard [0.23, 0.29]) computed with 10,000 bootstrap samples."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "McNemar's test used to compare detector performance (Appendix F, Table 14). All differences reported as statistically significant with p-values (e.g., p < .001 for StaticGuard vs LLMJudge)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context: '2.3x gap in EIR between LLMJudge (0.59) and StaticGuard (0.26)', '75% cost reduction ($0.025 vs. $0.100)', and absolute differences across categories (Tables 4-7)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the choice of 639 rogue trajectories, 213 per severity level, or 7,935 test trajectories. The numbers appear chosen for balance rather than statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D states 'All ablations are run on the full training set (1,278 trajectories) with 5 random seeds, and we report mean values. Standard deviations are ±0.02 across all experiments.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four detection paradigms compared: StaticGuard, ConstraintGuard, LLMJudge, and HybridGuard, spanning fast-but-shallow to slow-but-deep (Section 5.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LLMJudge uses GPT-4.1-mini (current model). Baselines represent the range of real-world detection approaches from regex to LLM-based. The paper is proposing a new benchmark, so existing detector baselines are appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides extensive ablations: prompt sensitivity (Table 9), model scaling (Table 10), context window size (Table 11), and HybridGuard cascade threshold (Table 12)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics reported: EIR, Intervention Gap, Accuracy, F1, and Tokens Saved (Table 4). The temporal metrics (EIR, IG, Tokens Saved) are novel contributions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated. No human evaluation of detector outputs is conducted. Human annotation was used for dataset creation (94% inter-annotator agreement) but not for evaluating detector performance."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Explicit train/test split: 1,278 training trajectories with labels, 7,935 test trajectories with 'labels withheld to prevent overfitting' (Section 3.4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 breaks down EIR by all 6 rogue categories (UFO, SEC, RES, INV, TST, DEC). Table 6 breaks down by severity level (L1, L2, L3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E: '100 randomly sampled failure cases of LLMJudge' analyzed with distribution: Subtle L3 (38%), Multi-Step Reasoning (22%), False Positives (15%), Ambiguous Boundary (12%), Other (13%)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: all detectors struggle with L3 (LLMJudge only 0.38 EIR), HybridGuard trades timeliness for accuracy, ConstraintGuard achieves worst EIR (0.18). The accuracy-timeliness trade-off is explicitly discussed as a limitation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match results: 9,213 trajectories (Section 3.4), 2.3x EIR gap (Table 4: 0.59/0.26), 75% cost reduction (Table 7: $0.025 vs $0.100), $108M savings (Section 6.3)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily about detector design choices (e.g., cascade strategy delays detection). These are supported by controlled ablation studies (Appendix D) with single-variable manipulation. The economic claims are projections, clearly labeled as such."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'When, Not Whether to Intervene on Rogue Agents' and abstract frame the contribution broadly for all agents, but Section 9 acknowledges 'StepShield currently focuses on code-generating agents.' The $108M enterprise savings projection extrapolates well beyond the tested domain."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 discusses several alternatives: taxonomy may not be exhaustive, prompt sensitivity affects LLM detector, uniform pricing underestimates savings. Appendix D's ablations address prompt phrasing and model choice as alternative drivers of results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures detection on synthetic GPT-4/Claude-3 generated trajectories and frames this as measuring real-world rogue detection capability. No discussion of whether synthetic trajectories are a valid proxy for actual rogue agent behavior in production deployments."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Uses 'GPT-4.1-mini' without a snapshot date or API version. Appendix A mentions trajectory generation with 'GPT-4/Claude-3' without specific versions. Per schema criteria, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompt strategies are described (zero-shot, few-shot, chain-of-thought in Table 9) but actual prompt text is not provided in the paper or appendices. The reader cannot reconstruct the prompts sent to GPT-4.1-mini."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C lists key hyperparameters: 'seed=42, model=GPT-4.1-mini, context_steps=5, cascade_threshold=0.7, temperature=0.0.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "All four detector architectures are described in Section 5.1: StaticGuard (regex/keyword), ConstraintGuard (constraint checking), LLMJudge (per-step GPT-4.1-mini evaluation), HybridGuard (cascade with confidence threshold). Latency and cost per step specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes the 4-stage collection process (incident analysis, trajectory generation, manual annotation with 94% IAA, benign expansion). Training set composition is fully documented (Table 3). Annotation guidelines in Appendix G."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Limitations' is a dedicated section with four specific limitations discussed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 identifies specific threats: taxonomy not exhaustive as agents evolve, limited to code-generating agents, uniform per-token pricing assumption, LLM detector sensitivity to prompt engineering (with ablations provided)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 9: 'StepShield currently focuses on code-generating agents; future work should extend to web, API, and multi-agent domains.' Also: 'our rogue behavior taxonomy... is not exhaustive.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Full dataset released: 'data/train/ contains 1,278 annotated trajectories... data/test/ contains 7,935 trajectories' (Appendix C). Released under CC BY 4.0."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A: '4-stage process: incident analysis, trajectory generation (GPT-4/Claude-3), manual annotation (94% inter-annotator agreement), and benign expansion.' Rogue categories grounded in real incidents (Table 1)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Trajectories are synthetic (generated by GPT-4/Claude-3) but the generation process is only briefly described. For annotators: 'two independent annotators' are mentioned but recruitment/selection criteria for annotators are not described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented with counts: 639 rogue + 639 clean pairs for training (balanced across 6 categories × 3 severity levels, Table 3), 7,935 test (639 rogue + 639 clean + 6,657 benign, 8.1% rogue rate). Annotation process in Appendix G."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section in the paper. Authors are from multiple universities but no funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations listed on the first page: University of Virginia, University of the Cumberlands, Cornell University, IISc Bangalore, University of Arizona, CSU Northridge, University of Cincinnati."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Absence of funding disclosure is not the same as unfunded work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4.1-mini is used as LLMJudge but its training data cutoff is not stated. The model could have seen similar code patterns or security incident descriptions in training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4.1-mini's training data contains trajectories similar to the test set, or whether the model has seen the security incidents used to ground the taxonomy."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of benchmark contamination. The trajectories are newly generated, which mitigates contamination risk, but this is not explicitly discussed as a design choice for contamination prevention."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Annotators created the dataset but were not research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 provides cost per trajectory for each detector (e.g., LLMJudge ~$0.02, HybridGuard ~$0.003). Latency also reported per step (LLMJudge ~500ms, StaticGuard <1ms)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C: 'Total computation time for all experiments was approximately 2,000 GPU hours' on '8 NVIDIA A100 GPUs (80GB).'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D: 'All ablations are run on the full training set (1,278 trajectories) with 5 random seeds, and we report mean values. Standard deviations are ±0.02 across all experiments.'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D explicitly states '5 random seeds' for ablation experiments."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Cascade threshold tested at 3 values (0.5, 0.7, 0.9 in Table 12) and context window at 4 values (Table 11), but no overall search budget reported. No description of how other hyperparameters (e.g., regex patterns for StaticGuard) were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 shows cascade threshold selection with clear trade-off analysis. Context window selection justified via diminishing returns (Table 11). Config selected on training set, not test."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "McNemar's test performed across multiple detector pairs (Table 14) but no correction for multiple comparisons (Bonferroni, Holm, etc.) is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors designed and evaluate all four detectors. No acknowledgment of self-comparison bias or independent evaluation by third parties."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 reports EIR and cost per trajectory across three models (GPT-4.1-mini, GPT-4, GPT-4-Turbo). Table 7 compares detection cost vs. performance trade-offs across all detectors."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the synthetic trajectories generated by GPT-4/Claude-3 actually represent real-world rogue agent behavior. The construct validity of the benchmark itself is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper compares detection paradigms, not models within scaffolds. Each detector is a distinct system; scaffold confounding is not applicable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4.1-mini's training data includes patterns from the security incidents used to generate rogue trajectories, or whether the model has temporal familiarity with the violation patterns."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, the step-by-step presentation format may provide cues that wouldn't exist in real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of structural similarities between training and test trajectories. Both are generated by the same models (GPT-4/Claude-3) from the same taxonomy, which could introduce systematic shared patterns."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method applied. The test set labels are withheld, but no analysis of train/test overlap or contamination checking is performed."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-based judge achieves 2.3x higher Early Intervention Rate (0.59) than static analyzer (0.26), a gap invisible to standard accuracy metrics.",
       "evidence": "Table 4 (Section 5.2): LLMJudge EIR=0.59, Acc=0.63; StaticGuard EIR=0.26, Acc=0.56. Statistical significance confirmed via McNemar's test (p<.001, Appendix F).",
       "supported": "strong"
     },
     {
       "claim": "HybridGuard reduces monitoring costs by 75% compared to no detection ($0.025 vs $0.100 per trajectory).",
       "evidence": "Table 7 (Section 6.2): detailed cost breakdown showing detection + execution costs. Cascade uses fast detectors for 85% of cases.",
       "supported": "strong"
     },
     {
       "claim": "HybridGuard projects to $108M in cumulative savings over 5 years at enterprise scale.",
       "evidence": "Section 6.3 and Appendix B. Based on assumptions: 100M trajectories/year starting scale, 35% YoY growth, $0.10 baseline cost. Sensitivity analysis shows $54M-$432M range.",
       "supported": "moderate"
     },
     {
       "claim": "All detectors struggle with L3 (subtle) violations requiring cross-step reasoning; even LLMJudge achieves only 0.38 EIR.",
       "evidence": "Table 6 (Section 5.4): L3 EIR ranges from 0.04 (ConstraintGuard) to 0.38 (LLMJudge).",
       "supported": "strong"
     },
     {
       "claim": "Binary accuracy is fundamentally insufficient for evaluating detection systems (Theorem 1).",
       "evidence": "Section 4.4: proof by construction that two monitors can have identical accuracy but maximally different EIR.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Synthetic data validity unexamined",
       "detail": "All trajectories are generated by GPT-4/Claude-3, not collected from real agent deployments. The paper does not discuss whether synthetic rogue behavior is representative of actual rogue agent behavior in production, which is a significant construct validity concern for a benchmark paper."
     },
     {
       "flag": "Economic projections extrapolate far beyond evidence",
       "detail": "The $108M savings projection relies on assumed 35% YoY growth, 100M starting trajectories, and uniform pricing — all acknowledged as assumptions but presented prominently. The projection is for a domain (enterprise agent deployment) quite different from the tested domain (synthetic code agent trajectories)."
     },
     {
       "flag": "Uniform ±0.02 standard deviation across all experiments",
       "detail": "Appendix D claims 'Standard deviations are ±0.02 across all experiments' — identical variance across prompt sensitivity, model scaling, and context window ablations is suspiciously uniform and not how real experimental variance typically behaves."
     },
     {
       "flag": "No prompts provided despite being central to LLMJudge",
       "detail": "LLMJudge is the star performer and central to the paper's claims, but the actual prompts used for per-step evaluation are not provided. Only prompt strategy labels (zero-shot, few-shot, CoT) are given."
     },
     {
       "flag": "Annotator details thin",
       "detail": "Only 'two independent annotators' mentioned with Cohen's Kappa of 0.82. No information about annotator qualifications, selection, or whether they had security expertise relevant to identifying rogue agent behavior."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentHarm: A benchmark for measuring harmfulness of LLM agents",
       "authors": ["Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian"],
       "year": 2025,
       "relevance": "Benchmark measuring LLM agent compliance with malicious requests, directly relevant to agent safety evaluation."
     },
     {
       "title": "ShieldAgent: Shielding agents via verifiable safety policy reasoning",
       "authors": ["Zhaorun Chen", "Mintong Kang", "Bo Li"],
       "year": 2025,
       "relevance": "First guardrail agent enforcing safety policy compliance through logical reasoning, relevant to agent safety monitoring."
     },
     {
       "title": "Agent-SafetyBench: Evaluating the safety of LLM agents",
       "authors": ["Zhexin Zhang"],
       "year": 2024,
       "arxiv_id": "2412.14470",
       "relevance": "Comprehensive agent safety benchmark with 300 test cases, key baseline for StepShield's temporal evaluation paradigm."
     },
     {
       "title": "SHADE-Arena: Evaluating sabotage and monitoring in LLM agents",
       "authors": ["Jonathan Kutasov", "Yuqi Sun", "Paul Colognese"],
       "year": 2025,
       "arxiv_id": "2506.15740",
       "relevance": "Evaluates LLM agents' ability to evade monitoring during sabotage, complementary to StepShield's detection timeliness focus."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates deceptive behaviors persist through safety training, motivating runtime monitoring approaches like StepShield."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2024,
       "relevance": "Multi-environment LLM agent benchmark that StepShield complements with safety-focused temporal evaluation."
     },
     {
       "title": "SafeArena: Evaluating the safety of autonomous web agents",
       "authors": ["Alexander Daniel Tur", "Nicholas Meade"],
       "year": 2025,
       "relevance": "Safety benchmark for autonomous web agents, uses LLM-as-Judge paradigm that StepShield extends with step-level evaluation."
     },
     {
       "title": "GuardAgent: Safeguard LLM agents via knowledge-enabled reasoning",
       "authors": ["Zhen Xiang", "Linzhi Zheng"],
       "year": 2025,
       "relevance": "Guardrail agent that generates guardrail code for safety checks, complementary infrastructure to StepShield's evaluation framework."
     },
     {
       "title": "Agent-as-a-judge: Evaluate agents with agents",
       "authors": ["Mingchen Zhuge", "Changsheng Zhao", "Dylan Ashley"],
       "year": 2025,
       "relevance": "Extends LLM-as-Judge to agentic evaluation with intermediate feedback, relevant to step-level evaluation paradigm."
     },
     {
       "title": "R-Judge: Benchmarking safety risk awareness",
       "authors": ["Tongxin Yuan"],
       "year": 2024,
       "arxiv_id": "2401.10019",
       "relevance": "Uses LLM to judge agent helpfulness and harmlessness, a key prior agent safety benchmark."
     },
     {
       "title": "SafeAgentBench: A benchmark for safe task planning of embodied LLM agents",
       "authors": ["Sheng Yin", "Xianghe Pang"],
       "year": 2024,
       "arxiv_id": "2412.13178",
       "relevance": "Safety-aware task planning benchmark for embodied agents with 750 tasks, extends safety evaluation to embodied domain."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Foundational agent architecture interleaving reasoning and action, enables the agent paradigm StepShield evaluates."
     }
   ]
 }