ai-research-survey

scan.json (28039B)

---

 {
   "paper": {
     "title": "Policy-as-Prompt: Turning AI Governance Rules into Guardrails for AI Agents",
     "authors": ["Gauri Kholkar", "Ratinder Ahuja"],
     "year": 2025,
     "venue": "NeurIPS 2025 Workshop on Regulatable ML",
     "arxiv_id": "2509.23994"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The Policy-as-Prompt framework converts unstructured design documents (PRDs, TDDs) into enforceable LLM guardrails via a two-stage pipeline: policy tree extraction and prompt-based classification. O1 significantly outperforms other models in policy extraction (60% F1 on HR vs ≤25% for others). For runtime policy enforcement, GPT-4o achieves 73%/71% accuracy on input/output classification while smaller models (Qwen3-1.7B) reach 66%/59%. The evaluation is limited to two enterprise domains (HR, SOC) with small proprietary gold sets (100 inputs/outputs each) that cannot be released.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation data consists of proprietary internal enterprise artifacts. Section 6 states: 'Reproducibility is further hindered by confidentiality of internal artifacts and logs: we cannot release the full corpora.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The proprietary nature of the data and the lack of released code make reproduction impossible."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates (e.g., '0.73' accuracy, '60.0' F1) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'o1 model significantly outperforms all other models' but provides no statistical significance tests — the comparison is based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw accuracy and F1 scores are reported. No formal effect sizes (Cohen's d, odds ratios) are provided for the model comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The gold set consists of 100 inputs and 100 outputs per application. No justification for this sample size is given, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 states 'All reported metrics are the average of multiple runs to ensure stability' but never reports standard deviation, variance, or the number of runs, making it impossible to assess result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models are compared: O1, GPT-OSS 120B, Llama 405B, Claude 3.5 for POLICY-TREE-GEN (Tables 2, 4), and GPT-4o, Qwen3-1.7B, Gemma-1B for POLICY-AS-PROMPT-GEN (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models used are contemporary: o1, GPT-4o, GPT-OSS 120B, Llama 3 405B, Claude Sonnet 3.5, Gemma 3 1B, and Qwen3 1.7B."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (POLICY-TREE-GEN with two-step parse/classify and enrich, POLICY-AS-PROMPT-GEN with input/output classifiers). No ablation study examines the contribution of individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Recall, F1, Macro-F1, per-class F1 (Table 2); Detection Precision, Micro-F1, Span Exact, Token-F1, Substr, Emb Cos (Table 4); Input/Output Accuracy (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Security engineers reviewed the generated policies as part of deployment approval ('Both policies were judged request review by security engineers'), but this is a workflow step, not a systematic human evaluation with reported inter-rater agreement or quantified assessments."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "100 gold inputs and outputs are used for testing, but there is no explicit separation of dev and test splits. It is unclear whether prompts were tuned using any portion of this gold set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by application (HR vs SOC), by model, and by per-class F1 (ID-I, ID-O, OOD-I, OOD-O) in Table 2. Input vs output accuracy is shown separately in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis is provided. The paper notes some models had low scores and two were excluded for poor performance, but does not show specific examples of where the approach breaks down or analyze error patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Gemma 3 1B and Qwen 1.7B were excluded from POLICY-TREE-GEN due to poor performance. Claude 3.5 Sonnet achieved very low F1 scores (8.0% on HR, 5.4% on SOC). These negative results are reported transparently."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the system 'reduces prompt-injection risk, blocks out-of-scope requests, and limits toxic outputs.' Tables 2-4 show the system can classify inputs/outputs as ID/OOD with measured accuracy, supporting these claims at the measured performance level (70-73% for GPT-4o)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the system 'reduces prompt-injection risk' and 'limits toxic outputs,' which are causal claims. The evaluation only measures classification accuracy on gold examples, not actual risk reduction in deployment. No causal design (RCT, A/B test) supports these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and title claim 'scalable AI safety and AI security assurance for regulatable ML' and 'Guardrails for AI Agents' broadly, but the evaluation covers only two enterprise domains (HR, SOC) with proprietary data. The limitations section acknowledges this but the framing throughout is much broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions prompt tuning bias and limited domains but does not discuss alternative explanations for the observed results — e.g., whether gold test set difficulty, policy structure simplicity, or model familiarity with similar tasks could explain the performance patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures classification accuracy on gold ID/OOD labels and frames this as 'reduces prompt-injection risk' and 'enables secure-by-design deployment.' The gap between accuracy on curated gold examples and actual risk reduction in production is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names: 'o1', 'GPT-4o', 'Claude Sonnet 3.5', 'Llama 3 405B', 'Gemma 3 1B', 'Qwen 3 1.7B'. No API version identifiers, snapshot dates, or model checksums are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in the appendix: Figure 2 (Pass 1 parsing prompt), Figure 3 (Pass 2 example extraction prompt), Figure 4 (HR input classifier prompt), and Figure 5 (SOC content classifier prompt)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, sampling settings) are reported for any of the models used."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-stage pipeline (POLICY-TREE-GEN → POLICY-AS-PROMPT-GEN) is described in Sections 2-3 with Figure 1 showing the overall architecture. The two-pass extraction process and the AI double-checking agent are described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Only minimal preprocessing is mentioned: 'all documents were converted to Markdown format, and any embedded images were replaced with textual descriptions generated by gpt-4o.' No details on how the 100 gold inputs/outputs were created, selected, or validated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitations' section with substantive discussion of generalizability, reproducibility, and evaluation constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies threats specific to this study: 'modest gold sets (100 inputs and 100 outputs per application)', 'prompting effort was greater for GPT-family models, potentially biasing results', and 'confidentiality of internal artifacts and logs' preventing reproduction."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'Our evaluation spans two enterprise-style domains (HR, SOC)...which constrains transferability to other settings and larger, more heterogeneous corpora.' This acknowledges the narrow scope of the evaluation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 states: 'Reproducibility is further hindered by confidentiality of internal artifacts and logs: we cannot release the full corpora.' No raw data is available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states artifacts were 'sourced from real-world, internal enterprise projects' and 'gold policies were created by security engineers,' but provides no detail on how projects were selected, how gold labels were determined, or the annotation process."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the enterprise projects were selected, how security engineers who created gold policies were chosen, or whether the selection could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The high-level pipeline is shown in Figure 1, but the full data pipeline from raw enterprise documents to final evaluation metrics is not documented. How gold examples were constructed from the artifacts is unexplained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. Both authors are from Pure Storage but there is no acknowledgments section or funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors list Pure Storage as their affiliation with institutional email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. The authors work at Pure Storage and evaluate a framework potentially useful for their employer's products. Independence of outcome cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (o1, GPT-4o, Claude 3.5, Llama 405B, etc.)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. While the evaluation data is proprietary (reducing contamination risk), this is not explicitly discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models could have been exposed to similar policy documents or classification tasks during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Security engineers who create gold policies and review outputs are part of the development team, not research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported despite the paper claiming 'lightweight' and 'real-time' monitoring. The framework involves multiple LLM calls per input but no cost quantification is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget (API costs, GPU hours, total runs) is stated despite evaluating multiple large language models across multiple runs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are stated to be 'the average of multiple runs' but no seed sensitivity analysis, standard deviation, or variation across runs is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 says 'All reported metrics are the average of multiple runs to ensure stability' but the exact number of runs is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The prompts appear tuned (especially for GPT models per the limitations section) but no search details are provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the final configurations or prompt versions were selected. The limitations section acknowledges GPT-biased tuning but provides no selection methodology."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple models are compared across two applications with many metrics. No correction for multiple comparisons is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own framework without acknowledging self-evaluation bias or using independent evaluators. Gold policies were created by security engineers, but whether these were independent of the framework developers is not stated."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 1B to 405B parameters are compared without discussing compute budget differences. Performance is not reported as a function of compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the gold set of 100 inputs/outputs actually measures real-world policy enforcement capability, or whether accuracy on this specific benchmark correlates with deployment safety."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same prompts and pipeline structure are used across all model comparisons within each stage (POLICY-TREE-GEN and POLICY-AS-PROMPT-GEN), holding the scaffold constant. The limitations section acknowledges prompt tuning was GPT-biased."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any models could have been trained on similar policy documents or classification tasks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup or prompt context leaks classification information to the models."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and the proprietary test artifacts."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "O1 significantly outperforms all other models in policy tree extraction, with 60% F1 on HR and 22.6% on SOC.",
       "evidence": "Table 2 shows O1 achieving 60.0% F1 and 53.3% recall on HR, compared to 25.0% F1 for GPT-OSS 120B and 14.5% for Llama 405B. On SOC, O1 achieves 22.6% F1 vs ≤10.3% for others.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o achieves 73% input and 71% output classification accuracy for policy enforcement, demonstrating 'significant utility in a real-world context.'",
       "evidence": "Table 3 shows GPT-4o accuracies of 0.73 (input) and 0.71 (output) for HR, and 0.70/0.68 for SOC. No confidence intervals, significance tests, or comparison to a no-guardrail baseline.",
       "supported": "weak"
     },
     {
       "claim": "Smaller models can effectively enforce policies with curated prompts.",
       "evidence": "Table 3 shows Qwen3-1.7B at 0.66/0.59 (HR) and 0.66/0.61 (SOC), roughly 7-12 percentage points below GPT-4o. Gemma-1B at 0.40/0.32 is substantially lower.",
       "supported": "weak"
     },
     {
       "claim": "The system reduces prompt-injection risk, blocks out-of-scope requests, and limits toxic outputs.",
       "evidence": "Table 1 shows examples of blocking prompt injection and PII-containing requests. Table 3 shows overall classification accuracy but does not isolate prompt-injection-specific performance. No before/after comparison of attack surface reduction.",
       "supported": "weak"
     },
     {
       "claim": "Models with high span quality metrics can still have low classification performance, indicating extraction and classification are separable capabilities.",
       "evidence": "Table 4 shows Llama 405B and Claude 3.5 with high Token-F1 (1.000 and 0.992) and Substr (100%) on HR but low Micro-F1 (10.9% and 4.0%), demonstrating the extraction-classification gap.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Unreproducible evaluation",
       "detail": "The entire evaluation relies on proprietary internal enterprise artifacts that cannot be released. The authors explicitly state 'we cannot release the full corpora,' making independent verification impossible."
     },
     {
       "flag": "No variance despite averaging",
       "detail": "The paper states results are 'the average of multiple runs to ensure stability' but reports no standard deviation, number of runs, or any measure of variability. This makes it impossible to assess whether differences between models are meaningful."
     },
     {
       "flag": "Uneven prompt tuning acknowledged",
       "detail": "The limitations section admits 'prompting effort was greater for GPT-family models, potentially contributing to performance gaps,' introducing systematic bias favoring GPT models in the evaluation."
     },
     {
       "flag": "Small evaluation set for broad claims",
       "detail": "Only 100 gold inputs and 100 gold outputs per application are used, yet the paper claims 'scalable AI safety and AI security assurance for regulatable ML.' The gap between the evaluation scope and the claim scope is large."
     },
     {
       "flag": "Company employees evaluating own framework",
       "detail": "Both authors are from Pure Storage and evaluate their own proposed framework. Gold policies were created by 'security engineers' whose independence from the authors is not stated."
     }
   ],
   "cited_papers": [
     {
       "title": "Security of AI agents",
       "authors": ["Yifeng He", "Ethan Wang", "Yuyang Rong", "Zifei Cheng", "Hao Chen"],
       "year": 2024,
       "arxiv_id": "2406.08689",
       "relevance": "Directly addresses security risks of autonomous AI agents, the core motivation for the Policy-as-Prompt framework."
     },
     {
       "title": "Building guardrails for large language models",
       "authors": ["Yi Dong", "Ronghui Mu", "Gaojie Jin", "Yi Qi", "Jinwei Hu", "Xingyu Zhao", "Jie Meng", "Wenjie Ruan", "Xiaowei Huang"],
       "year": 2024,
       "arxiv_id": "2402.01822",
       "relevance": "Foundational work on LLM guardrails — the concept this paper operationalizes via policy-derived prompts."
     },
     {
       "title": "TrustAgent: Towards safe and trustworthy LLM-based agents",
       "authors": ["Wenyue Hua", "Xianjun Yang", "Mingyu Jin", "Zelong Li", "Wei Cheng", "Ruixiang Tang", "Yongfeng Zhang"],
       "year": 2024,
       "arxiv_id": "2402.01586",
       "relevance": "Proposes static safety principles for LLM agents, which the Policy-as-Prompt paper argues are insufficient for dynamic settings."
     },
     {
       "title": "Prompt injection attack against LLM-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li", "Kailong Wang"],
       "year": 2024,
       "arxiv_id": "2306.05499",
       "relevance": "Describes prompt injection attacks that the Policy-as-Prompt system aims to detect and block."
     },
     {
       "title": "Commercial LLM agents are already vulnerable to simple yet dangerous attacks",
       "authors": ["Ang Li", "Yin Zhou", "Vethavikashini Chithrra Raghuram", "Tom Goldstein", "Micah Goldblum"],
       "year": 2025,
       "arxiv_id": "2502.08586",
       "relevance": "Demonstrates real-world vulnerability of commercial LLM agents to attacks, motivating the need for guardrail frameworks."
     },
     {
       "title": "Contextual agent security: A policy for every purpose",
       "authors": ["Lillian Tsai", "Eugene Bagdasarian"],
       "year": 2025,
       "arxiv_id": "2501.17070",
       "relevance": "Argues for context-aware security for AI agents, which the Policy-as-Prompt framework claims to implement."
     },
     {
       "title": "LLM agents should employ security principles",
       "authors": ["Kaiyuan Zhang", "Zian Su", "Pin-Yu Chen", "Elisa Bertino", "Xiangyu Zhang", "Ninghui Li"],
       "year": 2025,
       "arxiv_id": "2505.24019",
       "relevance": "Proposes security principles (including least privilege) for LLM agents, directly related to the guardrail enforcement approach."
     },
     {
       "title": "Capture: Context-aware prompt injection testing and robustness enhancement",
       "authors": ["Gauri Kholkar", "Ratinder Ahuja"],
       "year": 2025,
       "arxiv_id": "2505.12368",
       "relevance": "Prior work by the same authors on prompt injection testing, forming the security testing foundation for this framework."
     },
     {
       "title": "ProPILE: Probing Privacy Leakage in Large Language Models",
       "authors": ["Siwon Kim", "Sangdoo Yun", "Hwaran Lee", "Martin Gubri", "Sungroh Yoon", "Seong Joon Oh"],
       "year": 2023,
       "arxiv_id": "2307.01881",
       "relevance": "Addresses privacy leakage in LLMs, relevant to the PII protection guardrails evaluated in the HR application."
     },
     {
       "title": "A survey of LLM-based agents in medicine: How far are we from Baymax?",
       "authors": ["Wenxuan Wang", "Zizhan Ma", "Zheng Wang"],
       "year": 2025,
       "arxiv_id": "2502.11211",
       "relevance": "Surveys LLM-based agents in regulated settings (medicine), contextualizing the need for governance guardrails."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The policy-to-guardrail pipeline addresses a real enterprise need, but 70-73% accuracy and unreleased code limit immediate adoption."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected approach of using LLMs to enforce policy compliance — no counterintuitive findings."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses AI safety/guardrails but does not demonstrate novel attacks or reveal new risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no critique of existing approaches or companies."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released. Prompts are shown in the appendix but cannot be independently run."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Pure Storage is a known enterprise storage company but not prominent in AI research."
     }
   }
 }