ai-research-survey

scan.json (26666B)

---

 {
   "paper": {
     "title": "Policy Compiler for Secure Agentic Systems",
     "authors": ["Nils Palumbo", "Sarthak Choudhary", "Jihye Choi", "Prasad Chalasani", "Somesh Jha"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.16708"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "PCAS enforces declarative Datalog-based authorization policies over a dependency graph of multi-agent interactions, providing deterministic policy compliance independent of model reasoning. On τ2-bench customer service tasks, PCAS improves policy compliance from 48% to 93% across Claude Opus 4.5, GPT-5.2, and Gemini 3 Pro. Prompt-embedded policies fail completely against prompt injection (100% attack success rate) while PCAS achieves 0%. Runtime overhead is modest (~20% latency increase, negligible cost increase).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Footnote 1 states 'Code for evaluations will be released soon.' No working URL or archive is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available benchmarks: τ2-bench and MALADE. The prompt injection scenario is fully described in the paper with the exact attack text provided in §5.2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or hardware details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions or scripts are provided. The code is not yet released."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as pass counts (e.g., 5/5, 0/5) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PCAS 'improves compliance from 48% to 93%' and reports per-model improvements without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context: '1.68–2.93× over a baseline' (§1), pass rates from specific baselines to instrumented values (e.g., Table 5), and absolute cost/latency numbers (Tables 4b, 6)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "5 trials per condition is used throughout with no justification for why 5 is sufficient. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as counts over 5 trials (e.g., 3/5) with no standard deviation or variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "All three case studies compare instrumented (PCAS) vs. non-instrumented (prompt-based policy) configurations, providing a clear controlled baseline comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use frontier models: Claude Opus 4.5, GPT-5.2, Gemini 3 Pro. The comparison table (Table 1) includes contemporary systems like Progent, NeMo Guardrails, Invariant Guardrails, ShieldAgent, FIDES."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The system has multiple components (dependency graph, reference monitor, policy engine) but no experiments isolate their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: task pass rate, policy compliance rate, attack success rate, runtime latency, and API cost (Tables 4-7)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant — the claims are about deterministic policy enforcement, which is verified automatically."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a learning-based system. There is no training or dev/test split to consider."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per task (6 tasks across 2 domains in τ2-bench, Table 5), per model, and per case study. Per-configuration breakdowns for prompt injection are provided in Table 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 RQ1 discusses GPT-5.2 failures in detail: wrong payment method, missing passenger, premature partial returns. §5.1 distinguishes policy violations from reasoning errors."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GPT-5.2 shows degraded performance on some instrumented tasks (booking drops from 4/5 to 2/5, Table 5), and the paper honestly reports this and attributes it to reasoning errors rather than enforcement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'compliance from 48% to 93%' and 'zero policy violations in instrumented runs,' both supported by Tables 5 and 7. The abstract claim of 'deterministic enforcement' is supported by the formal framework in §3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that PCAS improves compliance is justified by the controlled comparison: instrumented vs. non-instrumented agents on the same tasks with the same models, isolating the enforcement mechanism (§5 methodology)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Secure Agentic Systems' generally, but evaluation covers only 3 case studies with specific domains (customer service, prompt injection, pharmacovigilance). The paper does not bound its claims to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.1 explicitly distinguishes policy violations from reasoning errors, acknowledging that task failures may be due to model limitations rather than enforcement issues. §5.3 RQ1 analyzes whether failures are enforcement-induced regressions."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures what it claims: policy compliance (was the action blocked?) and task success (did the agent complete the task?). These are directly measured, not proxied. §5.1 explicitly distinguishes the two failure modes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'Claude Opus 4.5', 'GPT-5.2', 'Gemini 3 Pro', 'GPT-4.1-mini', 'GPT-4.1' — marketing names without snapshot dates or API version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Complete Datalog policy rules are provided in §5.2-5.4 and Appendix A. The prompt injection attack text is provided verbatim in §5.2. The anti-exfiltration prompt baseline is referenced but not fully provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is stated for all experiments: T=0.2 for prompt injection (§5.2) and MALADE (§5.4), T=0 for τ2-bench (§5.3)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The PCAS architecture is described in detail: dependency graph, reference monitor, policy engine, instrumentation layer, authorization flow (§4, Figures 1a-1b, Algorithm 1). MALADE's multi-agent architecture is described in §5.4."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 describes task selection criteria ('three representative tasks from each domain'), domain selection rationale (omitting telecom due to saturation), and evaluation configuration details."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§6.1 'Limitations' is a dedicated subsection discussing dependency tracking coverage, policy specification effort, and the gap between enforcement and reasoning."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§6.1 discusses specific threats: agents with code execution may use side channels bypassing instrumentation, and natural language to Datalog translation requires careful manual review. §4.2 discusses scope limitations around enforcement vs. reasoning."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§4.1 clearly defines the threat model and TCB. §4.2 'Current Scope' states what PCAS does not address: automatic policy synthesis from natural language, reasoning errors, and recovery failures."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (logs, traces, agent outputs) is made available. Only aggregate results are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation procedure is well-described: 5 trials per model per configuration, specific tasks from τ2-bench and MALADE, exact attack text for prompt injection (§5.2-5.4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (τ2-bench) and existing systems (MALADE)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from task selection through execution to measurement is documented. §5 describes the methodology, configurations, and how pass/fail is determined."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Wisconsin–Madison and Langroid."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. One author is from Langroid, a company that could benefit from the work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. One author (Prasad Chalasani) is affiliated with Langroid, which appears to be a company related to LLM agents, but no declaration of financial interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "PCAS is not evaluating pre-trained model capability on benchmarks. It evaluates a runtime enforcement mechanism. The models are used as components, not the subjects being benchmarked."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper tests a policy enforcement system, not model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination is not relevant when the evaluation measures policy compliance enforcement, not model capability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "API costs are reported per trial in Tables 4b, 6b, and 7b for all configurations, with both instrumented and non-instrumented costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Runtime per trial is reported in Tables 4b, 6a, 7b. Total trial counts are given (180 for τ2-bench, 30 for MALADE, 20 for prompt injection). Per-trial costs allow computing total budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "5 trials are run per configuration but no analysis of sensitivity to random seeds or variance across trials is provided. Results are reported as aggregate counts."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The number of runs is clearly stated: '5 independent trials per model per configuration' for τ2-bench (§5.3), 'five independent trials' for prompt injection (§5.2), '5 independent trials per configuration' for MALADE (§5.4)."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning is performed — the system uses fixed Datalog policies and fixed model temperatures. There is nothing to search over."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection from multiple options. The system and baselines each have one fixed configuration."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons across the many model×task×configuration comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own PCAS system against baselines they configured. No discussion of potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Tables 6a-6b and 7b explicitly compare runtime and cost between instrumented and non-instrumented configurations, showing the overhead of enforcement (~20% latency, negligible cost)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 discusses why τ2-bench is well-suited (policy-intensive domains, complex conversational dynamics). The paper justifies omitting the telecom domain (already saturated). Task selection is explained per domain."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The instrumented vs. non-instrumented comparison is the core experimental design — the scaffold (PCAS) IS the variable being tested, and the same models are used in both conditions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a policy enforcement system, not model knowledge. Whether models have seen τ2-bench tasks is irrelevant — the measure is policy compliance under enforcement, not model capability."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning — the evaluation measures enforcement effectiveness, not model capability on benchmark tasks."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no train/test split or learning involved in PCAS."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — PCAS is not a learned system and leakage is not a meaningful threat to the evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "PCAS improves policy compliance from 48% to 93% on τ2-bench customer service tasks across frontier models",
       "evidence": "Table 5 shows per-task pass rates: non-instrumented total 43/90 (48%), instrumented 84/90 (93%). Breakdown by model and task provided.",
       "supported": "strong"
     },
     {
       "claim": "PCAS achieves zero policy violations in instrumented runs across all case studies",
       "evidence": "§5.1-5.4: all failures in instrumented runs are attributed to reasoning errors (wrong payment method, missing passenger), not policy violations. cancel_reservation blocked 25 times, book_reservation blocked 5 times with no false positives (§5.3).",
       "supported": "strong"
     },
     {
       "claim": "Prompt-embedded policies fail to prevent prompt injection attacks (100% ASR vs 0% with PCAS)",
       "evidence": "Table 4a: non-instrumented baseline has 5/5 attack success rate; all three instrumented configurations achieve 0/5 ASR.",
       "supported": "strong"
     },
     {
       "claim": "Runtime overhead of enforcement is modest (~20% latency, negligible cost increase)",
       "evidence": "Tables 6a-6b show ~20% mean latency increase for τ2-bench. Token costs actually decrease 3.3% on average because the natural language policy is removed from the system prompt. Table 7b shows MALADE overhead ($0.090 vs $0.071).",
       "supported": "strong"
     },
     {
       "claim": "Linear message histories are insufficient for policy enforcement in multi-agent systems; dependency graphs are required",
       "evidence": "§3 provides formal arguments. §5.4 demonstrates with DependsSameAgent predicate that per-session authorization requires causal context across agent boundaries. Conceptually motivated but the necessity is argued, not empirically tested against a linear-history alternative.",
       "supported": "moderate"
     },
     {
       "claim": "PCAS improves MALADE prediction accuracy from 14/15 to 15/15 while eliminating all 42 policy violations",
       "evidence": "Table 7a shows accuracy and compliance results. The 42 unauthorized FDA API accesses in non-instrumented runs are all blocked in instrumented runs.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small sample sizes",
       "detail": "All evaluations use only 5 trials per condition. With such small N, the reported pass rates have wide confidence intervals (e.g., 3/5 has a 95% CI of roughly 12-77%). No statistical tests or power analysis justify this sample size."
     },
     {
       "flag": "Cherry-picked task selection",
       "detail": "Only 3 tasks per domain (6 total) were selected from τ2-bench, chosen because they 'elicit a specific category of policy violation.' The telecom domain was omitted because performance 'is already saturated.' This selection may overstate PCAS's impact by focusing on tasks where enforcement is most helpful."
     },
     {
       "flag": "No ablation of system components",
       "detail": "PCAS has multiple components (dependency graph, reference monitor, policy engine with Datalog) but no ablation tests whether the full dependency graph is needed vs simpler approaches like linear trace checking."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "Authors evaluate their own system and wrote both the Datalog policies and the evaluation. No independent evaluation or third-party policy authoring is included."
     },
     {
       "flag": "Langroid affiliation undisclosed as conflict",
       "detail": "One author is affiliated with Langroid, which appears to be an LLM agent company. This potential conflict of interest is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "τ2-bench: Evaluating conversational agents in a dual-control environment",
       "authors": ["Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan"],
       "year": 2025,
       "arxiv_id": "2506.07982",
       "relevance": "Benchmark used for evaluating customer service agent policy compliance."
     },
     {
       "title": "AgentDojo: A dynamic environment to evaluate prompt injection attacks and defenses for LLM agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang", "Mislav Balunovic"],
       "year": 2024,
       "relevance": "Dynamic benchmark for evaluating prompt injection attacks and defenses in LLM agent settings."
     },
     {
       "title": "Progent: Programmable privilege control for LLM agents",
       "authors": ["Tianneng Shi", "Jingxuan He", "Zhun Wang"],
       "year": 2025,
       "arxiv_id": "2504.11703",
       "relevance": "Domain-specific language for LLM agent privilege control; key baseline for runtime enforcement comparison."
     },
     {
       "title": "Defeating prompt injections by design",
       "authors": ["Edoardo Debenedetti", "Ilia Shumailov"],
       "year": 2025,
       "arxiv_id": "2503.18813",
       "relevance": "CaMeL architecture for separating data from instructions to defend against prompt injection."
     },
     {
       "title": "Securing AI agents with information-flow control",
       "authors": ["Manuel Costa", "Boris Köpf", "Aashish Kolluri"],
       "year": 2025,
       "arxiv_id": "2505.23643",
       "relevance": "FIDES system applying information-flow control for prompt injection defense; closest related work to PCAS."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection attacks against LLM agents."
     },
     {
       "title": "ShieldAgent: Shielding agents via verifiable safety policy reasoning",
       "authors": ["Zhaorun Chen", "Mintong Kang", "Bo Li"],
       "year": 2025,
       "relevance": "Agent safety via logical rules and Markov logic networks; baseline comparison for policy enforcement."
     },
     {
       "title": "AgentSpec: Customizable runtime enforcement for safe and reliable LLM agents",
       "authors": ["Haoyu Wang", "Christopher M Poskitt", "Jun Sun"],
       "year": 2025,
       "arxiv_id": "2503.18666",
       "relevance": "Lightweight DSL for runtime enforcement of LLM agent safety constraints."
     },
     {
       "title": "How not to detect prompt injections with an LLM",
       "authors": ["Sarthak Choudhary", "Divyam Anshumaan", "Nils Palumbo", "Somesh Jha"],
       "year": 2025,
       "relevance": "Evaluation showing LLM-based prompt injection detectors can be bypassed by adaptive attacks."
     },
     {
       "title": "The attacker moves second: Stronger adaptive attacks bypass defenses against LLM jailbreaks and prompt injections",
       "authors": ["Milad Nasr", "Nicholas Carlini"],
       "year": 2025,
       "arxiv_id": "2510.09023",
       "relevance": "Demonstrates that 12 published defenses against prompt injection can all be bypassed with >90% success."
     },
     {
       "title": "MALADE: Orchestration of LLM-powered agents with retrieval augmented generation for pharmacovigilance",
       "authors": ["Jihye Choi", "Nils Palumbo", "Prasad Chalasani"],
       "year": 2024,
       "relevance": "Multi-agent pharmacovigilance system used as case study for PCAS evaluation."
     },
     {
       "title": "Systems security foundations for agentic computing",
       "authors": ["Mihai Christodorescu", "Earlence Fernandes", "Somesh Jha"],
       "year": 2025,
       "arxiv_id": "2512.01295",
       "relevance": "Foundational framework for security in agentic AI systems."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analysis of failure modes in multi-agent LLM systems, motivating the need for policy enforcement."
     },
     {
       "title": "AI agents under threat: A survey of key security challenges and future pathways",
       "authors": ["Zehang Deng", "Yongjian Guo"],
       "year": 2025,
       "relevance": "Survey of security challenges in AI agent systems including unauthorized actions and data exfiltration."
     }
   ]
 }