ai-research-survey

scan.json (24932B)

---

 {
   "paper": {
     "title": "Combining Cost-Constrained Runtime Monitors for AI Safety",
     "authors": ["Tim Tian Hua", "James Baskerville", "Henri Lemoine", "Mia Hopman", "Aryan Bhatt", "Tyler Tracy"],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2507.15886",
     "doi": "10.48550/arXiv.2507.15886"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: github.com/tim-hua-01/combining_monitors_public (Section 1, end of introduction). An Alignment Forum practitioner's guide is also linked."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available BigCodeBench dataset (v.0.1.3, Apache-2.0 license) and references the public code repository which would contain the generated backdoored code and monitor scores."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix H describes the hardware (AMD EPYC 7413, 754 GB RAM) and mentions scikit-learn for KDE, but no requirements.txt, Dockerfile, or detailed dependency/version listing is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions a GitHub repository and names the experiment script (neurips_experiments.py in Appendix H), but does not provide step-by-step reproduction instructions in the paper itself. A reader would need to find the repository and figure out the setup."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports quantile-based 90% confidence intervals on test set results (Section 4.1: 'We report means and quantile-based 90% CIs on the test set'). Appendix A shows standard errors on relative performance comparisons."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Figure 6) shows that the difference between ΠSR and baselines is 'statistically significant at the 10% level' for budgets between $0.50 and $1.75, using quantile-based bootstrap standard errors."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context: 'more than double our recall rate compared to a naive baseline' (abstract), and specific recall values at given budgets are shown in figures with both absolute and relative comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,140 aligned outputs and 318 misaligned outputs with 500 random train/test splits, but does not justify why these sample sizes are adequate or discuss statistical power. The paper acknowledges sensitivity to tail distributions (footnote 7) but does not perform a power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are averaged over 500 random train/test splits of misaligned scores (Section 4.1), and 90% quantile-based confidence intervals are reported. Appendix A reports bootstrap standard errors for relative performance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Πnaive (from Greenblatt et al., 2024), ΠNP, ΠNPS single-monitor baselines, and a two-monitor baseline ΠBM that always calls both monitors (Section 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are contemporary: Πnaive from Greenblatt et al. (2024) is the standard AI control monitoring approach, and ΠBM represents a natural two-monitor extension. These are appropriate for this setting."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically builds up from Πnaive → ΠNP (adding likelihood ratios) → ΠNPS (adding stochastic monitoring) → ΠSR (adding two monitors), effectively ablating each contribution. Section 5.2 explicitly examines the trade-off between monitoring and auditing."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates safety (recall on misaligned outputs) across multiple budget levels and audit cost ratios (Figures 1, 2b, 3a, 3b, 5), and also reports cost/FPR trade-offs. Pareto curves are used to show the full cost-safety trade-off."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a mathematical framework paper with automated experiments on code backdoor detection. Human evaluation of the system's outputs is not relevant to the claims about optimal monitor combination."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly states: 'We assess policy safety by measuring recall on the test split of misaligned responses.' The 318 misaligned samples are split into 218 train and 100 test across 500 random splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Performance is broken down by budget level, audit cost ratio (ca/c1), and policy class. Figures show Pareto curves across multiple budget configurations. Appendix G shows out-of-distribution performance on a different attack set."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations) discusses when the approach can fail: distribution shift between training and actual attacks, the 'winner's curse' effect. Appendix G shows degraded absolute performance on out-of-distribution backdoors from Wen et al. (2025)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that stochastic monitoring gains diminish as audit-to-monitor cost ratio increases (Section 5.2, Figure 3b). Appendix G shows much lower absolute safety on OOD attacks, though relative gains are maintained."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) more than doubling recall compared to naive baseline, (2) combining monitors yields Pareto dominance, and (3) a principled methodology — all are supported by the experimental results in Sections 5 and 6 and corresponding figures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablations (removing/adding components like likelihood ratios, stochastic monitoring, second monitor). These are controlled single-variable manipulations within a mathematical framework, which is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly states: 'our empirical results should be primarily viewed as demonstrating how one could apply our framework, rather than a definitive measure of absolute monitor performance.' Section 7 discusses limitations regarding distribution shift."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses that the optimality assumes training distribution matches real attacks, and that distributional shifts could undermine the approach. Appendix G tests an alternative attack distribution. The paper also acknowledges the brute-force search is intractable for many monitors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'instruction-tuned versions of Llama 3.1 70B and 8B' (Section 4) as monitors, and 'o1-mini' for generating code. However, o1-mini lacks a specific version/snapshot date. Llama 3.1 is a specific release (Grattafiori et al., 2024). On balance, the monitor models are specified but the code-generating model (o1-mini) lacks a version date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides the complete system prompt and monitor prompt template with placeholders for question and code. The fill values (BigCodeBench questions and o1-mini solutions) are from the public dataset and reproducible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports: 5 samples per monitor with averaged scores, Gaussian noise addition, Gaussian kernel KDE with Silverman's rule, 218/100 train/test split, 500 random splits, 200 quantiles for grid search. Monitor costs (1.0 and 0.21) and audit cost (20x) are specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The monitors are simple prompted LLMs that output a suspicion score. The framework is a mathematical optimization over monitor outputs, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents: o1-mini generates solutions to all 1,140 BigCodeBench questions; backdoored solutions are verified to pass test cases but fail on specific inputs (yielding 318 misaligned samples); monitor scores are averaged over 5 samples with minimal Gaussian noise added; KDE with Gaussian kernel and Silverman's rule is used."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 begins with a 'Limitations' subsection that discusses distribution shift, overfitting to training attacks, and scalability limitations of the brute-force search approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) the optimality assumes the real attack distribution matches the training distribution, (2) ΠNP could underperform if the true attack distribution is easier, (3) brute-force search is intractable for more than 4 monitors. Appendix G empirically tests OOD robustness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly bounds scope: 'our empirical results should be primarily viewed as demonstrating how one could apply our framework, rather than a definitive measure of absolute monitor performance.' Section 7 notes the approach only works for few monitors and that distribution shift is a real concern."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "BigCodeBench is publicly available (Apache-2.0 license). The code repository (github.com/tim-hua-01/combining_monitors_public) presumably includes the generated monitor scores and backdoored code, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes: o1-mini prompted to solve all 1,140 BigCodeBench coding questions for aligned data; o1-mini separately prompted to create backdoored solutions verified against criteria, yielding 318 misaligned responses; monitors called 5 times per sample with scores averaged."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data consists of AI-generated code and monitor scores on the BigCodeBench benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: BigCodeBench questions → o1-mini generates aligned solutions (1,140) and backdoored solutions (verified, 318) → monitors score each solution 5 times → scores averaged with Gaussian noise → KDE for likelihood ratios → policy optimization. Each stage is described with counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was supported through the MARS (Mentorship for Alignment Research Students) program at the Cambridge AI Safety Hub (caish.org/mars).' Author affiliations include MARS and Redwood Research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: MARS for Hua, Baskerville, Lemoine, Hopman; Redwood Research for Bhatt; MARS & Redwood Research for Tracy. These are AI safety research organizations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "MARS/Cambridge AI Safety Hub is an academic mentorship program. Redwood Research is an AI safety organization. Neither has a commercial product being evaluated. The monitors used are open-source Llama models. The funder has no financial stake in a particular outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While the affiliations are disclosed, there is no explicit declaration of competing interests or absence thereof."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 3.1 70B and 8B as monitors evaluating code for backdoors. The training cutoff of these models is not stated. While contamination is less critical here (the monitors are evaluating code patterns, not benchmark knowledge), the cutoff is still relevant since BigCodeBench problems could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether BigCodeBench problems or similar code patterns appeared in Llama 3.1's training data. The monitors' ability to detect backdoors could be influenced by exposure to similar code during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "BigCodeBench was published (2025) potentially after Llama 3.1's training cutoff, but this is not discussed. The paper does not address whether the benchmark or similar problems could have been in the monitors' training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study is purely computational, using AI-generated code and automated monitors."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative costs: Llama 3.1 70B costs 1 unit per sample, Llama 3.1 8B costs 0.21 units (Section 4). Audit cost is 20x the 70B monitor cost. The entire framework is built around cost optimization, with explicit cost budgets in all experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H provides detailed compute specifications: AMD EPYC 7413, total runtime of 500.2 seconds (~8.3 minutes) for all experiments, 87-90 CPU workers in parallel. Per-experiment task rates are also provided."
       }
     }
   },
   "claims": [
     {
       "claim": "Likelihood-ratio-based policies (ΠNP) outperform naive threshold-based policies (Πnaive) due to the 'winner's curse' effect in monitor scores.",
       "evidence": "Figure 2b shows ΠNP achieves higher safety (recall) than Πnaive at low audit FPR. Figure 2a shows the log likelihood ratio dips between scores 7.2-8.5 for Llama 3.1 70B (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Stochastic monitor calling (ΠNPS) more than doubles recall compared to deterministic monitoring when audit cost is 20x monitor cost and budget is $1.20.",
       "evidence": "Section 5.2 states the optimal ΠNPS policy calls the monitor 71% of the time, audits top 3.4% by LLR, achieving 2.4% FPR vs. 1% for deterministic case, more than doubling safety.",
       "supported": "strong"
     },
     {
       "claim": "Combining two monitors (Llama 3.1 70B and 8B) via ΠSR yields Pareto improvements over any single-monitor policy.",
       "evidence": "Figure 5 shows ΠSR Pareto dominates ΠNPS (70B only), ΠNPS (8B only), and ΠBM (both always-on) at budgets between $0.50 and $1.75. Figure 6 in Appendix A confirms statistical significance at 10% level.",
       "supported": "strong"
     },
     {
       "claim": "The framework is robust to out-of-distribution attacks: ΠSR fitted on training data still outperforms baselines on OOD backdoors.",
       "evidence": "Appendix G tests on 300 backdoors from Wen et al. (2025). Figure 10 shows ΠSR outperforms baselines across most budgets, though absolute safety is much lower.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper develops a mathematical framework for optimally combining multiple runtime safety monitors under a unified cost budget, using the Neyman-Pearson lemma. In a backdoor detection experiment on BigCodeBench using Llama 3.1 70B and 8B as monitors, likelihood-ratio-based policies outperform naive threshold policies, stochastic monitor calling can more than double recall, and combining two monitors yields Pareto improvements over single-monitor approaches. The framework degrades gracefully under distribution shift, maintaining relative advantages over baselines on out-of-distribution attacks.",
   "red_flags": [
     {
       "flag": "Single empirical setting",
       "detail": "All experiments use the same BigCodeBench backdoor detection task with the same two Llama monitors. While the theoretical framework is general, empirical validation is limited to one narrow setting, making it unclear how well results transfer to other monitoring scenarios (e.g., jailbreak detection, content moderation)."
     },
     {
       "flag": "Small misaligned sample size",
       "detail": "Only 318 misaligned (backdoored) samples are available, split into 218 train and 100 test. The paper acknowledges sensitivity to tail distributions (footnote 7) but does not perform a power analysis. With 100 test samples, recall estimates have relatively high variance."
     },
     {
       "flag": "o1-mini version unspecified",
       "detail": "The paper uses o1-mini to generate both aligned and backdoored code but does not specify which version/snapshot of o1-mini was used, affecting reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "AI control: Improving safety despite intentional subversion",
       "authors": ["Ryan Greenblatt", "Buck Shlegeris", "Kshitij Sachan", "Fabien Roger"],
       "year": 2024,
       "relevance": "Foundational AI control paper that proposes the monitoring framework this paper builds upon, directly relevant to AI safety evaluation methodology."
     },
     {
       "title": "Sabotage evaluations for frontier models",
       "authors": ["Joe Benton", "Misha Wagner", "Eric Christiansen"],
       "year": 2024,
       "arxiv_id": "2410.21514",
       "relevance": "Evaluates sabotage capabilities of frontier AI models, directly relevant to the threat model this paper's monitoring framework addresses."
     },
     {
       "title": "Constitutional classifiers: Defending against universal jailbreaks across thousands of hours of red teaming",
       "authors": ["Mrinank Sharma", "Meg Tong", "Jesse Mu"],
       "year": 2025,
       "arxiv_id": "2501.18837",
       "relevance": "Presents safety classifiers for defending against jailbreaks, a key application area for runtime monitoring combinations."
     },
     {
       "title": "Cost-effective constitutional classifiers via representation re-use",
       "authors": ["Hoagy Cunningham", "Alwin Peng", "Jerry Wei"],
       "year": 2025,
       "relevance": "Proposes multi-stage classifiers using cheap model-internals monitors followed by larger monitors, directly related to the multi-monitor combination problem."
     },
     {
       "title": "Monitoring reasoning models for misbehavior and the risks of promoting obfuscation",
       "authors": ["Bowen Baker", "Joost Huizinga", "Leo Gao"],
       "year": 2025,
       "arxiv_id": "2503.11926",
       "relevance": "Studies chain-of-thought monitoring for detecting misbehavior in reasoning models, relevant to runtime safety monitoring."
     },
     {
       "title": "Reasoning models don't always say what they think",
       "authors": ["Yanda Chen", "Joe Benton", "Ansh Radhakrishnan"],
       "year": 2025,
       "arxiv_id": "2505.05410",
       "relevance": "Demonstrates that reasoning models' chain-of-thought can be unfaithful, motivating the need for multiple monitoring approaches."
     },
     {
       "title": "Adaptive deployment of untrusted LLMs reduces distributed threats",
       "authors": ["Jiaxin Wen", "Vivek Hebbar", "Caleb Larson"],
       "year": 2025,
       "relevance": "Studies adaptive deployment strategies for untrusted LLMs with safety considerations, relevant to the AI control framework."
     },
     {
       "title": "Control tax: The price of keeping AI in check",
       "authors": ["Mikhail Terekhov", "Zhen Ning David Liu", "Caglar Gulcehre"],
       "year": 2025,
       "relevance": "Explores the cost of using different language models as monitors in AI control settings, directly relevant to cost-constrained monitoring."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2025,
       "relevance": "The benchmark used in this paper's experiments for code generation evaluation."
     },
     {
       "title": "A sketch of an AI control safety case",
       "authors": ["Tomek Korbak", "Joshua Clymer", "Benjamin Hilton", "Buck Shlegeris", "Geoffrey Irving"],
       "year": 2025,
       "arxiv_id": "2501.17315",
       "relevance": "Proposes structure for AI control safety cases where monitoring is a cornerstone component."
     },
     {
       "title": "How to evaluate control measures for LLM agents? A trajectory from today to superintelligence",
       "authors": ["Tomek Korbak", "Mikita Balesni", "Buck Shlegeris", "Geoffrey Irving"],
       "year": 2025,
       "arxiv_id": "2504.05259",
       "relevance": "Discusses evaluation methodology for AI control measures, relevant to the broader safety evaluation landscape."
     },
     {
       "title": "Detecting strategic deception using linear probes",
       "authors": ["Nicholas Goldowsky-Dill", "Bilal Chughtai", "Stefan Heimersheim", "Marius Hobbhahn"],
       "year": 2025,
       "arxiv_id": "2502.03407",
       "relevance": "Proposes activation probing for detecting deception, one type of monitor that could be combined using this paper's framework."
     }
   ]
 }