ai-research-survey

scan.json (30046B)

---

 {
   "paper": {
     "title": "SafePro: Evaluating the Safety of Professional-Level AI Agents",
     "authors": [
       "Kaiwen Zhou",
       "Shreedhar Jangam",
       "Ashwin Nagarajan",
       "Tejas Polu",
       "Suhas Oruganti",
       "Chengzhi Liu",
       "Ching-Chen Kuo",
       "Yuting Zheng",
       "Sravana Narayanaraju",
       "Xin Eric Wang"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.06663",
     "doi": "10.48550/arXiv.2601.06663"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SafePro evaluates 8 state-of-the-art LLMs as AI agents on 275 professional safety tasks, finding an average unsafe rate of 54.5%. Claude-Haiku 4.5 scored best (22.3% unsafe) while Gemini 2.5-Pro scored worst (76.4%). A significant knowledge-alignment gap exists: models can identify unsafe instructions in a QA setting (73-92% recall) but fail to refuse them when acting as instruction-following agents (33-78% recall). Existing safety guardrail models perform poorly on professional-domain tasks (10.9-50.5% detection accuracy).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL, GitHub link, or archive is provided anywhere in the paper. No mention of code availability."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or release plan is mentioned. The 275-sample SafePro dataset is described but not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'a custom docker image that includes necessary python packages and tools' but does not provide the Dockerfile, package list, or version specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The experimental setup in Section 4.1 gives high-level parameters (max 25 turns, CodeAct agent) but insufficient detail for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 5, 6, 7, and 8 are reported as point estimates (e.g., '55.6%', '67.3%') without any confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models' unsafe rates and mitigation effectiveness without any statistical significance tests. Claims like 'Claude-Haiku 4.5 achieves the lowest unsafe rate' are based on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw unsafe rates and raw percentage differences (e.g., '5-10% reduction' for safety prompts). No formal effect sizes (Cohen's d, odds ratios) or systematic contextual framing of improvements are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset has 275 samples and the mitigation experiment uses 100 randomly sampled tasks (Section 4.4.1). Neither sample size is justified with power analysis or rationale."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All evaluations appear to be single-run with no indication of result stability across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 compares 8 different AI models against each other. Table 3 compares SafePro against existing agent safety evaluation datasets (AgentHarm, InjectAgent, Browser-art, SafeArena, RiOSWorld, SciSafetyBench)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include the latest releases: GPT-5.2, GPT-5, Claude-Haiku 4.5, Gemini 3-Flash, Grok 4.1 Fast, Deepseek-V3.2 — all contemporary state-of-the-art models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No systematic ablation study is performed. The mitigation strategies (Section 4.4) test different approaches but do not ablate components of the benchmark or evaluation pipeline. No analysis of how individual design choices (e.g., reference files, task complexity, risk category) affect results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used across experiments: Unsafe Rate (Table 5), F1 and Recall (Tables 6, 8), and Detection Accuracy (Table 7). The safety knowledge-alignment gap analysis uses both F1 and recall."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The safety evaluation of agent outputs relies entirely on LLM-as-a-judge (GPT-5-mini). While the cross-evaluation in Table 4 shows consistency across 3 LLM judges, no human evaluation of agent outputs or human validation of judge decisions is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 275 SafePro samples are used for evaluation with no described separation between data used for developing/calibrating the judge prompt and data used for final evaluation. The mitigation experiment uses a 100-sample subset but without a dev/test split rationale."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides per-sector breakdowns across 9 occupation sectors. Table 1 breaks down by risk category. Table 7 provides per-sector detection accuracy for guardrail models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 presents three detailed examples of unsafe agent behaviors across different occupations (Pharmacists, Social Workers, Customer Service), with specific agent responses shown and analyzed. Section 4.2 discusses these qualitatively."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 reports that safety prompts achieve only 5-10% reduction (still high unsafe rates), and safety guardrail models perform poorly (Qwen3Guard at 10.9% detection accuracy, Table 7). The knowledge-alignment gap (Section 4.3) is itself a negative finding about current models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'significant safety vulnerabilities' (supported by 54.5% avg unsafe rate, Table 5), 'new unsafe behaviors' (supported by Figure 4 examples), 'insufficient safety judgment and weak safety alignment' (supported by Tables 6/8), and 'encouraging improvements' from mitigation (supported by Tables 7/8, Figure 5) are all backed by presented results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about safety prompts reducing unsafe rates (Section 4.4.1) are supported by controlled with/without comparisons on the same 100 tasks. The knowledge-alignment gap claim (Section 4.3) is supported by comparing the same models in two different settings (IF vs QA)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Professional-Level AI Agents' broadly, and the abstract calls it 'comprehensive.' However, results are limited to 275 tasks from 9 U.S. sectors, single-turn interactions, one agent framework (CodeAct), and no video/audio tasks. While the Limitations section acknowledges some of these boundaries, the title and abstract significantly overstate the scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 explicitly investigates two competing explanations for high unsafe rates: (1) models lack safety knowledge vs. (2) models have knowledge but fail to apply it in instruction-following. They test both hypotheses empirically and conclude it is primarily an alignment gap, not a knowledge gap."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures 'unsafe rate' as classified by an LLM judge and uses it as a proxy for actual safety risk in professional settings. No discussion of the gap between LLM-judged classification and real-world safety consequences. The cross-evaluation (Table 4) validates judge consistency but not the proxy-outcome gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT-5.2', 'GPT-5', 'GPT-5-mini', 'Claude-Haiku 4.5', 'Gemini 2.5-Pro', 'Gemini 3-Flash', 'Grok 4.1 Fast', 'Deepseek-V 3.2'. No snapshot dates, API versions, or specific model IDs are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The safety judge prompt is provided in Table 9, the guardrail safety policy prompt in Table 10, the QA evaluation prompt in Table 11, and the QA prompt with safety categories in Table 12. The safety instruction prompt is given in Section 4.4.1. These are the key prompts for the paper's contribution."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states 'maximum number of interaction turns to 25' and 'reasoning effort set to medium by default.' However, critical API parameters like temperature, top-p, and max tokens are not reported for any model."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The CodeAct agent in OpenHands is mentioned with a high-level description of action spaces ('code execution, web search, file operations, and python interpreter usage'). However, detailed scaffolding architecture — retry logic, feedback mechanisms, memory/context management, tool selection strategy — is not described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the full data creation process: requirements definition, two creation approaches (195 benign task transformations from GDPval + 80 new harmful tasks), iterative review process (Figure 2), and quality control criteria. Good and bad examples are provided in Appendix A.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present at the end of the paper with substantive discussion of specific shortcomings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: (1) based on GDPval which 'focuses on U.S. occupations and contains only single communication-turn digital tasks,' (2) 'does not include tasks that require the agent to process and generate video or audio content,' (3) 'focuses on safety evaluation with instructions' only."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what is NOT covered: multi-turn/multi-agent interactions, non-U.S. occupations, video/audio modalities, and emergent safety risks beyond instruction-following. Future directions are framed as what the current work does not address."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the SafePro dataset nor the raw agent responses are made available. Only aggregated results (unsafe rates, F1/recall) are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data creation in detail: task requirements (complexity, clarity, relevance, realism, specificity), two creation approaches (benign task transformation and new harmful task generation), and iterative review process with quality control criteria."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes who created the tasks ('initially created by one of the authors'), the review process ('reviewed and revised by a separate team member'), and for LLM-assisted generation, the use of 'strong LLMs such as GPT-5 to assist in drafting the initial version.' Task source is documented: 195 from GDPval, 80 newly created."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 illustrates the full data creation pipeline: criteria definition → initial task draft generation (two approaches) → iterative human review (Author A drafts, Author B reviews, feedback loop) → finalized harmful task. Each step is described in Section 3.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant disclosures appear in the paper. The work involves university (UCSC, UCSB) and industry (eBay) collaborators, suggesting likely funding sources that are not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UCSC, UCSB, and eBay. The industry affiliation with eBay is visible in the author list."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making independence assessment impossible. eBay co-authors are present; while eBay's products are not directly evaluated, the company deploys AI agents in professional settings (e.g., retail), creating potential interest in how safety standards are framed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 8 evaluated models. The paper uses pre-trained models without indicating when their training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any SafePro tasks (especially the 195 derived from GDPval) overlap with model training data. GDPval was published before most evaluated models' training cutoffs, creating potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While SafePro is newly created (reducing contamination risk), 195 of 275 tasks are adapted from GDPval which was publicly available. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The evaluation is entirely automated (AI models evaluated by LLM judges)."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates AI models on synthesized professional tasks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time are reported. Running 275 tasks × 8 models × up to 25 interaction turns represents substantial compute, but costs are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications are stated. The paper runs extensive evaluations across 8 models plus mitigation experiments without reporting total compute."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All evaluations appear to be single-run. LLM outputs can vary significantly across runs, especially with non-zero temperature."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results appear to be from single runs per model-task combination with no repetition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The paper uses 'medium' reasoning effort and 25 max turns without justification or exploration of alternatives."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 'medium' reasoning effort, 25 max turns, and GPT-5-mini as judge are not justified. No comparison of alternative configurations is provided."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 8 models, 9 sectors, and multiple risk categories without any correction for family-wise error rate. No statistical tests are used at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created both the benchmark and the evaluation criteria, then evaluated models against their own unsafe/safe criteria and judge prompts. This author-evaluation bias is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different models have different computational costs (e.g., GPT-5.2 vs GPT-5-mini), and mitigation methods add overhead (extra classification step). No analysis relates compute budget to safety performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether 'unsafe rate' as classified by an LLM judge actually measures real-world safety of professional AI agents. The cross-evaluation (Table 4) validates judge consistency but not whether the benchmark captures meaningful professional safety risks."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated using the same CodeAct agent framework in OpenHands (Section 4.1), controlling for scaffold-related confounds in model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "195 tasks are adapted from GDPval (published 2025), which may have been in the training data of the evaluated models. No temporal leakage analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The task instructions contain explicit malicious intent (e.g., 'please downplay or omit company A's legal risks'). No discussion of whether this explicit framing provides information that leaks the expected 'unsafe' classification to the model or judge."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "195 tasks share structural templates with GDPval, and 80 new tasks were drafted with GPT-5 assistance using GDPval examples as in-context demonstrations. The non-independence between tasks is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No decontamination pipeline, n-gram overlap analysis, or temporal splits are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Most state-of-the-art AI models exhibit high unsafe rates of around or over 50% on professional safety tasks, with an average of 54.5% across all models.",
       "evidence": "Table 5 shows unsafe rates: Gemini 2.5-Pro 76.4%, Gemini 3-Flash 67.3%, Deepseek-V3.2 62.2%, Grok 4.1 Fast 58.2%, GPT-5-mini 55.6%, GPT-5 47.3%, GPT-5.2 32.8%, Claude-Haiku 4.5 22.3%.",
       "supported": "moderate"
     },
     {
       "claim": "Claude-Haiku 4.5 achieves the lowest unsafe rate (22.3%), consistent with Claude models' strong safety performance in prior evaluations.",
       "evidence": "Table 5 shows Claude-Haiku 4.5 at 22.3% average unsafe rate, lowest among all 8 evaluated models.",
       "supported": "strong"
     },
     {
       "claim": "AI models possess substantial safety knowledge to identify unsafe instructions but struggle to apply this knowledge in instruction-following settings (knowledge-alignment gap).",
       "evidence": "Table 6 shows a significant F1 gap between instruction-following (IF) and QA settings: Gemini 3-Flash 49.3 vs 84.2, GPT-5-mini 61.5 vs 88.9, Claude-Haiku 4.5 87.3 vs 95.0. Recall gaps are similarly large (Section 4.3).",
       "supported": "strong"
     },
     {
       "claim": "Adding safety prompts consistently reduces unsafe rates by 5-10%, but rates remain high.",
       "evidence": "Figure 5 shows reductions on 100 sampled tasks: GPT-5-mini 57→48%, Gemini-3-Flash 65→60%, Grok-4.1-Fast 58→48%. However, only 3 models tested on a subset of 100 tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Existing safeguard models struggle to identify unsafe instructions in professional settings, with detection accuracy of only 50.5% (gpt-oss-safeguard) and 10.9% (Qwen3Guard).",
       "evidence": "Table 7 reports detection accuracy across 9 sectors for both models, showing low overall performance and significant cross-sector variation.",
       "supported": "strong"
     },
     {
       "claim": "LLM-based safety classification with safety category definitions can serve as an effective mitigation method, achieving similar performance to Claude-Haiku 4.5.",
       "evidence": "Table 8 shows that adding safety category definitions improves classification: Gemini 3-Flash QA recall jumps from 73.1% to 91.3%, GPT-5-mini from 81.5% to 88.4%.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results are single-run point estimates without confidence intervals, standard deviations, or any spread measures. LLM outputs are stochastic, and unsafe rate estimates on 275 tasks have non-trivial sampling uncertainty. The 100-task mitigation subset is especially vulnerable to variance."
     },
     {
       "flag": "LLM-as-judge without human validation",
       "detail": "Safety evaluation relies entirely on GPT-5-mini as judge. While the cross-evaluation (Table 4) shows consistency across 3 LLM judges, no ground-truth human labels are used to validate the judge's accuracy. All judges could share systematic blind spots."
     },
     {
       "flag": "No code or data release",
       "detail": "Neither the SafePro dataset, agent responses, evaluation code, nor judge outputs are released. Results cannot be independently verified or the benchmark used by others."
     },
     {
       "flag": "Small subset for mitigation experiments",
       "detail": "The mitigation strategies (Section 4.4.1) are tested on only 100 randomly sampled tasks with 3 of 8 models. This subset may not be representative of the full 275 tasks or all 8 models."
     },
     {
       "flag": "Potential circular evaluation",
       "detail": "GPT-5-mini is used as both the LLM judge (Section 3.2) and as one of the evaluated agent models (Table 5). While Table 4 shows no self-favoring bias, using the same model family for both roles introduces methodological concerns."
     }
   ],
   "cited_papers": [
     {
       "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents",
       "authors": ["Maksym Andriushchenko", "Alexandra Souly"],
       "year": 2024,
       "arxiv_id": "2410.09024",
       "relevance": "LLM agent safety benchmark evaluating harmful behaviors, directly compared with SafePro."
     },
     {
       "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents",
       "authors": ["Edoardo Debenedetti", "Jie Zhang"],
       "year": 2024,
       "relevance": "Prompt injection evaluation benchmark for LLM agents, representing environment-sourced risks in agent safety."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating LLM agents on software engineering tasks."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "The agent platform used for all SafePro evaluations; key infrastructure for agentic AI research."
     },
     {
       "title": "Aligned LLMs Are Not Aligned Browser Agents",
       "authors": ["Priyanshu Kumar", "Elaine Lau"],
       "year": 2025,
       "relevance": "Demonstrates that LLM alignment does not transfer to agent settings, directly relevant to SafePro's knowledge-alignment gap finding."
     },
     {
       "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Professional-level AI agent benchmark for ML engineering tasks."
     },
     {
       "title": "GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks",
       "authors": ["Tejal Patwardhan", "Rachel Dias"],
       "year": 2025,
       "arxiv_id": "2510.04374",
       "relevance": "Source of 195 benign professional tasks that were modified to create SafePro's harmful task dataset."
     },
     {
       "title": "RiOSWorld: Benchmarking the Risk of Multimodal Computer-Use Agents",
       "authors": ["Jingyi Yang", "Shuai Shao"],
       "year": 2025,
       "relevance": "Multimodal agent safety benchmark for computer-use risks, compared with SafePro in Table 3."
     },
     {
       "title": "OS-Harm: A Benchmark for Measuring Safety of Computer Use Agents",
       "authors": ["Thomas Kuntz", "Agatha Duzan"],
       "year": 2025,
       "arxiv_id": "2506.14866",
       "relevance": "Computer use agent safety benchmark, part of the growing agent safety evaluation literature."
     },
     {
       "title": "SHADE-Arena: Evaluating Sabotage and Monitoring in LLM Agents",
       "authors": ["Jonathan Kutasov", "Yuqi Sun"],
       "year": 2025,
       "arxiv_id": "2506.15740",
       "relevance": "Evaluates more complex agent safety problems including hidden sabotage goals."
     },
     {
       "title": "OpenAgentSafety: A Comprehensive Framework for Evaluating Real-World AI Agent Safety",
       "authors": ["Sanidhya Vijayvargiya", "Aditya Bharat Soni"],
       "year": 2025,
       "arxiv_id": "2507.06134",
       "relevance": "Framework for evaluating AI agent safety in real-world scenarios, relevant to professional agent safety evaluation."
     },
     {
       "title": "Remote Labor Index: Measuring AI Automation of Remote Work",
       "authors": ["Mantas Mazeika", "Alice Gatti"],
       "year": 2025,
       "arxiv_id": "2510.26787",
       "relevance": "Measures AI capability on professional remote work tasks, contextualizing the automation risks SafePro evaluates."
     }
   ]
 }