ai-research-survey

scan.json (22886B)

---

 {
   "paper": {
     "title": "The Scales of Justitia: A Comprehensive Survey on Safety Evaluation of LLMs",
     "authors": [
       "Songyang Liu",
       "Chaozhuo Li",
       "Jiameng Qiu",
       "Xi Zhang",
       "Feiran Huang",
       "Litian Zhang",
       "Yiming Hei",
       "Philip S. Yu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.11094",
     "doi": "10.48550/arXiv.2506.11094"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey organizes LLM safety evaluation along four dimensions: why to evaluate (distinguishing safety from capability evaluation), what to evaluate (toxicity, robustness, ethics, bias/fairness, truthfulness, plus privacy/copyright/code generation), where to evaluate (metrics, 31 datasets/benchmarks), and how to evaluate (human, rule-based, and model-based evaluators plus 7 integrated frameworks). The paper identifies six challenges including the lack of unified evaluation standards, the need for dynamic benchmarks, and the gap between evaluating text-only LLMs versus multimodal models and agents.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or analysis scripts are mentioned or linked anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not release a structured dataset of the papers reviewed, its taxonomy mapping, or any extracted data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a survey, analysis code and environment could still have been released."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions for reproducing the survey's paper selection or analysis process are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments; no quantitative results are produced."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments; no comparative claims requiring statistical tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments; no effect sizes to report."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I compares this survey against five prior surveys (Guo et al. 2023, Dong et al. 2024, Yi et al. 2024, Ran et al. 2024, Röttger et al. 2025) on seven dimensions including taxonomy of tasks, metrics, datasets, evaluators, and frameworks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison surveys span 2023-2025, which are contemporary and relevant baselines for a 2025 survey."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper has no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper does not run experiments with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper does not evaluate system outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper does not use test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed per-category breakdowns across toxicity (Section III.A), robustness (III.B), ethics (III.C), bias and fairness (III.D), truthfulness (III.E), and other domains (III.F), each with subcategories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Each subsection in Sections III-V ends with a discussion of unresolved challenges and limitations of current approaches. Section VI is entirely devoted to challenges and future directions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings throughout, e.g., that 'no attack or defense is universally effective' (Section III.B), that LLMs 'still exhibit systematic biases' (Section III.D), and that current evaluation frameworks 'focus more on integrating existing methods than on advancing evaluation methodology' (Section V.D)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'structured overview' with a four-dimensional taxonomy, which is delivered in Sections II-V. The claimed coverage of tasks, metrics, datasets, benchmarks, evaluation methods, and future directions are all present in the paper."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that synthesizes existing work and makes no causal claims of its own."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to be 'comprehensive and systematic' (abstract, contributions) but does not describe any search methodology, databases searched, time period covered, or inclusion/exclusion criteria. The title claims 'comprehensive survey' but without documented scope boundaries, the comprehensiveness claim cannot be verified. The paper acknowledges MLLMs and agents are underexplored (Section VI.E) but doesn't state this as a scope boundary upfront."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Pure survey/taxonomy paper presenting no empirical results of its own."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no measurements of its own."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not run any models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper uses no scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe any systematic paper selection pipeline. There are no search queries, no database descriptions, no screening stages, no filtering criteria, and no PRISMA-style flow diagram. How the approximately 215 references were identified and selected is entirely undocumented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section VI 'Challenges and Future Directions' discusses challenges in the field of LLM safety evaluation, but there is no section discussing limitations of the survey itself — e.g., potential gaps in coverage, search methodology limitations, or reviewer bias."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to the validity of the survey's own methodology or conclusions are discussed. The challenges in Section VI are about the field, not about the survey."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what is excluded from its scope. The focus on text-based LLMs (vs. MLLMs/agents) emerges only in Section VI.E as a future direction rather than being declared upfront as a scope boundary. No time period, language, or venue restrictions are stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The corpus of surveyed papers, the taxonomy mapping, and the comparative analysis data are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the surveyed papers were identified. No search databases, queries, or time period are mentioned. The paper jumps directly into its taxonomy without describing the literature search process."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "How papers were found and selected for inclusion is not described. For a survey, this is analogous to participant recruitment — the paper selection process should be documented to assess coverage and bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No pipeline from initial search to final included papers is documented. There is no indication of how many candidate papers were found, how many were screened, or how many were excluded at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or sponsorships are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Beijing University of Posts and Telecommunications, Jinan University, Beihang University, China Academy of Information and Communications Technology, and University of Illinois at Chicago."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. This is non-disclosure rather than evidence of independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no method of its own to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not follow PRISMA or any structured review protocol. There is no search strategy, no reproducible queries, no flow diagram showing paper identification/screening/inclusion, and no protocol registration. The paper organizes by taxonomy but does not document its search methodology."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of the papers it reviews. All cited works are treated equally regardless of their rigor. For example, Table II lists 31 datasets/benchmarks without any quality assessment of the underlying evaluation methodologies."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether the published literature on LLM safety evaluation skews toward positive results, novel attacks, or certain evaluation dimensions, nor does it use funnel plots or other bias detection methods."
       }
     }
   },
   "claims": [
     {
       "claim": "This is the first survey to present a systematic and holistic synthesis of recent advancements in safety evaluation of LLMs.",
       "evidence": "Section I contributions list and Table I comparing against five prior surveys (Guo 2023, Dong 2024, Yi 2024, Ran 2024, Röttger 2025) showing none cover all seven dimensions.",
       "supported": "moderate"
     },
     {
       "claim": "Existing safety evaluations are fragmented, addressing single aspects such as datasets or evaluators, lacking a holistic analysis of the entire pipeline.",
       "evidence": "Table I shows prior surveys covering at most 4 of 7 dimensions (taxonomy, metrics, datasets, evaluators, frameworks, challenges). Section I discusses this gap.",
       "supported": "moderate"
     },
     {
       "claim": "No attack or defense is universally effective, and robustness does not correlate with model size.",
       "evidence": "Cited from HarmBench results in Section III.B.2, referencing [31] and [138]. This is a re-stated finding, not original to this survey.",
       "supported": "moderate"
     },
     {
       "claim": "The research community has yet to reach consensus on standardized procedures for conducting LLM safety evaluations.",
       "evidence": "Section VI.A discusses inconsistencies in ASR definitions, benchmark usage, and evaluation methods across studies. Supported by observations throughout Sections IV-V showing varied metrics and frameworks.",
       "supported": "strong"
     },
     {
       "claim": "Current safety evaluation metrics, especially ASR, lack unified definitions making cross-study comparison difficult.",
       "evidence": "Section IV.A.1 states 'How to evaluate the success of a jailbreak attempt against an LLM has not been unified [17].' Sections IV.A.1-2 describe how different studies use different definitions and implementations.",
       "supported": "strong"
     },
     {
       "claim": "Safety evaluation should evolve from safe to responsible LLMs, encompassing transparency, interpretability, and auditability.",
       "evidence": "Section VI.F proposes this as a future direction but provides no empirical evidence or concrete framework for achieving it.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic search methodology",
       "detail": "The paper claims to be 'comprehensive and systematic' but does not describe how papers were identified, what databases were searched, what search terms were used, or what inclusion/exclusion criteria were applied. Without a documented search methodology, the comprehensiveness claim is unverifiable."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "The survey treats all cited papers equally regardless of their methodological rigor. Papers with robust evaluation designs are summarized alongside papers with minimal evidence, with no indication of relative quality. This risks laundering weak results alongside strong ones."
     },
     {
       "flag": "No survey-specific limitations discussed",
       "detail": "Section VI discusses challenges in the field but never discusses limitations of the survey itself — potential gaps in coverage, language/venue biases in paper selection, or reviewer subjectivity in taxonomy classification."
     },
     {
       "flag": "Self-citation pattern",
       "detail": "Multiple papers from the authors' own groups are cited (references [5], [6], [10], [11], [12], [78], [113], [114], [146], [150]). While not inherently problematic, the volume of self-citations in a survey claiming objectivity warrants noting."
     },
     {
       "flag": "Narrative review claiming systematic status",
       "detail": "The paper reads as a well-organized narrative review but repeatedly calls itself 'systematic' and 'comprehensive.' Without PRISMA guidelines, structured search, or quality assessment, it does not meet the standard definition of a systematic review."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Y. Chang", "X. Wang", "J. Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey on general LLM evaluation providing the baseline against which safety-specific evaluation is distinguished."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["J. Achiam", "S. Adler", "S. Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the primary LLMs evaluated across multiple safety dimensions in the surveyed papers."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["A. Zou", "Z. Wang", "N. Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Introduces GCG adversarial attack and AdvBench benchmark, foundational work in LLM safety jailbreak evaluation."
     },
     {
       "title": "DecodingTrust: A comprehensive assessment of trustworthiness in GPT models",
       "authors": ["B. Wang", "W. Chen", "H. Pei"],
       "year": 2023,
       "relevance": "Comprehensive trustworthiness evaluation framework for GPT models covering robustness, bias, and safety dimensions."
     },
     {
       "title": "HarmBench: A standardized evaluation framework for automated red teaming and robust refusal",
       "authors": ["M. Mazeika", "L. Phan", "X. Yin"],
       "year": 2024,
       "arxiv_id": "2402.04249",
       "relevance": "Standardized red-teaming benchmark covering 18 attack methods with finding that robustness does not correlate with model size."
     },
     {
       "title": "JailbreakBench: An open robustness benchmark for jailbreaking large language models",
       "authors": ["P. Chao", "E. Debenedetti", "A. Robey"],
       "year": 2024,
       "relevance": "Open benchmark for jailbreak evaluation with standardized pipeline and public leaderboard."
     },
     {
       "title": "Purple Llama CyberSecEval: A secure coding benchmark for language models",
       "authors": ["M. Bhatt", "S. Chennabasappa", "C. Nikolaidis"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Framework evaluating security of LLM-generated code, finding ~30% insecure code generation rate across tested models."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["N. Perry", "M. Srivastava", "D. Kumar"],
       "year": 2023,
       "relevance": "Empirical study on AI coding assistant security implications, directly relevant to LLM code generation safety evaluation."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["H. Inan", "K. Upasani", "J. Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Foundational open-source safety evaluation model fine-tuned from Llama2, widely used as automated safety judge."
     },
     {
       "title": "JailbreakEval: An integrated toolkit for evaluating jailbreak attempts against large language models",
       "authors": ["D. Ran", "J. Liu", "Y. Gong"],
       "year": 2024,
       "arxiv_id": "2406.09321",
       "relevance": "Toolkit for standardized jailbreak evaluation highlighting the lack of unified evaluation methods."
     },
     {
       "title": "SALAD-Bench: A hierarchical and comprehensive safety benchmark for large language models",
       "authors": ["L. Li", "B. Dong", "R. Wang"],
       "year": 2024,
       "arxiv_id": "2402.05044",
       "relevance": "Large-scale safety benchmark with 30K prompts across 66 dimensions, one of the most comprehensive safety datasets catalogued."
     },
     {
       "title": "A survey of large language models",
       "authors": ["W. X. Zhao", "K. Zhou", "J. Li"],
       "year": 2023,
       "arxiv_id": "2303.18223",
       "relevance": "Broad LLM survey providing context for distinguishing general evaluation from safety-specific evaluation."
     }
   ]
 }