ai-research-survey

scan.json (25254B)

---

 {
   "paper": {
     "title": "CCFC: Core & Core–Full–Core Dual-Track Defense for LLM Jailbreak Protection",
     "authors": ["Jiaming Hu", "Haoyu Wang", "Debarghya Mukherjee", "Ioannis Ch. Paschalidis"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2508.14128"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link or code repository URL is provided in the paper. The only URL provided is to a HuggingFace dataset for attack prompts (https://huggingface.co/datasets/UWNSL/SafeDecoding-Attackers), which is a third-party resource, not the authors' code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references publicly available datasets: AdvBench (Zou et al., 2023) for harmful benchmarks and Just-Eval for utility evaluation (500 instructions). Attack prompts are linked via a HuggingFace dataset URL. All evaluation data is from public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided—no requirements.txt, Dockerfile, conda environment, or library versions are listed. The paper does not describe the computational environment used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method is described algorithmically but there are no concrete commands or instructions for reproducing experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (e.g., ASR percentages and Just-Eval scores). No confidence intervals, error bars, or ± notation are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CCFC 'consistently outperforms' baselines based on comparing raw numbers in Tables 1 and 2. No statistical significance tests (p-values, t-tests, etc.) are used to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw ASR percentages but does not provide standardized effect sizes (Cohen's d, odds ratios). The abstract claims '50-75% reduction' which provides some context, but the results tables lack any formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 50 attack prompts per jailbreak attack method and 500 Just-Eval instructions for utility evaluation, but provides no justification for why these sample sizes were chosen or whether they are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results appear to be single-run numbers with no indication of result stability across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baseline defenses are compared: Self-Examination, Paraphrase, Retokenization, Self-Reminder, ICD, and DATDP. A 'No Defense' baseline is also included. Results are shown in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DATDP (Armstrong et al., 2025), ICD (Wei et al., 2023), Self-Reminder (Xie et al., 2023), SmoothLLM (Robey et al., 2023), and others. DATDP from 2025 is contemporary. The selection covers the main prompt-level defense approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "CCFC has multiple components (core extraction, dual-track processing, double safety check), but no ablation study is presented to show the individual contribution of each component. For example, there is no comparison of core-only vs. CFC-only vs. full CCFC."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ASR (Attack Success Rate) for robustness evaluation and five Just-Eval dimensions (helpfulness, clarity, factuality, depth, engagement) for utility evaluation, reported in Tables 1 and 2 respectively."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated: ASR is computed using Dic-Judge (a keyword-based classifier), and utility is measured via Just-Eval automated scores. No human evaluation of defense outputs is performed, which is relevant given claims about response quality preservation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe any separation between development/tuning data and test data. The few-shot examples used in core extraction and the evaluation queries appear to use the same benchmark sets without explicit held-out splitting."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-attack-type breakdown (GCG, AutoDAN, PAIR, DeepInception, AdvBench) and Table 2 provides per-dimension utility scores (Helpfulness, Clear, Factual, Deep, Engaging), both separated by model (Vicuna, Llama-2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 6) discusses potential failure modes: the method may struggle with 'highly sophisticated attacks that seamlessly integrate malicious intent within semantically coherent contexts' and 'the extraction process could potentially miss subtle adversarial elements.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows CCFC performing at or near the best level. No configurations or approaches that failed are reported. No negative results or unsuccessful design choices are mentioned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims CCFC 'cuts attack success rates by 50-75% versus state-of-the-art defenses.' Looking at Table 1 for Vicuna, comparing CCFC to DATDP (the best baseline): GCG 6% vs 12%, AutoDAN 0% vs 2%, PAIR 2% vs 2%, DeepInception 2% vs 8%. The reductions relative to DATDP are not consistently 50-75%. The claim appears to be relative to other, weaker baselines rather than the 'state-of-the-art.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about why CCFC works (e.g., core extraction 'effectively implementing distraction elimination strategies,' CFC track 'disrupts structure-dependent attacks'), but without ablation studies, these causal mechanisms are not empirically validated. The improvement could come from any single component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CCFC provides 'universal applicability across attack types' and is 'a practical solution for enhancing LLM safety,' but tests only on two 7B open-source models (Vicuna-7B and LLaMA2-7B-chat) with four attacks. No testing on larger models, closed-source models, or broader attack categories. The title and abstract make broad claims not bounded to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider whether the improvements might be due to the additional inference passes (CCFC uses multiple forward passes), the specific few-shot examples chosen, or other confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Vicuna-7B' and 'LLaMA2-7B-chat' but does not provide exact version identifiers, checkpoint hashes, or snapshot dates. These model names alone are insufficient as multiple versions exist."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The few-shot core extraction prompt is provided in full in Appendix Tables 3 and 4. The prompt includes the system instruction and demonstration examples used for the extraction phase."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported—no temperature, top-p, max tokens, or other sampling parameters for the LLM inference. The paper does not describe any configuration details for the model calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The CCFC framework is described in detail in Section 4 with formal notation: core extraction via few-shot prompting (Eq. 2), dual-track parallel defense (Eqs. 3-4), and double safety check logic (Eq. 5). The workflow is illustrated in Figure 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the attack prompts were generated or preprocessed, beyond pointing to a HuggingFace dataset URL. No details on how the 50 prompts per attack were selected from AdvBench or how Just-Eval instructions were chosen."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 includes a 'Limitations' subsection that discusses the method's reliance on few-shot core extraction quality and potential struggles with sophisticated attacks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is brief and somewhat generic. It mentions that core extraction 'may struggle with highly sophisticated attacks' but does not discuss specific threats like the limited model selection (only 7B models), reliance on keyword-based ASR evaluation, or the small attack sample size."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to 7B open-source models or to the specific attack types tested. The conclusion claims 'universal applicability' without stating scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw model outputs, intermediate core extractions, and safety check decisions are not available. Only aggregated ASR percentages and Just-Eval scores are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes using AdvBench (Zou et al., 2023) with 50 harmful queries per attack method and 500 Just-Eval instructions for utility evaluation. Attack prompts are sourced from a referenced HuggingFace dataset."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All evaluation uses automated benchmarks and models."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from attack prompt generation to final ASR computation is not fully documented. It is unclear how Dic-Judge keyword matching works exactly, what the safe phrases list is, or how edge cases in classification are handled."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are provided in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Boston University (Dept. of Math & Statistics, Dept. of ECE & Systems Eng., Dept. of Biomedical Eng., Faculty of Computing & Data Sciences) and University at Albany (Dept. of Computer Science)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of funder from outcomes cannot be assessed. The absence of funding disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a defense mechanism against jailbreak attacks, not a pre-trained model's capability on a benchmark. The defense operates at the prompt level and does not depend on model training data knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above—the paper tests a defense strategy, not model knowledge. Train/test overlap for the underlying model is not relevant to the defense evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above—the evaluation is about whether the defense blocks attacks, not whether the model has memorized benchmark answers. Contamination is not relevant to this evaluation setup."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'minimal overhead' and 'a constant number of additional forward passes' but does not quantify inference cost, latency, tokens consumed, or wall-clock time. No concrete cost comparison with baselines is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No information about GPU hours, hardware used, total compute time, or computational resources is provided anywhere in the paper."
       }
     }
   },
   "claims": [
     {
       "claim": "CCFC cuts attack success rates by 50-75% versus state-of-the-art defenses against strong adversaries.",
       "evidence": "Table 1 shows ASR results. On Vicuna, CCFC achieves 6% (GCG), 0% (AutoDAN), 2% (PAIR), 2% (DeepInception). Compared to DATDP (best baseline): 12%, 2%, 2%, 8%. The 50-75% claim is not consistently supported against the best baseline.",
       "supported": "weak"
     },
     {
       "claim": "CCFC maintains helpfulness of LLMs when handling queries from benign users.",
       "evidence": "Table 2 shows Just-Eval scores. CCFC achieves 4.353 avg on Vicuna (vs 4.339 No Defense) and 4.398 avg on Llama-2 (vs 4.432 No Defense). CCFC is close to undefended performance.",
       "supported": "moderate"
     },
     {
       "claim": "CCFC achieves 0% ASR on AdvBench harmful benchmark for both models.",
       "evidence": "Table 1 shows 0% ASR on AdvBench for both Vicuna and Llama-2 with CCFC.",
       "supported": "strong"
     },
     {
       "claim": "CCFC consistently outperforms state-of-the-art prompt-level defenses.",
       "evidence": "Table 1 shows CCFC achieves lowest or tied-lowest ASR across most attack types. On Llama-2, ICD also achieves 0% across all attacks. DATDP is very close to CCFC on several metrics. The claim of consistent superiority is overstated.",
       "supported": "moderate"
     },
     {
       "claim": "CCFC provides universal applicability across attack types without model modifications.",
       "evidence": "The paper tests on only two 7B open-source models and four attack types. 'Universal applicability' is not demonstrated—no closed-source models, larger models, or broader attack categories are tested.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CCFC proposes a dual-track prompt-level defense against LLM jailbreak attacks that combines semantic core extraction with a Core-Full-Core structural disruption approach and a double safety check. On Vicuna-7B and LLaMA2-7B-chat, CCFC achieves near-zero attack success rates against GCG, AutoDAN, PAIR, and DeepInception attacks, matching or exceeding DATDP (the strongest baseline) while maintaining utility scores close to undefended models. However, the evaluation is limited to two small open-source models, uses no ablation study, and lacks statistical rigor (no error bars, significance tests, or multi-run variance).",
   "red_flags": [
     {
       "flag": "No ablation study",
       "detail": "CCFC has three distinct components (core extraction, dual-track processing, double safety check) but no ablation is performed to determine which component drives the improvement. The causal claims about why each component works are not empirically validated."
     },
     {
       "flag": "Overstated abstract claims",
       "detail": "The abstract claims '50-75% reduction versus state-of-the-art defenses' but DATDP (the actual best baseline) shows very similar performance. The 50-75% figure appears cherry-picked against weaker baselines rather than the best comparators."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or variance. With only 50 attack prompts per condition, individual results could vary significantly between runs."
     },
     {
       "flag": "Very narrow model scope with broad claims",
       "detail": "Only two 7B open-source models are tested, yet the paper claims 'universal applicability' and 'a practical solution for safer LLM deployment.' No testing on larger, more capable, or closed-source models."
     },
     {
       "flag": "Keyword-based ASR evaluation",
       "detail": "ASR is computed using Dic-Judge, a keyword-based classifier detecting refusal phrases. This is a known weak evaluation method that can be easily fooled by responses that avoid standard refusal phrases but are still safe, or responses that include refusal-like phrases but still contain harmful content."
     },
     {
       "flag": "Missing computational cost analysis",
       "detail": "The paper claims 'minimal overhead' but provides no quantitative cost comparison. CCFC requires at least 3 forward passes (core extraction, core track, CFC track) plus safety checking, which could be significant compared to single-pass baselines."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Introduces the GCG attack and AdvBench benchmark, foundational to evaluating LLM jailbreak defenses."
     },
     {
       "title": "Jailbreaking black box large language models in twenty queries",
       "authors": ["Patrick Chao", "Alexander Robey", "Edgar Dobriban", "Hamed Hassani", "George J Pappas", "Eric Wong"],
       "year": 2025,
       "relevance": "Introduces the PAIR attack method, a key black-box jailbreak approach tested in this evaluation."
     },
     {
       "title": "AutoDAN: Generating stealthy jailbreak prompts on aligned large language models",
       "authors": ["Xiaogeng Liu", "Nan Xu", "Muhao Chen", "Chaowei Xiao"],
       "year": 2023,
       "arxiv_id": "2310.04451",
       "relevance": "Proposes genetic-algorithm-based jailbreak prompt generation, one of the attack methods evaluated."
     },
     {
       "title": "DeepInception: Hypnotize large language model to be jailbreaker",
       "authors": ["Xuan Li", "Zhanke Zhou", "Jianing Zhu", "Jiangchao Yao", "Tongliang Liu", "Bo Han"],
       "year": 2023,
       "arxiv_id": "2311.03191",
       "relevance": "Introduces a narrative-based jailbreak attack that proved challenging for most baselines in this evaluation."
     },
     {
       "title": "SmoothLLM: Defending large language models against jailbreaking attacks",
       "authors": ["Alexander Robey", "Eric Wong", "Hamed Hassani", "George J Pappas"],
       "year": 2023,
       "arxiv_id": "2310.03684",
       "relevance": "A perturbation-based defense against jailbreaks, representing the prompt-level defense approach."
     },
     {
       "title": "Defending ChatGPT against jailbreak attack via self-reminders",
       "authors": ["Yueqi Xie", "Jingwei Yi", "Jiawei Shao", "Justin Curl", "Lingjuan Lyu", "Qifeng Chen", "Xing Xie", "Fangzhao Wu"],
       "year": 2023,
       "relevance": "Proposes the Self-Reminder defense baseline used in this evaluation."
     },
     {
       "title": "Gradient Cuff: Detecting jailbreak attacks on large language models by exploring refusal loss landscapes",
       "authors": ["Xiaomeng Hu", "Pin-Yu Chen", "Tsung-Yi Ho"],
       "year": 2024,
       "relevance": "A gradient-based model-level defense method contrasted with the prompt-level approach of CCFC."
     },
     {
       "title": "SafeDecoding: Defending against jailbreak attacks via safety-aware decoding",
       "authors": ["Zhangchen Xu", "Fengqing Jiang", "Luyao Niu", "Jinyuan Jia", "Bill Yuchen Lin", "Radha Poovendran"],
       "year": 2024,
       "arxiv_id": "2402.08983",
       "relevance": "A decoding-level defense approach, provides the attack prompt dataset used in this paper's experiments."
     },
     {
       "title": "Defense against the dark prompts: Mitigating best-of-n jailbreaking with prompt evaluation",
       "authors": ["Stuart Armstrong", "Matija Franklin", "Connor Stevens", "Rebecca Gorman"],
       "year": 2025,
       "arxiv_id": "2502.00580",
       "relevance": "Introduces DATDP, the strongest baseline defense compared against CCFC in this evaluation."
     },
     {
       "title": "Robust prompt optimization for defending language models against jailbreaking attacks",
       "authors": ["Andy Zhou", "Bo Li", "Haohan Wang"],
       "year": 2024,
       "relevance": "Proposes gradient-based defensive suffix optimization (RPO), a model-level defense approach."
     },
     {
       "title": "Certifying LLM safety against adversarial prompting",
       "authors": ["Aounon Kumar", "Chirag Agarwal", "Suraj Srinivas", "Aaron Jiaxun Li", "Soheil Feizi", "Himabindu Lakkaraju"],
       "year": 2023,
       "arxiv_id": "2309.02705",
       "relevance": "Proposes Erase-and-check defense method using external safety filters, a model-level defense baseline."
     },
     {
       "title": "Tree of attacks: Jailbreaking black-box LLMs automatically",
       "authors": ["Anay Mehrotra", "Manolis Zampetakis", "Paul Kassianik", "Blaine Nelson", "Hyrum Anderson", "Yaron Singer", "Amin Karbasi"],
       "year": 2024,
       "relevance": "Extends black-box jailbreak attacks with tree-structured search, relevant to understanding advanced attack strategies."
     }
   ]
 }