ai-research-survey

scan.json (19530B)

---

 {
   "paper": {
     "title": "ConceptGuard: Neuro-Symbolic Safety Guardrails via Sparse Interpretable Jailbreak Concepts",
     "authors": ["Darpan Aswal", "Céline Hudelot"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2508.16325"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Footnote 1 states 'The code will be released upon publication.' A promise of future release counts as NO."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmark datasets: TechHazardQA, CatQA, AdvBench, HarmfulQA, and Alpaca-Cleaned. All are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using SAELens toolkit and Llama-3.2-1B-Instruct but does not provide a requirements.txt, Dockerfile, or detailed environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, reproduction scripts, or step-by-step instructions are provided. The methodology is described but not in a form that enables direct reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports point estimates (TPR, FPR, Precision, F1) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims guardrails outperform baselines but provides no statistical significance tests. Comparisons are based solely on raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw metric values are reported. No effect sizes such as Cohen's d or relative improvement with baseline context are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 500 test prompts per class and 1500 training prompts per class. No justification or power analysis is provided for these sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No standard deviation, variance across seeds, or multiple-run statistics are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes four baselines: Model-only refusal, LLM-as-a-Judge, Llama-Guard-3-1B, and a linear classifier on raw activations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Llama-Guard-3-1B (2024) and the underlying Llama-3.2-1B-Instruct model, which are recent and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Full-SAE Guardrail vs Pruned-SAE Guardrail vs Raw Guardrail, effectively ablating the feature pruning and SAE components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports four metrics: True Positive Rate/Recall, False Positive Rate, Precision, and F1-score."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the guardrail's outputs is performed. Evaluation is entirely automated using metrics and GPT-5.1 as a judge. For a safety system making subjective blocking decisions, human evaluation of edge cases would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses TechHazardQA, CatQA, and AdvBench; testing uses a separate dataset (HarmfulQA) with independently generated safe prompts. Clear train/test separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides breakdowns across four attack configurations (Base, Hypothetical, BoN-Base, BoN-Hypothetical). Table 3 shows feature distribution by category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses false positives in BoN attacks, analyzes why benign prompts trigger blocking via UMAP visualization (Figure 3), and identifies model confusion from perturbed prompts."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the raw guardrail sometimes outperforms the SAE guardrails in TPR, and discusses elevated FPRs for activation-based guardrails in BoN attacks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SAEs extract interpretable jailbreak concepts and enable generalizable defenses. Table 1 and the feature analysis in Section 5 support these claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation comparing full-SAE, pruned-SAE, and raw guardrails constitutes controlled single-variable manipulation. Claims about feature pruning reducing FPR are supported by the ablation design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Llama-3.2-1B-Instruct but makes broad claims about 'LLM internals' and 'jailbreak attacks' generally. The title and abstract do not bound findings to this single small model."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses alternative explanations for elevated FPRs in BoN attacks, analyzing whether model confusion from perturbed prompts rather than guardrail failure explains the results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Llama-3.2-1B-Instruct' (Section 4.2) which is a specific model version. GPT-5.1 is named for the judge/labeling role."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes using GPT-5.1 for label categorization and as an LLM judge but does not provide the actual prompt text used for these tasks. The appendix mentions details but no full prompts are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports SAE training parameters: JumpReLU SAE, latent expansion factor of 4, L0 coefficient λs=0.5, L0 sparsity ~197, k=100 for top-k selection, Otsu threshold τ*≈0.3433."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a pipeline of SAE training, feature extraction, and linear classification."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes sampling 500 prompts from each of 3 datasets for training (1500 harmful + 1500 benign) and 500 from HarmfulQA + 500 generated for testing. The feature pruning pipeline is documented in Section 3.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A 'Limitations and Future Work' section appears at the end of Section 6, discussing multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: single hook point limitation, potentially aggressive pruning heuristic, dependence on initial safety fine-tuning, and linear classifier expressiveness limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the limitations section discusses what could be improved, it does not explicitly state what the results do NOT show (e.g., no claim about working on larger models, no claim about real-world deployment). The broad framing in abstract/title is not bounded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (activations, SAE features, classifier weights) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the data sources (TechHazardQA, CatQA, AdvBench, HarmfulQA, Alpaca-Cleaned) and the sampling procedure (500 per dataset)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: activation extraction → SAE training → feature classification (junk ratio + TTR filtering) → feature categorization → linear classifier training. Sections 3.1-3.3 and 4.1-4.2 provide details including counts (3488 rich features, 4704 junk features)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Université Paris-Saclay and CentraleSupélec. No product being evaluated is affiliated with the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It trains guardrails using model activations and tests the guardrails' classification ability, not the LLM's knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the study tests a defense mechanism (guardrail classifier), not the LLM's benchmark performance."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination of the LLM's training data is not relevant since the evaluation is of the guardrail classifier trained on activation features."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or computational overhead of the guardrail is reported, despite the approach adding SAE encoding and classification on top of normal inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No information about GPU hours, hardware used, SAE training time, or total computational budget is provided."
       }
     }
   },
   "claims": [
     {
       "claim": "SAEs extract human-interpretable jailbreak concepts from LLM internals, with ~43% of features being semantically rich.",
       "evidence": "Section 5.1 and Section 3.2 describe the feature pruning yielding 3488 rich features (~43%) and 4704 junk features. Figure 2 shows example activations.",
       "supported": "moderate"
     },
     {
       "claim": "ConceptGuard's Full-SAE guardrail achieves 0.95 F1 on base attacks and 0.91 F1 on BoN-Hypothetical, outperforming Llama-Guard-3-1B (0.72 and 0.26 respectively).",
       "evidence": "Table 1 reports these metrics directly across all four attack configurations.",
       "supported": "moderate"
     },
     {
       "claim": "Concept-guided guardrails trained only on direct attack samples generalize to stealthy jailbreaks (hypothetical framing, BoN perturbations).",
       "evidence": "Table 1 shows SAE guardrails maintain high TPR across attack configurations while baselines degrade. Section 5.2 discusses this generalization.",
       "supported": "moderate"
     },
     {
       "claim": "Jailbreak attacks share a common activation geometry in the representation space.",
       "evidence": "Section 5.2 and Figure 3 UMAP visualization show clustering of harmful vs benign activations. The generalization from base to BoN attacks provides indirect evidence.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ConceptGuard uses Sparse Autoencoders to extract interpretable jailbreak-related concepts from Llama-3.2-1B-Instruct activations, identifying ~43% of SAE features as semantically rich. A linear classifier operating on these rich features achieves competitive F1 scores (0.84-0.95) across four attack types while maintaining interpretability. The guardrails generalize from direct attacks to stealthy jailbreaks (BoN, hypothetical framing) despite training on only 1500 samples, providing evidence for shared activation geometry across jailbreak types.",
   "red_flags": [
     {
       "flag": "Single small model only",
       "detail": "All experiments use only Llama-3.2-1B-Instruct. Generalizability claims about 'LLM internals' are not bounded to this single 1B-parameter model. Larger models may have fundamentally different activation structures."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results appear to be single-run with no error bars, confidence intervals, or variance across seeds. The small test set (500 per class) makes point estimates unreliable without uncertainty measures."
     },
     {
       "flag": "GPT-5.1 used as ground truth judge without validation",
       "detail": "GPT-5.1 is used as an LLM judge for response classification, relevance assessment, and label categorization. No validation of GPT-5.1's accuracy on these tasks is provided, creating a dependency on an unvalidated oracle."
     },
     {
       "flag": "No inference latency or cost analysis",
       "detail": "For a guardrail system intended for deployment, the paper does not report inference overhead (SAE encoding + classification latency) relative to baseline approaches."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Studies deceptive capabilities in LLMs that persist through safety training, directly relevant to AI safety evaluation."
     },
     {
       "title": "Jailbroken: How does LLM safety training fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Analyzes failure modes of LLM safety training, foundational to understanding jailbreak attacks."
     },
     {
       "title": "Tree of attacks: Jailbreaking black-box LLMs automatically",
       "authors": ["Anay Mehrotra"],
       "year": 2024,
       "relevance": "Automated jailbreak attack method (TAP) relevant to LLM safety evaluation."
     },
     {
       "title": "Best-of-n jailbreaking",
       "authors": ["John Hughes", "Sara Price"],
       "year": 2024,
       "arxiv_id": "2412.03556",
       "relevance": "BoN jailbreaking method used as a key attack configuration in this paper's evaluation."
     },
     {
       "title": "Improving alignment and robustness with circuit breakers",
       "authors": ["Andy Zou", "Long Phan"],
       "year": 2024,
       "relevance": "Circuit breaker approach to LLM safety, alternative defense mechanism to guardrails."
     },
     {
       "title": "Refusal in language models is mediated by a single direction",
       "authors": ["Andy Arditi", "Oscar Obeso"],
       "year": 2024,
       "relevance": "Mechanistic interpretability finding about refusal geometry in LLMs, directly related to this paper's hypothesis about shared activation geometry."
     },
     {
       "title": "Building guardrails for large language models",
       "authors": ["Yi Dong", "Ronghui Mu"],
       "year": 2024,
       "arxiv_id": "2402.01822",
       "relevance": "Survey of LLM guardrail approaches, providing context for safety mechanism design."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng"],
       "year": 2023,
       "relevance": "Foundation work on LLM-as-a-judge evaluation methodology used in this paper."
     },
     {
       "title": "BeaverTails: Towards improved safety alignment of LLM via a human-preference dataset",
       "authors": ["Jiaming Ji"],
       "year": 2023,
       "relevance": "Safety alignment dataset and methodology for LLM safety evaluation."
     },
     {
       "title": "Steering language model refusal with sparse autoencoders",
       "authors": ["Kyle O'Brien", "David Majercak"],
       "year": 2024,
       "arxiv_id": "2411.11296",
       "relevance": "Closely related work using SAEs for LLM safety steering, direct predecessor to ConceptGuard's approach."
     }
   ]
 }