ai-research-survey

scan.json (26393B)

---

 {
   "paper": {
     "title": "Microsaccade-Inspired Probing: Positional Encoding Perturbations Reveal LLM Misbehaviours",
     "authors": ["Rui Melo", "Rui Abreu", "Corina S. Pasareanu"],
     "year": 2025,
     "venue": "Under review at ICLR 2026",
     "arxiv_id": "2510.01288",
     "doi": "10.48550/arXiv.2510.01288"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MIP introduces lightweight positional encoding perturbations to detect LLM misbehaviours across factuality, jailbreak, toxicity, and backdoor detection tasks. The method achieves near-perfect AUC on jailbreak and backdoor detection (up to 1.0), outperforming the LLMScan baseline in most settings across three models (Llama-3.2-3B, Llama-3.1-8B, Qwen2.5-14B). Toxicity detection remains the weakest area, with PCA visualizations confirming minimal class separation. The O(1) intervention complexity offers significant computational advantages over per-token or per-layer approaches.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A states 'implementation of MIP' and 'intervention scripts' will be made publicly available on GitHub and Hugging Face 'upon publication.' This is a promise of future release, not a current release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (Questions1000, WikiData, SciQ, Surge AI Toxicity, Real Toxicity Prompts, MTBA, Sleeper, VPI, AutoDAN, GCG, PAP). Appendix C describes the sources and Appendix A promises preprocessed datasets will also be released."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B specifies NVIDIA RTX A6000 GPU (48GB VRAM), driver version 550.144.03, CUDA 12.4, Intel Xeon Gold 5315Y CPU, 44 GiB RAM. Appendix D.1 describes 4-bit quantization with BitsAndBytes using nf4 and bfloat16."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper promises code release 'upon publication' but does not include a README or commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates for ACC and AUC. No confidence intervals or error bars are provided for the main results. Appendix G reports ± std dev for ablation studies but not for the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MIP outperforms LLMScan baselines but provides no statistical significance tests. Comparisons are made by visually comparing numbers in Table 1."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d effect sizes are reported for head-wise attribution analysis (Section 5.2, Figure 4a), providing magnitude of perturbation-induced differences across attention heads and layers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C mentions datasets of 'at least 1,000 entries' but does not justify why this size is sufficient. No power analysis is provided. The 80/10/10 split is described but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 1 are single-run numbers with no variance reported. Appendix G reports std dev for ablation noise types but not for the primary experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "LLMScan (Zhang et al., 2025) is used as the primary baseline across all tasks. Table 1 includes baseline ACC-AUC ranges."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LLMScan (Zhang et al., 2025) is a contemporary ICML 2025 paper and represents the state-of-the-art probing method for LLM misbehaviour detection."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and Appendix G ablate the perturbation mechanism, comparing sinusoidal PE intervention against random and Gaussian noise perturbations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both AUC and ACC (accuracy) for all experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for this automated misbehaviour detection task — the ground truth labels come from the benchmark datasets."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E describes an 80/10/10 train/val/test split with stratified sampling and fixed random seed. Results are reported on the held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-dataset breakdowns across all 4 task categories (factuality, jailbreak, toxicity, backdoor) with 12 individual dataset results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses toxicity detection as a failure case where 'PCA visualizations show minimal class separation, and LDA confirms that toxicity is intrinsically harder to discriminate.' Also notes weakness on SciQ with Qwen2.5-14B."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that toxicity detection is comparatively weaker, that WikiData with Llama-3.2-3B shows reduced performance, and that SciQ with Qwen2.5-14B achieves only 0.52 ACC (near chance). These are explicitly discussed as challenges."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims MIP 'detects failures across diverse settings including factuality, safety, toxicity, and backdoor attacks' and is 'computationally efficient.' Table 1 supports cross-task detection and Section 5.3/Appendix F supports O(1) efficiency."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'positional perturbations are highly effective at surfacing latent signals of alignment violations' and that 'misbehaviours are associated with localized deviations.' The ablation compares perturbation types but does not isolate the causal mechanism — the observed differences could be due to other factors in the pipeline."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MIP is 'model-agnostic' (Section 7) but tests only 3 models from 2 families (Llama and Qwen), all open-source. No testing on closed-source models (GPT-4, Claude) or non-instruct models. The title claims to 'Reveal LLM Misbehaviours' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for why the perturbation features are discriminative. For example, the MLP classifier might be picking up on input length or complexity differences between normal and adversarial inputs rather than positional encoding effects specifically."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures what it claims: binary classification accuracy and AUC for misbehaviour detection. The metrics match the claims without proxy gaps."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D specifies exact model names: Llama-3.2-3B-Instruct, Llama-3.1-8B-Instruct, Qwen2.5-14B-Instruct, with parameter counts, layer counts, and head counts."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting for its evaluation — it perturbs positional encodings and analyzes internal representations. The prompts/inputs come from the benchmark datasets."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E reports MLP hyperparameters: hidden sizes (128, 64), dropout 0.3, learning rate 1e-3, weight decay 1e-4, AdamW optimizer, 80 epochs with early stopping patience 10. Appendix D.1 reports quantization settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. MIP is a single forward pass perturbation + classifier pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes dataset sources and curation. Appendix E documents the 80/10/10 split strategy. For toxicity, they explicitly state selection of 500 toxic and 500 non-toxic examples for balance."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) includes a 'Limitations' subsection that discusses unexplored failure modes (bias, subtle misinformation, fairness violations)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is brief and generic: 'other forms of failure modes such as bias, subtle misinformation, or fairness violations remain unexplored.' No specific threats to the validity of the presented results are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states: 'our claim of generality should be interpreted as potential generality, pending further empirical confirmation' and lists unexplored failure types. This bounds the scope of the generalization claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The intervention features, classifier inputs, and trained models are not released. Only the datasets used are public. The actual experimental data (perturbation features, split indices) are promised for future release."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes the source and nature of each dataset. The paper explains which versions and curations were used (e.g., 'Following Zhang et al. (2025), we adopt their curated versions')."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All datasets are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: input text → token embedding → PE perturbation → forward pass → extract attention matrices and next-token distributions → compute features → MLP classifier. Appendix E documents the split and training process."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Carnegie Mellon University, FEUP, INESC-ID. The paper does not evaluate products from these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. There is an ethics statement but it does not address financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama-3.2, Llama-3.1, and Qwen2.5 models but does not state their training data cutoff dates. This matters because some benchmark datasets (Questions1000, WikiData) could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs may have seen the benchmark datasets during pretraining. The factuality tasks (Questions1000, WikiData) are particularly vulnerable since they test factual knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (Questions1000, WikiData, SciQ) predate the models' training cutoffs. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and Appendix F discuss computational efficiency with O(1) complexity analysis. Figure 5 shows cumulative FLOPs comparison across intervention strategies on the Sleeper dataset."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the hardware is specified (Appendix B) and FLOPs are compared relatively, no total compute budget (GPU hours, wall-clock time for all 66 experiment configurations) is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E mentions a fixed random seed for data splits but does not report results across multiple seeds. Main results appear to be single-seed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs produced the results in Table 1. Appendix E describes training with early stopping but does not specify number of independent runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "MLP architecture (128, 64) and hyperparameters are stated but no search budget or justification for why these were chosen is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the MLP architecture and hyperparameters were selected. The paper presents one configuration without discussing alternatives tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares across 66 experiment configurations (12 datasets × 3 models + variants) with no multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement MIP and compare against their own implementation/reproduction of LLMScan baselines. No acknowledgment of self-comparison bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 directly plots cumulative FLOPs vs. examples for MIP, Layer Intervention, Token Intervention, and LLMScan, showing performance at matched compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the binary classification task (normal vs. misbehaviour) with pre-labeled datasets actually measures real-world misbehaviour detection capability. No discussion of construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark datasets (many from 2017-2022) were in the training data of models released in 2024-2025. This is critical for the factuality task."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the features (attention perturbation differences) contain information that leaks the label. For example, adversarial inputs (GCG) have very different surface characteristics than normal inputs."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap or non-independence between train and test splits beyond stating the split is stratified."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "MIP achieves near-perfect separation on adversarial jailbreak prompting, with AUC/ACC reaching 1.0 on GCG across all three models.",
       "evidence": "Table 1 shows AUC=1.0 for GCG across all models and ACC of 1.0 for Llama models, 0.96 for Qwen.",
       "supported": "strong"
     },
     {
       "claim": "MIP outperforms LLMScan baselines across most tasks and datasets.",
       "evidence": "Table 1 compares MIP ACC/AUC against LLMScan baseline ranges across 12 datasets × 3 models. MIP exceeds the baseline range in most configurations.",
       "supported": "moderate"
     },
     {
       "claim": "MIP requires O(1) intervention complexity, compared to O(n) for per-token and O(L) for per-layer approaches.",
       "evidence": "Section 5.3 provides the complexity analysis and Appendix F (Figure 5) shows cumulative FLOPs comparison.",
       "supported": "strong"
     },
     {
       "claim": "Toxicity is intrinsically harder to discriminate via positional perturbations.",
       "evidence": "Section 5.1 notes lower performance, PCA shows minimal class separation, Cohen's d shows little localized signal in attention heads for toxicity datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Backdoor activations manifest in distinct internal patterns reliably exposed by perturbations, with discriminative signals concentrated in mid-to-late layers.",
       "evidence": "Figures 4a and 4b show Cohen's d hotspots and per-head AUC scores concentrated in layers 21-23 for Llama-3.2-3B.",
       "supported": "moderate"
     },
     {
       "claim": "MIP is model-agnostic.",
       "evidence": "Tested on 3 models (Llama-3.2-3B, Llama-3.1-8B, Qwen2.5-14B). However, all are open-source instruct models — no testing on closed-source or non-instruction-tuned models.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or significance testing on main results",
       "detail": "Table 1 reports only point estimates with no error bars, confidence intervals, or significance tests. With 66 configurations and single-run results, it is impossible to tell whether observed differences are meaningful or due to random variation in the MLP training."
     },
     {
       "flag": "Potential confound: input surface characteristics",
       "detail": "Some adversarial inputs (especially GCG) have radically different surface-level characteristics than normal inputs (gibberish tokens, unusual lengths). The MLP may be detecting these surface differences rather than positional-encoding-specific signals. This alternative explanation is not discussed."
     },
     {
       "flag": "Contamination risk for factuality benchmarks",
       "detail": "Questions1000, WikiData, and SciQ predate the models' training cutoffs. The factuality task tests whether the model 'knows' it is lying — but contamination could affect both the model's factual knowledge and its attention patterns, confounding the results."
     },
     {
       "flag": "Limited model diversity for 'model-agnostic' claim",
       "detail": "Only 3 open-source instruction-tuned models from 2 families tested. The claim of model-agnosticism would require testing across more diverse architectures (e.g., encoder-only, closed-source, non-instruction-tuned)."
     },
     {
       "flag": "Self-comparison bias with baseline",
       "detail": "The LLMScan baseline results appear to be from the authors' own reproduction. The baseline ranges in Table 1 are not clearly sourced — they could underperform due to implementation differences."
     }
   ],
   "cited_papers": [
     {
       "title": "LLMScan: Causal Scan for LLM Misbehavior Detection",
       "authors": ["Mengdi Zhang", "Goh Kai Kiat", "Peixin Zhang", "Jun Sun", "Lin Xin Rose", "Hongyu Zhang"],
       "year": 2025,
       "relevance": "Primary baseline for LLM misbehaviour detection via layer-wise interventions."
     },
     {
       "title": "Representation Engineering: A Top-Down Approach to AI Transparency",
       "authors": ["Andy Zou"],
       "year": 2023,
       "arxiv_id": "2304.12210",
       "relevance": "Linear Artificial Tomography (LAT) probing technique for factuality detection in LLMs."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs That Persist Through Safety Training",
       "authors": ["Evan Hubinger"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Backdoor attack benchmark studying deceptive models that persist through safety training."
     },
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG adversarial attack method used as a jailbreak benchmark in this paper."
     },
     {
       "title": "SmoothLLM: Defending Large Language Models Against Jailbreaking Attacks",
       "authors": ["Alexander Robey", "Eric Wong", "Hamed Hassani", "George J. Pappas"],
       "year": 2024,
       "relevance": "Defense against jailbreak attacks using input perturbations."
     },
     {
       "title": "How to Catch an AI Liar: Lie Detection in Black-Box LLMs by Asking Unrelated Questions",
       "authors": ["Lorenzo Pacchiardi"],
       "year": 2024,
       "arxiv_id": "2309.15840",
       "relevance": "Lie detection in LLMs using behavioural analysis without model internals access."
     },
     {
       "title": "Jailbroken: How Does LLM Safety Training Fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Analysis of failure modes in LLM safety training and jailbreak attacks."
     },
     {
       "title": "AutoDAN: Generating Stealthy Jailbreak Prompts on Aligned Large Language Models",
       "authors": ["Xiaogeng Liu", "Nan Xu", "Muhao Chen", "Chaowei Xiao"],
       "year": 2023,
       "arxiv_id": "2310.04451",
       "relevance": "Automated adversarial prompt generation for jailbreaking aligned LLMs."
     },
     {
       "title": "Real-ToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
       "authors": ["Samuel Gehman"],
       "year": 2020,
       "arxiv_id": "2009.11462",
       "relevance": "Toxicity benchmark for evaluating language model degeneration."
     },
     {
       "title": "Towards Monosemanticity: Decomposing Language Models with Dictionary Learning",
       "authors": ["T. Bricken"],
       "year": 2023,
       "relevance": "Sparse autoencoder interpretability technique for understanding LLM internal representations."
     },
     {
       "title": "A Mixture of Linear Corrections Generates Secure Code",
       "authors": ["Weichen Yu", "Ravi Mangal", "Terry Zhuo", "Matt Fredrikson", "Corina S. Pasareanu"],
       "year": 2025,
       "arxiv_id": "2507.09508",
       "relevance": "Mechanistic editing approach for steering LLM behaviour toward secure code generation."
     }
   ]
 }