ai-research-survey

calibration.json (19688B)

---

 {
   "paper_slug": "advancing-llm-safe-2025",
   "calibration_date": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 47,
   "disagreement_count": 3,
   "agreement_rate": 0.94,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false, reasoning that automated evaluation against benchmark labels is appropriate and human evaluation is irrelevant. Opus set applies=true because safety classification has inherently subjective elements — the ground truth labels in this paper are derived from keyword matching ('Sorry', 'unable' = safe; 'sure', 'certainly' = harmful), not human annotation. Human evaluation of whether responses are truly safe or harmful is relevant to validating these proxy labels and the system's real-world utility. The criterion could reasonably be expected of this paper type, so applies=true, answer=false is more appropriate."
     },
     {
       "category": "setup_transparency",
       "question": "hyperparameters_reported",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet gave answer=true based on the ranker hyperparameters reported in Section 4.1 (learning rate 0.001, weight decay 0.0001, dropout 0.1, momentum 1.0, bottom 25% layers). However, the paper uses LLMs to generate candidate responses via 'stochastic decoding with moderate temperature' (Section 3.1) without specifying the actual temperature value, top-p, or max tokens. The schema explicitly states: 'If the paper uses an LLM API without stating temperature/sampling settings, NO — these significantly affect output.' The LLM sampling parameters are critical to reproducibility since they determine candidate diversity, which directly affects SRR's performance."
     },
     {
       "category": "conflicts_of_interest",
       "question": "funder_independent_of_outcome",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false reasoning that no funding is disclosed so independence cannot be assessed. The schema says 'NA if unfunded' — but the paper never states it is unfunded; it simply has no funding disclosure at all. An absent funding statement is different from a confirmed absence of funding. Since we cannot confirm the paper is unfunded, the criterion applies (the paper could have funding that is undisclosed), and the answer is false because independence cannot be verified without disclosure."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'Our code will be available upon publication.' Per the schema, a promise of future release counts as NO. No working URL or archive is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HarmBench, SorryBench, JailbreakBench, BBQ, Harmcopy, and MATH. Per the schema, 'If the data is a standard public benchmark they didn't modify, YES.' The custom train/test splits (50 per dataset for training) are not released, but the underlying datasets are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists ranker hyperparameters (learning rate, weight decay, dropout, momentum) but provides no requirements.txt, Dockerfile, or environment setup section with library versions. No framework versions (PyTorch, HuggingFace, etc.) are specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Algorithm 1 gives pseudocode but not actual commands or a README. Code is promised but not released."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-8 are reported as single point estimates (accuracy percentages) with no confidence intervals, error bars, or +/- notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'our method greatly outperforms the reward model' but performs no statistical significance tests. No p-values, t-tests, or bootstrap tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw accuracy numbers in tables but does not report formal effect sizes (Cohen's d, odds ratios, relative risk, or percentage improvement). Showing two numbers side by side in a table constitutes raw differences, not effect size reporting per the schema."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 50 examples per dataset for training and the rest for testing. No justification for this split size is given, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviation, variance, or results across multiple random seeds are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A GPT-2-based reward model serves as the baseline for ranking accuracy (Tables 1-4). For real-world application (Tables 5-7), 'first accuracy' (choosing the highest-probability response) is the baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The primary baseline is GPT-2 (2019), which the authors acknowledge is '20 times larger than the ranker model' but is fundamentally outdated. Contemporary safety defense methods (SafeDecoding, LlamaGuard, self-reminder) are discussed in related work but not used as experimental baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is present. Key design choices — layer selection (bottom 25%), single-layer transformer encoder, listwise KL-divergence loss, cosine similarity scoring — are not individually evaluated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only ranking accuracy is used as an evaluation metric throughout all experiments (Tables 1-8). No secondary metrics such as attack success rate, refusal rate, false positive rate, or F1 are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is conducted. Safety labels are derived from keyword matching ('Sorry', 'unable' = safe; 'sure', 'certainly' = harmful) rather than human annotation. Human evaluation of whether responses are truly safe is relevant given the limitations of keyword-based labeling as a safety proxy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly states: 'For each dataset, we extract 50 of them as the training dataset, and the rest is used as the testing dataset.' The train/test separation is clear."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are broken down by model and dataset but not by category within datasets. HarmBench contains prompts in 'various areas' (Section 4.1) and BBQ has multiple bias categories, but no within-dataset category breakdowns are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure case analysis or qualitative error examples are presented. The Limitations section mentions general areas for improvement but does not examine specific cases where SRR fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The BBQ fairness result (52.52% average accuracy, with Vicuna at 50.64% — essentially random) is honestly reported as a negative result. Section 4.4 explicitly acknowledges the low performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SRR 'significantly improves robustness to adversarial prompts,' supported by Tables 5-7 showing improvements over baseline. The claim about selecting safe responses using hidden states is supported by the methodology and experimental results. Claims are appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'SRR significantly enhances the safety alignment of LLMs' and 'SRR can effectively improve the safety mechanisms of LLMs.' While the controlled benchmark comparisons support the narrower claim that SRR ranks better than GPT-2 reward model, the broader causal claims about improving 'safety mechanisms' and 'safety alignment' extend beyond what comparison against a single weak baseline demonstrates."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only three 7B open-source models but makes broad claims about 'LLM safety' generally. The title 'Advancing LLM Safe Alignment' and claims like 'SRR serves as a practical and effective safeguard module for LLM alignment' are not bounded to the tested model sizes, architectures, or attack types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for results are discussed. The keyword-based labeling confound (the ranker may learn keyword patterns rather than genuine safety-relevant representations) is not addressed. No consideration of whether simpler methods could achieve similar results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Qwen2.5-7b-Instruct', 'Mistral-7-v0.3', and 'Vicuna-7b-v1.5' with references. These are specific open-source model checkpoints with version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper references 'In-context Attack and In-context Defense' for response sampling but does not provide the actual prompt text or demonstrations used. The reader cannot reconstruct what was sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Ranker hyperparameters are reported (learning rate 0.001, weight decay 0.0001, dropout 0.1, momentum 1.0, bottom 25% layers). However, the LLM sampling parameters are not specified — Section 3.1 says 'stochastic decoding with moderate temperature' without giving the exact temperature value, top-p, or max tokens. Per the schema, 'If the paper uses an LLM API without stating temperature/sampling settings, NO.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The SRR framework is described in detail in Section 3: candidate response generation (3.1), ranker architecture with three steps (3.2), training objectives (3.3), and pseudocode in Algorithm 1. The inference pipeline is also described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the preprocessing pipeline: sample 20 responses each from In-context Attack and In-context Defense per prompt, filter using keyword criteria (safe: 'Sorry', 'unable', 'illegal', 'understand'; harmful: 'sure', 'certainly'), split 50 for training and the rest for testing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations' and contains substantive discussion of the approach's limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations in Section 6 are generic: 'might need task-specific fine-tuning,' 'adaptability to special-domain safety scenarios requires further testing,' 'effectiveness partly relies on diverse candidate responses.' These are not specific to this study's design — they don't address the keyword-based labeling confound, the weak baseline, the small sample sizes, or the limited model diversity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper makes broad claims about 'real-world deployment' and 'safeguard module for LLM alignment' without bounding these to the three 7B models and three safety benchmarks actually tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The sampled LLM responses, filtered training/test splits, keyword-labeled data, and trained ranker weights are not released. Only the original benchmark datasets are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection procedure: sampling from base models using In-context Attack and In-context Defense, 20 times each per prompt. Keyword filtering criteria for labeling safe vs. harmful responses are explicitly stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data comes from LLM-generated responses to existing safety benchmarks. This criterion does not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: (1) sample 20 attack and 20 defense responses per prompt, (2) filter by keywords to assign safety labels, (3) split into 50 training / rest test, (4) construct candidate pairs for ranking."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present. There is no mention of grants, funding agencies, or corporate sponsors anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with affiliations at Peking University (State Key Lab of General Artificial Intelligence, School of Intelligence Science and Technology, School of Mathematical Sciences, Institute for Artificial Intelligence). The paper evaluates open-source models not affiliated with the authors' institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The schema says 'NA if unfunded,' but the paper does not state it is unfunded — it simply lacks any funding disclosure. An absent funding statement differs from confirmed absence of funding. Independence cannot be verified without disclosure."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears in the paper. The absence of a disclosure statement does not confirm absence of conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen2.5-7b-Instruct, Mistral-7b-v0.3, and Vicuna-7b-v1.5 but does not state training data cutoff dates for any model. This matters because the safety benchmarks may have been in the models' training data, which could affect the distribution of safe vs. harmful responses generated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between safety benchmark prompts and base model training data. HarmBench and JailbreakBench are public benchmarks that could appear in pre-training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HarmBench (ICML 2024), JailbreakBench (2024), and SorryBench (ICLR 2025) were published before the likely training cutoffs of the base models. The paper does not address whether benchmark contamination could affect results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. All experiments use LLM-generated responses evaluated against automated benchmark labels."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SRR is 'lightweight' and overhead is 'negligible compared to full decoding,' but no actual cost figures are provided. No token counts, wall-clock time, or cost-per-example are reported. Generating 20 candidates per prompt is a significant expense that is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, training time, or total compute budget is reported. The ranker has <5M parameters, but the cost of running the 7B base model to extract representations is not quantified."
       }
     }
   }
 }