ai-research-survey

scan.json (32212B)

---

 {
   "paper": {
     "title": "Inference time LLM alignment in single and multidomain preference spectrum",
     "authors": [
       "Sadat Shahriar",
       "Zheng Qi",
       "Nikolaos Pappas",
       "Srikanth Doss",
       "Monica Sunkara",
       "Kishaloy Halder",
       "Manuel Mager",
       "Yassine Benajiba"
     ],
     "year": 2024,
     "venue": "arXiv (submitted to ICLR 2025)",
     "arxiv_id": "2410.19206",
     "doi": "10.48550/arXiv.2410.19206"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Alignment Vectors (AVs), obtained by subtracting base model weights from DPO-aligned model weights, enable inference-time preference tunability by varying a scalar lambda coefficient. Single-domain experiments on Mistral-7B across medical, financial, and legal domains show that lambda can shift responses between avoidance, generic, and expert proficiency levels. Multidomain AV arithmetic achieves 22 of 27 possible domain-behavior combinations, 12x faster than joint training, though with substantially lower accuracy (e.g., 46% vs 100%). AVs transfer across fine-tuning stages of the same model family, maintaining safety alignment while adjusting domain proficiency.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository link is provided. The paper makes no mention of releasing source code for the alignment vector method or training pipeline."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The dataset will be available through this link' (Section 1) but no actual URL is provided in the paper text. A promise of future release counts as NO."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA A100 GPUs' and 'Mistral-7B-Instruct-v0.3' but provides no requirements.txt, Dockerfile, library versions, or detailed environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a conceptual level but lacks runnable scripts or commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported anywhere. Tables 1-5 present only point estimates of preference accuracy and GPT-4 judged accuracy."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. All comparisons between methods (prompting vs model editing vs joint training) are based solely on comparing raw accuracy numbers without any statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The main accuracy results are presented as raw numbers without computed effect sizes. Speed comparisons (12x faster, half inference cost) provide context for efficiency claims, but the core performance comparisons lack formal effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification given for the 38k synthetic queries or the 80/20 train/test split. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. Results appear to be from single training runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are included: 'default' (no alignment, no prompting), 'prompting' (instruction-augmented queries), and 'joint training' (training on combined multi-domain data). These are defined in Section 5.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses several contemporary inference-time alignment methods (ITI, DeAl, SteerLM, CPO, InferAligner) in Section 2 but does not benchmark against any of them. Only prompting and joint training are compared. The related work argues these methods have different objectives, but no empirical comparison is provided."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper does not test alternative AV extraction methods, different loss functions, or component removal. Varying lambda explores parameter sensitivity but is not an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: 'preference accuracy' (log-probability based, Section 5.1) and 'GPT-4 judged generation accuracy' (Section 5.1). Both are reported in Tables 1-3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation in Section 3.3 assesses the quality of the synthetic training data, not the system's outputs. The schema requires humans to evaluate what the system produced. System evaluation uses only automated metrics (preference accuracy and GPT-4 judging)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 states: 'We utilize 80% of the generated data in each domain for training and 20% for testing. For the validation process, we allocated 3% of the data for training time validation.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by domain (Medical, Financial, Legal in Tables 1-4), by proficiency level (Exp, Gen, Avd), and by lambda value. Table 4 further shows cross-domain generalization effects."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 notes that only 22/27 multidomain combinations work. Section 6.3 discusses that negative lambda for safety produces 'inconsistent and mixed' scores and the model doesn't become fully unsafe. Section 6.2 discusses over-generalization as a failure mode."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: 5/27 multidomain combinations fail to achieve desired behavior (Section 6.2), negative lambda for safety doesn't achieve full unsafe behavior (Section 6.3), and over-generalization compromises multidomain precision (Section 6.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — dynamically adjustable behavior via AVs (Tables 1-3), inference cost reduction vs prompting (Section 6.1), AV transferability (Section 6.4, Figure 4), 12x speed advantage over retraining (Section 6.2) — are all supported by experimental results in the paper, though with caveats about accuracy trade-offs."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims ('adjusting λ controls proficiency level') are supported by controlled single-variable manipulation: lambda is varied while other factors are held constant, and the resulting behavior changes are observed (Tables 1-3, Figure 2). This controlled interventionist design is adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly bounds generalization: 'we tested our approach only on Mistral-7b, so validation with other open-source LLMs and SLMs is necessary' and 'our method works only for LLMs with the same architecture.' The title focuses on 'preference spectrum' rather than making universal claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. The paper does not discuss whether negative-lambda 'avoidance' behavior is genuine preference reversal or model degradation, whether cross-domain generalization reflects general capability changes rather than domain-specific effects, or whether the preference accuracy metric could create artifacts."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two complementary metrics — log-probability-based preference accuracy and GPT-4 judged generation accuracy on actual outputs — bridging the gap between internal model behavior and observable output quality. The claims match the granularity of these measurements (proficiency level control, not broader claims about domain expertise)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Mistral-7B-Instruct-v0.3 is fully specified (Section 5.3). However, 'Claude-3-Sonnet' for data generation lacks a snapshot date, and 'GPT-4' for evaluation judging is stated without any version identifier (Section 5.1, referencing Zheng et al., 2024)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides actual prompt text for CreatePersona and PersonaHub query generation methods, and Appendix C provides the full instructions for generating avoidance, generic, and expert responses, including the actual prompt text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "DPO training reports beta=0.1, batch size 4, one epoch (Section 5.3). However, learning rate is not stated, and temperature/top-p/sampling settings for Claude-3-Sonnet (data generation) and GPT-4 (evaluation) API calls are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a direct model editing approach (weight arithmetic) without agent pipelines, tools, or multi-step scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full data generation pipeline: persona sampling from PersonaHub (7,000 from 200,000), CreatePersona hierarchical generation, response generation at three levels, cleanup ('truncation and reformatting'), with final counts per domain (Table 6 in Appendix A)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations and Future Work' provides substantive discussion of four specific limitations: basic AV extraction approach, architecture constraint, single model testing, and grid search for multidomain alignment."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this study: 'we tested our approach only on Mistral-7b' (single model threat), 'our method works only for LLMs with the same architecture' (structural constraint), 'basic approach for obtaining alignment vectors' with concrete alternatives named (SVD, thresholding)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states boundaries: only tested on Mistral-7b, only works for same-architecture models, relied on grid search for multidomain. Section 1 also bounds scope to three domains and three proficiency levels."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The paper promises dataset availability 'through this link' but no functional URL is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: PersonaHub sampling (7,000 personas from 200,000), CreatePersona hierarchical generation (5 initial pairs, 3 randomization rounds), response generation with three-level instructions, and final counts per domain (Table 6)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Three human annotators are used for data quality evaluation (Section 3.3) but no information is provided about who they are, how they were recruited, their qualifications, or whether they have domain expertise (medical/financial/legal)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from personas → queries → three-level responses → train/test split is documented with counts at each stage. Figure 1 provides a visual overview. Table 6 gives final data amounts per domain and method."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present. The work was done at AWS AI Labs (most authors) and during an internship (first author), but no explicit funding statement is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Houston (first author, internship) and AWS AI Labs (remaining authors). The footnote states 'Work done during internship at AWS AI Labs.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "AWS AI Labs (an Amazon subsidiary) has a commercial interest in LLM alignment methods for their products. While the paper uses Mistral (not an AWS model), the alignment technique could benefit AWS's LLM offerings. No funding independence is discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation uses custom synthetic data generated specifically for this study via Claude-3-Sonnet, not pre-existing benchmarks. The model is fine-tuned via DPO and evaluated on held-out synthetic data that would not exist in Mistral's pre-training corpus."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates on custom synthetic data generated for this study, not on pre-existing benchmarks where pre-training contamination would be a concern."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no pre-existing benchmark is used for evaluation. All evaluation data is synthetically generated for this study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not conduct a human subjects study. The three annotators in Section 3.3 perform data quality validation, not participation in a study of human behavior."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Annotators assess synthetic data quality."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 reports: 'each evaluation takes around 60 seconds.' The abstract and Section 6.1 claim inference cost is reduced by half vs prompting. Section 6.2 computes total grid search time (155 hours) vs joint training time (1,944 hours)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 states: 'each job, along with its corresponding validation runs, takes about 72 hours of training on A100 GPUs.' Total comparisons: 1,944 GPU-hours for joint training vs 155 hours for grid search. Hardware specified as NVIDIA A100 GPUs (Section 5.3)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from single training runs with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. The paper says 'trained each model for one epoch' but does not state whether this was repeated across runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "For multidomain, the grid search space is described (21^3 = 9,261 evaluations, Section 6.2). However, no hyperparameter search budget is reported for the DPO training itself — beta=0.1 is stated without justifying how it was selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The lambda values shown in tables appear to be selected to demonstrate desired behaviors (Exp, Gen, Avd) without clear documentation of how these specific values were chosen. For DPO, beta=0.1 is stated without selection justification."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (prompting, joint training) themselves and do not acknowledge any potential bias from self-implementation. No independent evaluation is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While the paper compares total training time (1,944 vs 155 hours), it does not systematically report accuracy as a function of compute. Joint training achieves near-perfect accuracy at 12x more compute, but this trade-off is not formally analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its preference accuracy metric (based on log-probabilities) truly measures proficiency level control, or whether the synthetic three-level data represents meaningful real-world proficiency distinctions."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this model editing approach."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. While the synthetic test data was generated for this study, the paper does not address whether Mistral's pre-training data contains similar domain-specific Q&A patterns that could inflate baseline performance."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation setup provides all three response levels for log-probability comparison, and the format closely mirrors training data structure. Whether this format matching introduces leakage is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Train and test data are generated from the same persona distributions using the same LLM (Claude-3-Sonnet). Potential structural similarities between splits are not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No deduplication analysis, overlap checks, or contamination tests between train and test sets are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Alignment Vectors act as tunable knobs, enabling users to control proficiency levels across a spectrum by adjusting lambda",
       "evidence": "Tables 1-3 and Figure 2 show that varying lambda shifts the dominant behavior between Expert, Generic, and Avoidance across medical, financial, and legal domains. E.g., medical domain: lambda=0.5 achieves 95% expert preference accuracy, lambda=-0.7 shifts to generic (44%), lambda=-1.2 shifts to avoidance (84%).",
       "supported": "moderate"
     },
     {
       "claim": "The model editing approach reduces inference cost by half compared to prompt engineering",
       "evidence": "Section 6.1 states 'instruction-augmented prompts are nearly as long as the original queries, which doubles the inference cost.' This is a rough estimate based on prompt length, not direct inference cost measurement.",
       "supported": "weak"
     },
     {
       "claim": "Multidomain diverse preference is achievable via AV arithmetic, 12x faster than joint training",
       "evidence": "Section 6.2: grid search (155 hours) vs joint training (1,944 hours). Table 5 shows 22/27 multidomain combinations achieve desired dominant behavior. However, accuracy is substantially lower than joint training (e.g., 46% vs 100% for medical avoidance).",
       "supported": "moderate"
     },
     {
       "claim": "AVs are transferable across different fine-tuning stages of the same model",
       "evidence": "Section 6.4 and Figure 4 show that proficiency AVs can be applied to a safety-aligned version of Mistral-7B, achieving 0.81 change in medical expert accuracy while safety accuracy changes by only 0.11. Demonstrated on one model variant only.",
       "supported": "moderate"
     },
     {
       "claim": "Model editing does not lead to significant regression in general domain performance",
       "evidence": "Table 4 shows safety preference accuracy changes within ±12% and helpfulness within ±18% across lambda values. However, 18% change in helpfulness could be considered significant, and these claims are not statistically tested.",
       "supported": "weak"
     },
     {
       "claim": "Single-domain AVs generalize to other specialized domains",
       "evidence": "Table 4(a-c) shows that adding a domain-specific AV (e.g., medical) also shifts preference accuracy in other domains (financial, legal). For example, medical AV at lambda=0.5 achieves 100% expert accuracy in financial and legal domains.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results are presented as point estimates from apparently single runs. No confidence intervals, standard deviations, or variance across seeds are reported, making it impossible to assess result stability."
     },
     {
       "flag": "Large accuracy gap between method and joint training baseline",
       "detail": "Table 5 shows joint training achieves near-perfect accuracy (90-100%) while the AV method achieves 36-88% for dominant behaviors. The 12x speed advantage comes at a substantial accuracy cost that is acknowledged but not deeply analyzed."
     },
     {
       "flag": "Corporate affiliation evaluating alignment method",
       "detail": "Most authors are from AWS AI Labs, which has commercial interest in LLM alignment methods. No competing interests statement or funding disclosure is provided."
     },
     {
       "flag": "Entirely synthetic evaluation",
       "detail": "Both training and evaluation data are synthetically generated by Claude-3-Sonnet, with no validation on real-world domain queries from actual medical, legal, or financial professionals. The synthetic three-level proficiency distinction may not reflect real domain expertise gradations."
     },
     {
       "flag": "Small-scale human evaluation for data quality only",
       "detail": "Human evaluation involves only 3 annotators reviewing 30 queries for data quality (Section 3.3), not system output quality. The annotators' domain expertise is not described. No human evaluation of the aligned model's generated outputs is performed."
     },
     {
       "flag": "GPT-4 as judge without version specification",
       "detail": "GPT-4 is used as an evaluation judge (Section 5.1) without specifying which version, making the evaluation non-reproducible as GPT-4 behavior changes across versions."
     },
     {
       "flag": "Missing contemporary baselines",
       "detail": "Several related inference-time alignment methods are discussed (ITI, DeAl, SteerLM, CPO) but none are included as empirical baselines. Only prompting and joint training are compared against."
     }
   ],
   "cited_papers": [
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper for LLM alignment, the primary training-time alignment approach this work aims to complement."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2024,
       "relevance": "DPO algorithm used as the alignment training method in this paper to produce the aligned models from which AVs are derived."
     },
     {
       "title": "Editing models with task arithmetic",
       "authors": ["Gabriel Ilharco", "Marco Tulio Ribeiro", "Mitchell Wortsman"],
       "year": 2023,
       "relevance": "Core methodological inspiration — the paper's Alignment Vectors are directly based on the task arithmetic concept of subtracting base model weights from fine-tuned model weights."
     },
     {
       "title": "InferAligner: Inference-time alignment for harmlessness through cross-model guidance",
       "authors": ["Pengyu Wang", "Dong Zhang", "Linyang Li"],
       "year": 2024,
       "arxiv_id": "2401.11206",
       "relevance": "Inference-time alignment method using safety-related vectors to steer harmful outputs, closely related approach but focused on safety only."
     },
     {
       "title": "Inference-time intervention: Eliciting truthful answers from a language model",
       "authors": ["Kenneth Li", "Oam Patel", "Fernanda Viégas"],
       "year": 2024,
       "relevance": "Inference-time intervention method targeting truthfulness via activation shifting, related inference-time alignment technique."
     },
     {
       "title": "DeAl: Decoding-time alignment for large language models",
       "authors": ["James Y Huang", "Sailik Sengupta", "Daniele Bonadiman"],
       "year": 2024,
       "arxiv_id": "2402.06147",
       "relevance": "Decoding-time alignment via heuristic-guided search, an alternative inference-time alignment approach discussed as related work."
     },
     {
       "title": "SteerLM: Attribute conditioned SFT as an (user-steerable) alternative to RLHF",
       "authors": ["Yi Dong", "Zhilin Wang", "Makesh Narsimhan Sreedhar"],
       "year": 2023,
       "arxiv_id": "2310.05344",
       "relevance": "Controllable generation via attribute-conditioned training, related approach to user-steerable alignment but requires specific training format."
     },
     {
       "title": "Controllable preference optimization: Toward controllable multi-objective alignment",
       "authors": ["Yiju Guo", "Ganqu Cui", "Lifan Yuan"],
       "year": 2024,
       "arxiv_id": "2402.19085",
       "relevance": "Multi-objective controllable alignment via control tokens, related approach to multi-dimensional preference control."
     },
     {
       "title": "BeaverTails: Towards improved safety alignment of LLM via a human-preference dataset",
       "authors": ["Jiaming Ji", "Mickel Liu", "Josef Dai"],
       "year": 2024,
       "relevance": "PKU-SafeRLHF safety preference dataset used in this paper for safety alignment experiments in Sections 6.3-6.4."
     },
     {
       "title": "Decoding-time realignment of language models",
       "authors": ["Tianlin Liu", "Shangmin Guo", "Leonardo Bianco"],
       "year": 2024,
       "arxiv_id": "2402.02992",
       "relevance": "Regularization-based inference-time alignment control, closely related approach that the authors argue lacks fine-grained preference level flexibility."
     },
     {
       "title": "TIES-merging: Resolving interference when merging models",
       "authors": ["Prateek Yadav", "Derek Tam", "Leshem Choshen"],
       "year": 2024,
       "relevance": "Advanced model merging technique cited as future work direction for improving alignment vector extraction beyond simple subtraction."
     },
     {
       "title": "A general theoretical paradigm to understand learning from human preferences",
       "authors": ["Mohammad Gheshlaghi Azar", "Zhaohan Daniel Guo", "Bilal Piot"],
       "year": 2024,
       "relevance": "IPO loss function framework used in this paper's DPO training for alignment."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Inference-time preference control via weight arithmetic is a useful deployment technique, but requires DPO training first and is tested on only one model."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Extending task arithmetic from task performance to alignment preference is a natural extension of existing work, not a surprising finding."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Brief discussion of safety implications — negative lambda can partially reverse safety alignment, but model doesn't become fully unsafe."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to established methods or institutions."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or working data link is released. Dataset link is promised but not provided."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "AWS AI Labs is a recognized but not top-tier ML research brand in the alignment space."
     }
   }
 }