ai-research-survey

scan.json (28926B)

---

 {
   "paper": {
     "title": "Spread Preference Annotation: Direct Preference Judgment for Efficient LLM Alignment",
     "authors": [
       "Dongyoung Kim",
       "Kimin Lee",
       "Jinwoo Shin",
       "Jaehyung Kim"
     ],
     "year": 2024,
     "venue": "International Conference on Learning Representations (ICLR 2025)",
     "arxiv_id": "2406.04412"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SPA achieves a 21.13% win rate on AlpacaEval 2.0 using only 3.3% of ground-truth preference labels from UltraFeedback, outperforming both standard DPO with the same data (7.68%) and Zephyr-7b-β trained on 100% of labels (10.03%). The direct preference judgment from model logits outperforms external reward models (PairRM) and LLM-as-judge baselines, and the self-refinement with decoupled noise detection contributes meaningful gains. The framework generalizes across Mistral-7B, Phi-2, LLaMA-3, and Phi-3 models, and can function even without seed preference data.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 provides a GitHub URL: https://github.com/kingdy2002/SPA. The Reproducibility Statement also mentions code and checkpoint release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: UltraFeedback ('argilla/ultrafeedback-binarized-preferences-cleaned', footnote 14) and UltraChat (footnote 11). All training data is from public HuggingFace datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B.4 mentions '4 A6000 GPUs' but no requirements.txt, Dockerfile, conda environment file, or detailed library version specifications are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The Reproducibility Statement says 'we will release our codes and the checkpoints' but does not include specific commands or a reproducing-results guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All main results in Tables 1-3 and 5-7 report only point estimates (e.g., '21.13%', '15.39%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SPA 'outperforms' and 'significantly improves' baselines based solely on comparing point estimates. No statistical significance tests (p-values, t-tests, bootstrap tests) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: e.g., 'win rate improved from 7.68% to 21.13%' (Table 1), 'length-control win rate from 7.58% to 15.39%', and improvement from 4.72% to 21.13% vs GPT-4."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of 2K seed samples (3.3%), or the 8K/20K/30K iteration split sizes. No power analysis or discussion of whether these sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 1-2) appear to be single-run numbers with no variance or standard deviation reported. Table 4 reports variance across 3 different seed data samplings, but this is a sensitivity analysis, not multi-run variance for the primary results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares against Iterative DPO with PairRM (external reward model) and Iterative DPO with LLM-as-judge. Table 1 compares against DPO and Zephyr-7b-β."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "PairRM (Jiang et al., 2023b) and Iterative DPO (Snorkel, 2024; Xu et al., 2023) are recent and competitive baselines. PairRM is described as showing 'state-of-art performance in AlpacaEval2.0 benchmark.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 presents ablations removing self-refinement (SR) and decoupled noise detection (DND), showing their individual contributions. Table 7 ablates the reference model choice."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on AlpacaEval 2.0 (both raw win rate and length-controlled win rate) and MT-Bench (average score across 8 categories), providing multiple complementary metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated: AlpacaEval 2.0 uses GPT-4 as judge, MT-Bench uses GPT-4 scoring. No human evaluation of outputs is conducted, despite the paper claiming to improve 'alignment with human preferences.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "AlpacaEval 2.0 (805 instructions) and MT-Bench are standard held-out evaluation benchmarks entirely separate from the UltraFeedback training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5a provides MT-Bench task-wise evaluation across 8 categories (Writing, Roleplay, Reasoning, Math, Coding, Extraction, STEM, Humanities)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section C and Figure 5a discuss that SPA shows 'almost no gain in Coding and degradation in Math.' Appendix A.1 discusses the length bias limitation. Table 10 quantifies the length increase."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports degradation in Math tasks and no gain in Coding (Figure 5a). It also reports that self-refinement without decoupled noise detection yields only marginal improvement (Table 6: 19.91% to 19.94%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'superior alignment performance on AlpacaEval 2.0 with only 3.3% of the ground-truth preference labels' which is supported by Table 1 (21.13% vs 10.03% for Zephyr-7b-β with 100% labels)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('SPA improves alignment') and supports them with controlled ablation studies (Table 6) that isolate the effect of each component (data expansion, self-refinement, decoupled noise detection) through single-variable removal."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the framework is 'highly competitive and practical for real-world applications' but all evaluation is on automated benchmarks (AlpacaEval, MT-Bench) with GPT-4 as judge. The title 'Efficient LLM Alignment' is broad relative to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses distribution shift as an alternative explanation for why PairRM underperforms. Appendix D specifically addresses the length bias confound with Table 10 showing length-regularized variants, and LC win rate is reported throughout to account for response length."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures AlpacaEval 2.0 win rate (GPT-4 as judge) and MT-Bench scores and frames these as 'alignment with human preferences.' The gap between automated evaluation by GPT-4 and actual human preference is not discussed — the paper uses 'AlpacaEval 2.0 approximately evaluates human preference' without deeper analysis."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: 'Mistral-7b-0.1' with HuggingFace path 'alignment-handbook/zephyr-7b-sft-full' (footnote 6), 'lole25/phi-2-sft-ultrachat-full' (footnote 8), and HuggingFace links for LLaMA-3-8B-Instruct and Phi-3-medium-4k-instruct."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Listing 1 provides the actual prompt template used for preference judgment, and Listing 2 provides a complete evaluation instruction example with filled values, both in Appendix B.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports comprehensive hyperparameters: β=0.1, batch size 32, learning rate 5×10⁻⁷, α=0.1, K=10, λ values (1/2, 1/4, 1/8), temperature 0.7, optimizer (AdamW), cosine scheduler with 10% warmup."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. SPA is a training framework using DPO with self-generated preference data, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 documents the data pipeline: 2K seed samples (3.3% of 60K) with gold labels, remaining split into 8K/20K/30K for iterations 1/2/3 with only prompts. Appendix B.4 documents response generation with max 300 token limit and SMA-based confidence measurement."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 'Limitation and Future Work' provides a dedicated discussion of limitations, specifically addressing the length bias tendency and suggesting future directions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 discusses a specific threat: SPA tends to increase response length, and acknowledges 'depending on the user, this behavior could be dispreferred.' Figure 5a identifies specific task categories (Coding, Math) where SPA fails to improve."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. It does not bound generalizations to specific conditions beyond noting length bias and coding/math limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While UltraFeedback is publicly available, the specific 2K seed samples, generated responses, self-annotated preference labels, and noise identification data are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes data sourcing from UltraFeedback ('argilla/ultrafeedback-binarized-preferences-cleaned'), the seed data sampling procedure, and response generation parameters (temperature 0.7, 2 responses per prompt)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public datasets (UltraFeedback, UltraChat)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: UltraFeedback → seed/iteration split → response generation from current model → preference labeling via logits → noise detection → DPO training. Each step is described in Sections 4 and 5.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses funding from IITP grants funded by the Korean government (MSIT), with specific grant numbers (No.2021-0-02068 and RS-2022-II220959)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: KAIST and Yonsei University. The paper does not evaluate any commercial product from these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Korean government agencies (IITP/MSIT), which have no financial stake in the specific outcome of this LLM alignment research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for Mistral-7b-0.1v or any other base model used. No discussion of when the models' pre-training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AlpacaEval 2.0 instructions or MT-Bench questions could have appeared in the Mistral base model's pre-training data or in the UltraFeedback/UltraChat fine-tuning data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "AlpacaEval and MT-Bench were publicly available before Mistral's training. No contamination analysis or discussion of whether evaluation data may have leaked into pre-training corpora."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.4 reports wall-clock timing: 'generating responses for 10K prompts takes approximately 1 to 2 hour, and preference judging for generated responses also takes about 1 to 2 hour. For training... it takes about 1 to 2 hours per epoch.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.4 states: '4 A6000 GPUs' and 'the total time required to complete response generation, preference judgment, and one epoch of training was between 5 to 6 hours per 10K prompt.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports results across 3 different random samplings of the initial seed preference dataset, showing variance. This addresses sensitivity to the seed data, though not random seed sensitivity in training."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The main results (Tables 1-2) do not state how many experimental runs produced them. Table 4 uses 3 seed data samplings for sensitivity analysis, but the primary results lack this information."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported (β=0.1, α=0.1, K=10, etc.) but no search budget, number of configurations tried, or search method is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports fixed hyperparameter values without explaining how they were selected or whether they were tuned on a validation set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement baselines (Iterative DPO with PairRM, LLM-as-judge) and compare against their own method without acknowledging that author-implemented baselines may systematically underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SPA requires 3 iterations of response generation plus training, consuming substantially more compute than single-stage DPO baselines. This compute difference is not discussed as a potential confound for the observed improvements."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses AlpacaEval 2.0 (GPT-4 as judge) and MT-Bench as proxies for 'alignment with human preferences' without questioning whether these automated metrics actually capture the claimed construct."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the comparisons. All methods differ only in training procedure."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AlpacaEval 2.0 or MT-Bench tasks existed before the Mistral model's training data collection period."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any evaluation setup provides information that would not be available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether UltraFeedback training prompts share structural similarities with AlpacaEval 2.0 or MT-Bench evaluation prompts."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied (no canary strings, membership inference, n-gram overlap, or decontamination pipeline)."
       }
     }
   },
   "claims": [
     {
       "claim": "SPA achieves 21.13% win rate on AlpacaEval 2.0 vs GPT-4 using only 3.3% of ground-truth preference labels, compared to 7.68% for standard DPO with the same data.",
       "evidence": "Table 1 reports SPA at 21.13% win rate and 15.39% LC win rate vs DPO at 7.68% and 9.03% respectively, both using 3.3% gold labels from UltraFeedback.",
       "supported": "strong"
     },
     {
       "claim": "SPA with 3.3% labeled data outperforms Zephyr-7b-β trained on 100% of UltraFeedback labels on AlpacaEval 2.0.",
       "evidence": "Table 1 shows SPA (21.13% WR, 15.39% LC WR) vs Zephyr-7b-β (10.03% WR, 11.75% LC WR). Both use the same base model and SFT data.",
       "supported": "strong"
     },
     {
       "claim": "Direct preference judgment from model logits outperforms external reward models (PairRM) and LLM-as-judge for iterative preference learning.",
       "evidence": "Table 2 shows SPA at 21.13%/15.39% vs PairRM at 9.46%/11.87% and LLM-as-judge at 9.18%/9.28% on AlpacaEval 2.0. Figure 3 shows the gap widens across iterations.",
       "supported": "moderate"
     },
     {
       "claim": "SPA can improve alignment even without seed preference data.",
       "evidence": "Figure 4 shows Mistral-7B-instruct-v0.1 improving from 6.31% to 9.79% win rate and from 10.14% to 11.59% LC win rate when applying SPA without seed data.",
       "supported": "weak"
     },
     {
       "claim": "Self-refinement with decoupled noise detection significantly improves SPA's performance.",
       "evidence": "Table 6 ablation: data expansion alone achieves 14.41% LC WR; adding SR gives 14.7%; adding both SR and DND gives 15.39%. Win rate improvement is more substantial (19.91% → 21.13%).",
       "supported": "moderate"
     },
     {
       "claim": "SPA generalizes across different LLM architectures and sizes.",
       "evidence": "Table 5 shows improvements across Phi-2-2.7B (5.67%→9.43% WR), LLaMA-3-8B (25.39%→34.84%), and Phi-3-14B (22.12%→24.14%).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Length inflation confound",
       "detail": "Table 10 shows SPA-trained models generate substantially longer responses (2749 chars at iter 2 vs 901 for SFT). The raw win rate gains (4.72%→21.13%) are much larger than LC win rate gains (7.58%→15.39%), suggesting response length drives a significant portion of the improvement. While LC win rate is reported, the paper leads with the more impressive raw numbers."
     },
     {
       "flag": "No significance tests for any comparison",
       "detail": "All claims of 'significantly outperforms' and 'superior performance' rest on comparing point estimates without statistical tests. The largest claimed difference (SPA 21.13% vs PairRM 9.46%) could potentially be due to random variation, since no error bars or significance tests are reported for the main results."
     },
     {
       "flag": "Uncontrolled compute budget",
       "detail": "SPA uses 3 iterations of response generation + preference labeling + DPO training (15-18 GPU-hours on 4 A6000s), while the DPO baseline uses a single training phase. Performance improvements may partly reflect the additional compute rather than the method's novelty."
     },
     {
       "flag": "All evaluation relies on GPT-4 as judge",
       "detail": "For a paper about improving 'alignment with human preferences,' all evaluation uses GPT-4 as the preference judge. No human evaluation is conducted. GPT-4 judge preferences may not align well with actual human preferences, particularly given the known length bias of LLM judges."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Christopher D Manning", "Stefano Ermon", "Chelsea Finn"],
       "year": 2023,
       "relevance": "Core method that SPA builds upon — directly optimizing LLM preferences without separate reward modeling."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper establishing the preference learning framework for LLM alignment."
     },
     {
       "title": "Self-rewarding language models",
       "authors": ["Weizhe Yuan", "Richard Yuanzhe Pang", "Kyunghyun Cho"],
       "year": 2024,
       "arxiv_id": "2401.10020",
       "relevance": "Related approach using LLM-as-judge for self-improvement in alignment, which SPA's direct preference judgment aims to outperform."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Influential approach to AI alignment using AI feedback instead of human feedback."
     },
     {
       "title": "Zephyr: Direct Distillation of LM Alignment",
       "authors": ["Lewis Tunstall", "Edward Beeching", "Nathan Lambert"],
       "year": 2023,
       "arxiv_id": "2310.16944",
       "relevance": "Key baseline using the same model architecture and SFT recipe, trained on full UltraFeedback data."
     },
     {
       "title": "UltraFeedback: Boosting Language Models with High-Quality Feedback",
       "authors": ["Ganqu Cui", "Lifan Yuan", "Ning Ding"],
       "year": 2023,
       "arxiv_id": "2310.01377",
       "relevance": "Source of the preference dataset used as training data in all SPA experiments."
     },
     {
       "title": "KTO: Model Alignment as Prospect Theoretic Optimization",
       "authors": ["Kawin Ethayarajh", "Winnie Xu", "Niklas Muennighoff", "Dan Jurafsky", "Douwe Kiela"],
       "year": 2024,
       "arxiv_id": "2402.01306",
       "relevance": "Alternative preference learning algorithm that removes reliance on pairwise labels, relevant to efficient alignment."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Defines MT-Bench evaluation used in this paper and establishes the LLM-as-judge paradigm that SPA compares against."
     },
     {
       "title": "SimPO: Simple Preference Optimization with a Reference-Free Reward",
       "authors": ["Yu Meng", "Mengzhou Xia", "Danqi Chen"],
       "year": 2024,
       "relevance": "Reference-free preference optimization method relevant to efficient LLM alignment approaches."
     },
     {
       "title": "Bootstrapping Language Models with DPO Implicit Rewards",
       "authors": ["Changyu Chen", "Zichen Liu", "Chao Du"],
       "year": 2025,
       "relevance": "Concurrent work proposing similar iterative self-improvement idea with focus on length regularization."
     },
     {
       "title": "Self-play Preference Optimization for Language Model Alignment",
       "authors": ["Yue Wu", "Zhiqing Sun", "Huizhuo Yuan"],
       "year": 2024,
       "arxiv_id": "2405.00675",
       "relevance": "Related self-play approach for preference optimization in LLM alignment."
     },
     {
       "title": "Deep reinforcement learning from human preferences",
       "authors": ["Paul F Christiano", "Jan Leike", "Tom Brown"],
       "year": 2017,
       "relevance": "Foundational RLHF paper establishing learning from human preferences framework."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Code released and method requires only 'a few lines of additional code to the original DPO codebase,' making it practically adoptable for teams doing LLM alignment with limited annotation budget."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Shows 3.3% of labeled data can outperform 100% with the right framework, which is somewhat surprising but follows a known pattern of iterative self-improvement methods."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper is about efficient alignment, not attacks or vulnerabilities."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with existing claims or institutions."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is on GitHub but requires significant GPU resources (4 A6000s) and multi-hour training to reproduce; not a quick demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "KAIST is well-known in ML research and ICLR is a top venue, but the authors and lab are not household names."
     }
   }
 }