ai-research-survey

scan-v5.json (27190B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Detecting Proxy Gaming in RL and LLM Alignment via Evaluator Stress Tests",
     "authors": [
       "Ibne Farabi Shihab",
       "Sanjeda Akter",
       "Anuj Sharma"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.05619",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 78.4%/81.7% precision/recall (RL), 74.2%/78.6% (LLM), 8.3-point win-rate improvement, and 54.6% hacking reduction are all supported by detailed experimental tables (Tables 1, 7, 9, 11).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about mitigation effectiveness are supported by ablation studies and control experiments (Table 30) showing that extra compute alone yields only +2.1% vs. +8.3% for detector-triggered intervention.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly bounds generalization to '4 tasks and 2 model sizes' and acknowledges that 'real-world deployment would face additional challenges,' appropriately scoping the claims.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Control experiments (Table 30) rule out extra compute and filtering as explanations for win-rate gains; Appendix N discusses false positive patterns including beneficial exploration misclassified as hacking.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicitly about this distinction — judge scores (proxy) vs. human preferences (true objective) — and carefully measures both throughout; Table 2 shows divergence between judge score and human rating on case studies.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' is a dedicated section, not a sentence in the conclusion, discussing scope constraints on tasks, model sizes, and deployment conditions.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations specifically name '4 tasks and 2 model sizes,' fixed judges assumption, concept drift, multi-stakeholder objective conflicts, and adversarial adaptation over longer horizons — not generic disclaimers.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'mitigation results represent controlled experimental conditions' and that 'larger-scale validation across more diverse domains and model architectures would strengthen generalizability claims.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment appears anywhere in the paper; the Acknowledgments section only mentions AI writing tools, not any funding source.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors' affiliations with Iowa State University departments (CS and Civil/Construction/Environmental Engineering) are disclosed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No external funder is disclosed, so this criterion is not applicable.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms including 'proxy gaming,' 'exploitable sensitivity,' 'content sensitivity,' and the formal G(y) statistic are precisely defined with mathematical notation in Section 3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states its contribution as a unified framework (EST) for detecting proxy gaming in both RL and LLM alignment, with validated benchmarks for both domains.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 substantively engages with RLHF, DPO, reward hacking detection, and LLM-as-judge evaluation literature, explicitly positioning EST's contributions relative to prior approaches' scalability and principled-framework limitations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository URL is provided. The paper claims benchmark data release but does not mention releasing the detection framework implementation code.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims 'We release benchmarks for both domains' but provides no URL, DOI, or repository location where the 2,156 RL episodes or 1,200 LLM instances can be accessed.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only the GPU model (NVIDIA A6000 48GB VRAM) is mentioned; no requirements.txt, Dockerfile, or dependency specifications are provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided; the paper describes the methodology but not how to replicate the experiments end-to-end.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Tables 7 and 11 report values with ± notation; Table 11 explicitly states '95% CI across 5-fold cross-validation' (e.g., Precision: 0.784±0.027).",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Table 12 reports p-values for all factorial design effects (e.g., Objective Alignment p < 0.001); inter-rater reliability uses Cohen's κ and Fleiss' κ throughout.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 12 reports Cohen's d for all experimental factors (e.g., Objective Alignment: Cohen's d = 2.08); the paper explicitly notes these are unusually large due to custom environment design.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "Sample sizes (2,156 RL episodes, 1,200 LLM instances) are described but not formally justified with power analysis or sample size calculations.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Tables 7, 11, 18, and 29 consistently report ± standard deviation values alongside mean performance metrics.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Extensive baseline comparisons in Tables 3, 16, and 29 include length-only, format features, KL regularization, judge ensembling, LSTM-Autoencoder, One-Class SVM, Isolation Forest, reward model ensemble disagreement, and probe-based detection.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include contemporary methods like reward model ensemble disagreement (F1 0.687), probe-based detection, hardened judges, and established ML anomaly detection methods — not outdated or trivially weak.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Ablation studies in Tables 17 and 29 systematically remove individual detection components (EST, correlation tracking, reasoning validity, format perturbation, content perturbation) measuring each contribution.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Results reported across precision, recall, F1, AUC-ROC, early warning latency (checkpoints), computational overhead %, human win-rate, and judge-human correlation.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Human evaluation is central: 1,200 human-annotated LLM gaming instances with 3 raters achieving Fleiss' κ ≥ 0.78, and 2,156 expert-annotated RL episodes with Cohen's κ = 0.847.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.1 states 'strict train-validation-test splits, holding out entire task-model-judge combinations for testing'; RL uses environment-stratified splits with 5-fold cross-validation.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 11 provides per-category RL detection performance (6 hacking categories); Table 1 provides per-task, per-model-size, and per-judge breakdowns for all 32 LLM conditions.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix N provides dedicated error and boundary case analysis, manually examining 100 classification errors (50 false positives, 50 false negatives) with representative qualitative examples.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Adaptive evasion tests (Table 24) show precision dropping from 74.2% to 65.9% under white-box attacks; zero-shot cross-environment transfer shows 10-15 F1 point degradation (Table 32).",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "GPT-4 is used as a judge without specifying version (e.g., gpt-4-0314, gpt-4-turbo) or snapshot date; Llama-3-8B/70B are named but no model card versions are pinned.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "No actual prompts or system instructions are provided for LLM fine-tuning, judge evaluation, or the EST perturbation generation steps.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Key hyperparameters are reported: detection threshold τ=0.6, τspec=0.3, correlation threshold ∆ρ=0.5, contamination γ=0.1, window size W=50, format penalty 20%, mitigation threshold α=0.1.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The online detection pipeline is described in detail including Algorithm 1 for detector-triggered mitigation, the 6-detector ensemble structure with complexity analysis (Appendix D), and per-checkpoint monitoring protocol (Table 14).",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Transformation validity thresholds (cosine similarity >0.85, NLI entailment >0.7), number of perturbations (5 per type), token-count control (±5%), and semantic validity audit procedures are documented with specific thresholds.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims to release benchmarks but provides no URL or repository link where the raw data can be accessed for independent verification.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection is described: 3 human annotators with consensus ≥2/3 for LLM instances; RL expert annotation with κ = 0.847 across 15 environments, 10 random seeds, and environment-stratified splits.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": false,
           "justification": "Annotators are mentioned ('3 human annotators,' '3 human raters') but no recruitment procedure, qualification criteria, or platform is described; the ethics statement mentions only informed consent and compensation.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline from LLM fine-tuning checkpoints → output sampling → perturbation generation → validity audit → detection scoring is described with algorithmic detail in Section 3 and Appendix D.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training cutoffs for Llama-3-8B, Llama-3-70B, and GPT-4 are not stated; this matters since tasks include TL;DR summarization (Reddit data) that could overlap with pretraining corpora.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of potential overlap between TL;DR summarization or other evaluation task data and the pretraining data of Llama-3 or GPT-4.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "LLM tasks include TL;DR summarization (Reddit-based) which could be present in Llama-3's pretraining data, affecting gaming behavior; this potential contamination is not addressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects experiment requiring pre-registration; annotation workers are used for labeling, not as research participants in an experimental study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "Human annotation is conducted (with informed consent and compensation noted in ethics statement), but this is annotation labor rather than a human subjects experiment requiring IRB review.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participant subjects; annotation workers are service providers and their demographics are not applicable to the study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects study with inclusion/exclusion criteria for research participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects randomized experiment.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects blinded experiment.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participant attrition applicable.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Computational overhead is reported as 2.1% for LLM and 4.2% for RL; Table 31 provides absolute GPU-hours for each mitigation technique (e.g., 0.47 GPU-hrs for combined approach on NVIDIA A6000).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Per-technique overhead is reported but the total compute budget for the full experimental suite (32 LLM conditions × fine-tuning runs, 15 RL environments × 10 seeds) is not stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "EST achieves 78.4% precision and 81.7% recall for RL reward hacking detection across 15 environments and 5 algorithms",
       "evidence": "Table 11 reports these figures on 2,156 expert-annotated episodes with 5-fold cross-validation and 95% confidence intervals (0.784±0.027, 0.817±0.023); Cohen's κ = 0.847 inter-rater agreement",
       "supported": "strong"
     },
     {
       "claim": "EST achieves 74.2% precision and 78.6% recall for LLM evaluator gaming detection across 32 experimental conditions",
       "evidence": "Tables 1 and 7 report these figures on 1,200 human-annotated instances with Fleiss' κ ≥ 0.78 inter-rater agreement across 4 tasks",
       "supported": "strong"
     },
     {
       "claim": "Closed-loop EST-triggered mitigation improves human win-rate by 8.3 points (52.1% → 60.4%) for LLM fine-tuning",
       "evidence": "Table 9 shows win-rate improvement; Table 30 control experiments show extra compute alone yields only +2.1%, ruling out compute as confound",
       "supported": "strong"
     },
     {
       "claim": "Closed-loop mitigation reduces RL reward hacking by 54.6% with 9.1% performance impact",
       "evidence": "Table 31 reports combined approach achieves 54.6% hacking reduction; Pareto frontier analysis in Figure 10 shows this is the best available trade-off",
       "supported": "moderate"
     },
     {
       "claim": "Proxy-true correlation tracking transfers directly between RL and LLM domains without modification",
       "evidence": "Table 8 defines direct transfer as ≥90% in-domain performance; correlation tracking achieves AUC 0.821 (RL) and 0.798 (LLM) without modification",
       "supported": "moderate"
     },
     {
       "claim": "EST provides early warning with median lead time of 3 checkpoints before human-noticeable quality decline",
       "evidence": "Figure 2 and Table 7 report 3.0±0.4 checkpoint lead time, defined as checkpoints between detection trigger and human win-rate dropping below 0.50",
       "supported": "moderate"
     },
     {
       "claim": "EST outperforms all baselines including reward model ensemble disagreement (F1 0.734 vs 0.687)",
       "evidence": "Tables 3 and 16 compare against 9+ baselines; EST achieves F1 0.734 vs. next-best standalone of 0.694 (correlation tracking) and 0.700 (hardened judge)",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "EST detects proxy gaming through invariance-based stress tests that separate exploitable format sensitivity from content-driven improvements, achieving 78.4%/81.7% precision/recall for RL hacking and 74.2%/78.6% for LLM evaluator gaming across 32 experimental conditions. Closed-loop mitigation triggered by EST improves human win-rates by 8.3 points and reduces RL hacking by 54.6%, with control experiments ruling out compute as the explanation. Cross-domain analysis reveals that correlation tracking and ensemble voting transfer directly between RL and LLM domains while perturbation design requires adaptation. The framework operates online with low overhead (2.1% LLM, 4.2% RL) and provides 3-checkpoint early warning before human-noticeable quality decline, enabling proactive intervention during fine-tuning.",
   "red_flags": [
     {
       "flag": "No code or data URL provided",
       "detail": "Despite claiming to release benchmarks (2,156 RL episodes, 1,200 LLM instances), no repository URL, DOI, or access location is provided anywhere in the paper."
     },
     {
       "flag": "GPT-4 version unspecified",
       "detail": "GPT-4 is used as a judge evaluator throughout without specifying version (gpt-4-0314, gpt-4-turbo, etc.) or snapshot date, making exact replication impossible."
     },
     {
       "flag": "Circular RL ground truth for most episodes",
       "detail": "For 13,091 of 15,247 RL episodes (86%), ground truth is established via detector consensus (3+ of 6 detectors agree) rather than human annotation — the authors acknowledge this circularity but use these episodes for pattern analysis and prevalence claims."
     },
     {
       "flag": "Custom environments maximize contrast",
       "detail": "The paper acknowledges 'unusually large effect sizes reflect our custom environments designed to maximize experimental contrast'; real-world effect sizes are acknowledged to be smaller (Cohen's d 0.8-1.2 vs. reported 1.24-2.08)."
     },
     {
       "flag": "Self-citation for key supporting claims",
       "detail": "Two key supporting citations (Shihab et al., 2025a on entropy regularization and Shihab et al., 2025b on reward function structure) are by the same authors and are arXiv preprints, not peer-reviewed work."
     },
     {
       "flag": "No prompts provided for LLM experiments",
       "detail": "The paper does not provide actual prompts used for LLM fine-tuning, judge evaluation rubrics, or the perturbation generation process, significantly limiting reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "Defining and Characterizing Reward Hacking (Skalse et al., 2022)",
       "relevance": "Provides the formal definition of proxy gaming used throughout the paper and theoretical grounding for EST's unhackability criterion"
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback (Ouyang et al., 2022)",
       "relevance": "Foundational RLHF work that introduces the training paradigm EST is designed to monitor for gaming"
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model (Rafailov et al., 2023)",
       "relevance": "One of the two training methods (DPO) evaluated in the LLM experiments"
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena (Zheng et al., 2023)",
       "relevance": "Establishes LLM-as-judge evaluation paradigm that the paper identifies as vulnerable to evaluator gaming"
     },
     {
       "title": "Scaling Laws for Reward Model Overoptimization (Gao et al., 2023)",
       "relevance": "Directly relevant to proxy-true divergence measurement; shows reward model scores diverge from human preferences with overoptimization"
     },
     {
       "title": "Specification Gaming: The Flip Side of AI Ingenuity (Krakovna et al., 2020)",
       "relevance": "Provides taxonomy of reward hacking behaviors that grounds the RL component of EST and defines specification gaming"
     },
     {
       "title": "Concrete Problems in AI Safety (Amodei et al., 2016)",
       "relevance": "Foundational safety paper establishing reward hacking as a core alignment challenge that motivates the EST framework"
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models (Wei et al., 2022)",
       "relevance": "Related to the reasoning validity detector component of EST that checks for valid reasoning chains during fine-tuning"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly applicable to anyone training LLMs with RLHF or LLM-as-judge evaluation pipelines — an increasingly important and widespread concern in AI development"
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The existence of proxy gaming is well-established; the invariance-based unified framework across domains is novel but not surprising in its conclusions"
     },
     "fear_safety": {
       "score": 2,
       "justification": "Addresses AI alignment safety concerns about reward hacking and evaluator gaming, with concrete detection and mitigation methods for recognized risks in deployed RLHF systems"
     },
     "drama_conflict": {
       "score": 1,
       "justification": "No major controversy; the paper confirms known problems and proposes systematic solutions rather than challenging dominant narratives"
     },
     "demo_ability": {
       "score": 1,
       "justification": "Claims benchmarks are released but provides no URL; cannot easily reproduce or demo the framework without code release"
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Iowa State University authors without prior high-profile publications; no famous lab or product association"
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "38720557",
         "title": "ReLoRA: High-Rank Training Through Low-Rank Updates",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38720557"
       },
       {
         "hn_id": "41035192",
         "title": "The Limitations of Compute Thresholds as a Governance Strategy",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41035192"
       }
     ],
     "top_points": 3,
     "total_points": 4,
     "total_comments": 0
   }
 }