ai-research-survey

scan.json (36057B)

---

 {
   "paper": {
     "title": "Detecting Proxy Gaming in RL and LLM Alignment via Evaluator Stress Tests",
     "authors": [
       "Ibne Farabi Shihab",
       "Sanjeda Akter",
       "Anuj Sharma"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.05619"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The Evaluator Stress Test (EST) detects proxy gaming in both RL (78.4% precision, 81.7% recall on 2,156 expert-annotated episodes) and LLM alignment (74.2% precision, 78.6% recall on 1,200 human-annotated instances) by measuring whether score improvements are driven by exploitable features or genuine content improvements. Closed-loop mitigation using detector-triggered interventions improves human win-rate by 8.3 points in LLM alignment and reduces hacking by 54.6% in RL. Cross-domain analysis shows proxy-true correlation tracking transfers directly between RL and LLM domains while perturbation design requires domain adaptation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We release benchmarks for both domains: 2,156 RL episodes and 1,200 LLM gaming instances' but provides no repository URL, download link, or archive. 'All benchmarks will be released for standardized evaluation' (Appendix C.7) is a future promise, not an actual release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as code — the paper promises to release benchmark data (2,156 RL episodes and 1,200 LLM instances) but no download URL or archive link is provided anywhere in the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions NVIDIA A6000 (48GB VRAM) in Appendix N and lists model names (Llama-3-8B, Llama-3-70B, GPT-4) but provides no requirements.txt, Dockerfile, library versions, or environment setup details sufficient to recreate the computational environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The algorithmic details (Appendix D, Algorithm 1) describe the detection framework conceptually but do not constitute executable reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 11 reports ±95% CI across 5-fold cross-validation for all RL detection categories. Table 7 reports ±std for overall LLM detection metrics. Table 18 reports mean ± std across random seeds for both domains."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 reports p-values for the factorial design experiment (e.g., reward density p < 0.001, objective alignment p < 0.001, reward complexity p = 0.003). Interaction effects also tested with p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 reports Cohen's d for each factor in the factorial design (e.g., objective alignment d = 2.08, reward density d = 1.24, reward complexity d = 0.67). The paper acknowledges these are 'unusually large effect sizes' due to custom environment design."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or explicit justification for why 2,156 RL episodes or 1,200 LLM instances were annotated. The paper states these sample sizes but does not justify why these numbers are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 11, 18, and 29 report standard deviations across experimental runs and cross-validation folds. Table 7 reports ±std for all main metrics. The paper uses 10 random seeds per RL configuration and 5-fold CV."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3/16 includes 10+ baselines for LLM detection (length-only, format-feature, style embedding, KL regularization, judge ensembling, hardened judges, probe-based detection, reward model ensemble disagreement, etc.). Table 29 includes LSTM-Autoencoder, One-Class SVM, Isolation Forest, and BC Divergence baselines for RL."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods: reward model ensemble disagreement, probe-based detection, hardened judges, and self-consistency (CoT). These represent the current state of detection approaches for evaluator gaming."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 17 ablates individual LLM detection components (EST, correlation tracking, reasoning validity, format perturbation, content perturbation). Table 29 ablates each RL detector. Table 30 ablates mitigation intervention components. Removing EST causes the largest F1 drop (0.734 to 0.694)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports precision, recall, F1-score, AUC-ROC, early warning latency, computational overhead, human win-rate, judge-human correlation, and false positive rate across multiple experimental conditions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is central: 1,200 response pairs annotated by 3 human raters (Fleiss' κ ≥ 0.78) for LLM gaming, 2,156 episodes annotated by 3 RL experts (Cohen's κ = 0.847). Human win-rates are used to evaluate mitigation effectiveness. Human validation of 100 transformation samples achieves 87% equivalence agreement."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'We use strict train-validation-test splits, holding out entire task-model-judge combinations for testing.' Appendix C.3.1 describes 'environment-stratified splits to prevent data leakage, with entire environment-algorithm combinations held out for testing.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks results by task, model size, and judge type. Table 11 breaks RL results by hacking category. Table 15 presents all 32 experimental conditions individually. Table 6 shows per-task, per-judge detection performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix N provides detailed error analysis of 100 classification errors (50 false positives, 50 false negatives) with three patterns for each. Boundary cases (18.3% of detector-labeled episodes) are analyzed in Appendix N.3. Adaptive evasion results in Appendix I show detection degradation under adversarial conditions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study shows components that hurt performance when removed. White-box evasion reduces precision from 74.2% to 65.9% (Table 24). Cross-environment transfer degrades by 10-15 F1 points (Table 32). The paper reports that data filtering has minimal impact on win-rate (+7.9% vs +8.3%), suggesting it's the least useful intervention."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 78.4% precision/81.7% recall for RL (Table 11), 74.2% precision/78.6% recall for LLM (Tables 1, 7), 8.3 point win-rate improvement (Table 9), 54.6% hacking reduction (Table 31), median 3-checkpoint early warning (Table 7), and overhead percentages match reported values."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are supported by controlled designs: ablation studies (Tables 17, 29, 30) use single-variable manipulation, the 2×2×2 factorial design (Table 12) varies individual factors, and control experiments (Table 30) confirm gains are from detector-guided interventions rather than extra compute."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Limitations) explicitly states the study is 'limited to 4 tasks and 2 model sizes' and notes the threat model 'assumes fixed judges during training.' The paper acknowledges custom testbed effect sizes would be 'more modest' in real-world deployments (Appendix C.5)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses length as a confound (Table 23 shows ρ=0.18 correlation, Figure 5 AUROC 0.734 vs 0.534 for length-only), controls for extra compute as an alternative explanation for mitigation gains (Table 30 control experiments), and discusses audit quality as a source of false positives (Appendix H)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution is explicitly about the proxy-true distinction. Section 3 formally defines proxy gaming as 'systematic increases in E[J(y)] that do not correspond to increases in E[H(y)]' and the entire EST framework is designed to distinguish proxy-exploitative gains from content-driven improvements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4' as a judge without specifying a version or snapshot date (e.g., gpt-4-0613). 'Llama-3-8B' and 'Llama-3-70B' are named but without specific checkpoint or release identifiers. No model version dates are provided anywhere."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided for any component: judge evaluation prompts, format perturbation generation prompts, content perturbation generation prompts, or model fine-tuning prompts are all described conceptually but the actual text used is never shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Detection framework hyperparameters are reported (Table 19: τspec=0.3, Δρ=0.5, γ=0.1, W=50). However, LLM fine-tuning hyperparameters (learning rate, batch size, epochs) and judge API settings (temperature, top-p, max tokens) are not reported, which significantly affect output quality."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The detection framework operates as a monitoring pipeline applied to standard fine-tuning runs, not as an agentic system with tools, memory, or retry logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3.1 describes the validation protocol with expert-validated (n=2,156) and detector-consensus (n=13,091) sets. Data splits are documented (environment-stratified, task-model-judge holdouts). Annotation procedures are described (3 raters, consensus ≥2/3, inter-rater agreement metrics). Transformation validity audits are detailed in Appendix H."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is titled 'Limitations' and provides substantive discussion covering task/model scope limitations, threat model assumptions (fixed judges), and real-world deployment challenges (concept drift, multi-stakeholder conflicts, adversarial adaptation)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: 'limited to 4 tasks and 2 model sizes,' 'fixed judges during training; adaptive evaluators that update during fine-tuning may require modified detection approaches,' and 'concept drift, multi-stakeholder objective conflicts, and adversarial adaptation over longer training horizons.' The paper also acknowledges custom testbeds inflate effect sizes."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly bounds scope: 'larger-scale validation across more diverse domains and model architectures would strengthen generalizability claims.' The paper states mitigation results 'represent controlled experimental conditions' and real-world deployment faces additional challenges. Custom testbed effect sizes are noted as unrepresentative of production settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Despite claiming to release benchmarks ('We release benchmarks for both domains'), no download URL, repository link, or data archive is provided. Raw episode data, annotations, and model outputs are not available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes RL data collection: 15 environments, 5 algorithms, 10 random seeds, 15,247 total episodes, 2,156 expert-annotated. Section 5.1 describes LLM data: 4 tasks, 2 model sizes, 2 training methods, 2 judges, 10 checkpoints, 1,200 annotated instances. Annotation procedures include consensus rules and agreement metrics."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "RL annotators are described only as 'three independent RL experts with extensive experience in reward hacking identification.' LLM annotators are described as '3 human annotators' with no further characterization. How annotators were recruited, their qualifications beyond vague descriptions, and potential selection bias are not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3.1 describes the pipeline from raw episodes (15,247) to expert-annotated subset (2,156) to train/test splits (environment-stratified). The LLM pipeline from model fine-tuning through checkpoint sampling to human annotation is described in Section 5.1. Transformation validity audits are documented with pass rates (Table 20)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The acknowledgments section only mentions 'the use of AI tools for improving writing flow and fixing grammatical errors.' No grants, sponsors, or funding agencies are listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Iowa State University departments (Computer Science, and Civil, Construction & Environmental Engineering). The authors are not affiliated with any company whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot determine funder independence because no funding source is disclosed. The absence of funding disclosure means independence cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The ethics statement addresses human annotation but does not include a competing interests declaration."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a detection framework for proxy gaming, not pre-trained model capability on benchmarks. The core evaluation is whether EST detects gaming behavior, not whether Llama-3 or GPT-4 achieve specific benchmark scores."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper tests a defense/detection framework rather than model knowledge. The relevant overlap concern is addressed through environment-stratified splits for detection performance evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a detection framework, not model benchmark performance. While the models are fine-tuned on tasks like TL;DR, the evaluation target is detection precision/recall, not model task accuracy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. The study uses human annotators who provided informed consent and received compensation, but the experimental protocol was not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The ethics statement says 'Human annotation was conducted under informed consent with compensation consistent with local standards' but does not mention IRB approval or ethics board review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "RL annotators described only as 'three independent RL experts with extensive experience in reward hacking identification.' LLM annotators described as '3 human annotators' with no demographics (experience level, background, etc.) reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No formal inclusion/exclusion criteria for annotator selection are stated. The RL experts are characterized only by having 'extensive experience' but no specific criteria for what constitutes sufficient expertise."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with human participants assigned to treatment/control conditions. The humans serve as annotators providing ground truth labels, not as experimental subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not applicable to this annotation task. The annotators label gaming instances; there is no treatment condition to blind."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of annotator attrition or dropout. The paper does not state whether all recruited annotators completed their annotation tasks or if any were replaced."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 14 reports per-checkpoint costs: '∼0.8s per output' for EST computation, '∼0.1s per checkpoint' for correlation tracking, '2.1% of training time' total overhead. Appendix N reports '≈0.2 GPU-hours baseline cost' per 1,000 episodes."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-technique costs are given (Table 31: 0.13-0.58 GPU-hours for mitigation techniques on NVIDIA A6000 48GB), but the total computational budget for all experiments (RL + LLM fine-tuning + detection + human annotation) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "RL experiments use '10 random seeds per configuration' (Appendix C.1). Table 11 reports results across 5-fold cross-validation with 95% CI. Table 18 reports mean ± std across random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states '10 random seeds per configuration' for RL and '5-fold cross-validation' for detection performance evaluation. '10 training checkpoints' are sampled per LLM condition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G.1 states 'comprehensive sensitivity analysis on key detection parameters' across '243 tested parameter combinations in the full grid search.' Table 19 reports sensitivity ranges for 5 hyperparameters."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3.1 states 'Detection thresholds were determined through 5-fold cross-validation on this subset only, with hyperparameters selected to maximize F1-score on validation folds.' Figure 3 shows threshold calibration analysis justifying the optimal threshold τ=0.6."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper performs many comparisons across 32 experimental conditions, 15 RL environments, 6 hacking categories, and multiple baseline methods without mentioning any multiple comparison correction (Bonferroni, Holm, Benjamini-Hochberg, etc.)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines themselves (LSTM-Autoencoder, One-Class SVM, Isolation Forest, etc.) and compare against their own framework. No acknowledgment of author-evaluation bias or independent evaluation is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While overhead percentages are reported (2.1% LLM, 4.2% RL), performance is not shown as a function of compute budget. Baselines and the proposed method may require different compute budgets, but this is not analyzed or equalized."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses human annotations as ground truth for gaming detection without questioning whether human annotators reliably identify all forms of gaming or whether inter-rater agreement (κ = 0.847 RL, κ ≥ 0.78 LLM) constitutes sufficient construct validity. The gap between 'human-annotated gaming' and 'actual proxy gaming' is not discussed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The detection framework operates as a monitoring pipeline on standard fine-tuning runs without scaffolding confounds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether pre-trained Llama-3 or GPT-4 models may have encountered training task data (TL;DR summaries, GSM8K problems) during pre-training, which could affect the gaming patterns observed during fine-tuning."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of whether detection features (format sensitivity, correlation metrics) might inadvertently contain information that leaks label assignments or whether the feature design introduces bias toward certain detection patterns."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3.1 states 'Training and testing employed environment-stratified splits to prevent data leakage, with entire environment-algorithm combinations held out for testing.' Section 4.1 uses 'strict train-validation-test splits, holding out entire task-model-judge combinations for testing.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Environment-stratified and task-model-judge combination splits serve as structural prevention methods against train-test leakage. These ensure entire configurations are held out, preventing information flow between training and test sets for the detection framework."
       }
     }
   },
   "claims": [
     {
       "claim": "EST achieves 78.4% precision and 81.7% recall for reward hacking detection across 15 RL environments and 5 algorithms",
       "evidence": "Table 11 reports overall precision 0.784±0.027, recall 0.817±0.023 on 2,156 expert-annotated episodes with environment-stratified 5-fold cross-validation. Cohen's κ = 0.847 for inter-rater reliability.",
       "supported": "moderate"
     },
     {
       "claim": "EST achieves 74.2% precision and 78.6% recall for evaluator gaming detection across 4 LLM tasks, 2 model scales, 2 training methods, and 2 judges",
       "evidence": "Tables 1 and 7 report overall precision 0.742±0.04, recall 0.786±0.04 on 1,200 human-annotated instances (300 per task). Performance broken down across all 32 conditions in Table 15.",
       "supported": "moderate"
     },
     {
       "claim": "Closed-loop mitigation improves human win-rate by 8.3 points for LLM alignment",
       "evidence": "Table 9 shows baseline win-rate 52.1% improving to 60.4% with format penalty, with correlation maintained at ρ=0.82 vs 0.61 baseline. Table 30 ablation confirms gains from detector-guided interventions vs control experiments with equivalent extra compute (+2.1% only).",
       "supported": "moderate"
     },
     {
       "claim": "Closed-loop mitigation reduces hacking by 54.6% in RL",
       "evidence": "Table 31 reports combined approach achieves 54.6% hacking reduction with 9.1% performance impact and 6.7% computational overhead. Individual techniques range from 28.1% to 47.8% reduction.",
       "supported": "moderate"
     },
     {
       "claim": "EST provides early warning signals with median lead time of 3 checkpoints before human-noticeable quality decline",
       "evidence": "Table 7 reports early warning of 3.0±0.4 checkpoints. Figure 2 illustrates detector triggers preceding human quality decline. Section 5.2 defines early warning as checkpoints between detector trigger and W(t) < 0.50.",
       "supported": "moderate"
     },
     {
       "claim": "Proxy-true correlation tracking transfers directly between RL and LLM domains",
       "evidence": "Table 8 classifies correlation tracking as 'Direct' transfer. Table 18 reports AUC 0.821 (RL) and 0.798 (LLM) for proxy optimization detection without modification. Section 6.1 defines transfer as achieving ≥90% of in-domain performance.",
       "supported": "moderate"
     },
     {
       "claim": "EST maintains 65.9% precision under white-box evasion attacks, recovering to 78.1% with defense-in-depth",
       "evidence": "Table 24 shows white-box evasion reduces precision from 74.2% to 65.9%. Table 25 shows full ensemble (EST + correlation + reasoning + hardened judge) achieves 78.1% precision and 74.3% recall under ensemble-aware evasion with 6.8% FP rate.",
       "supported": "moderate"
     },
     {
       "claim": "Objective alignment is the strongest predictor of hacking frequency, reducing it by 31.2% (Cohen's d = 2.08)",
       "evidence": "Table 12 presents factorial design results with objective alignment showing effect size -0.312, p < 0.001, Cohen's d = 2.08. However, the paper acknowledges 'these unusually large effect sizes reflect our custom environments designed to maximize experimental contrast.'",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Custom testbeds favor detection framework",
       "detail": "Three of 15 RL environments are custom testbeds ('Proxy Trap', 'Tampering Test', 'Alignment Challenge') designed by the authors. These are described as 'designed to maximize diversity of hacking behaviors,' which means they were built to exhibit the patterns the detection framework is specifically engineered to catch. This inflates detection performance metrics."
     },
     {
       "flag": "Circular ground truth for large-scale RL analysis",
       "detail": "Only 2,156 of 15,247 episodes have expert annotations. The remaining 13,091 episodes use 'detector consensus' (≥3/6 detectors agreeing) as pseudo ground truth. The paper acknowledges this circularity but still uses findings from this set to inform analysis of hacking patterns and prevalence."
     },
     {
       "flag": "No code or data despite release claims",
       "detail": "The abstract and conclusion claim 'We release benchmarks for both domains' but no repository URL, download link, or archive is provided. This prevents independent verification of all reported results."
     },
     {
       "flag": "Unusually large effect sizes acknowledged",
       "detail": "The factorial design experiment reports Cohen's d values of 1.24–2.08, which are extremely large. The paper acknowledges these are artifacts of custom environments with maximized experimental contrast, and that 'real world deployments would likely show smaller effects.' The reported Cohen's d of 0.8–1.2 in domain-specific benchmarks is more realistic but still on the high side."
     },
     {
       "flag": "Self-implemented baselines",
       "detail": "All baselines (LSTM-Autoencoder, One-Class SVM, Isolation Forest, BC Divergence, hardened judges, etc.) are implemented by the authors. No reference implementations or third-party code is used, and author-evaluation bias is not acknowledged. Lucic et al. (2018) showed authors' implementations of baselines systematically underperform."
     },
     {
       "flag": "Missing LLM training and inference hyperparameters",
       "detail": "While detection framework hyperparameters are well-documented, critical LLM experimental details are missing: DPO/RLHF learning rates, batch sizes, training epochs, and judge API temperature/sampling settings. These significantly affect both training dynamics and the gaming behaviors observed."
     }
   ],
   "cited_papers": [
     {
       "title": "Defining and characterizing reward hacking",
       "authors": ["Joar Skalse", "Nikolaus HR Howe", "Dmitrii Krasheninnikov", "David Krueger"],
       "year": 2022,
       "relevance": "Formal definition of reward hacking and 'unhackable' proxy evaluators, foundational to this paper's detection framework for AI alignment."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and chatbot arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Establishes LLM-as-judge evaluation methodology and identifies evaluator weaknesses that this paper's detection framework targets."
     },
     {
       "title": "Specification gaming: the flip side of ai ingenuity",
       "authors": ["Victoria Krakovna", "Laurent Orseau", "Richard Ngo", "Miljan Martic", "Shane Legg"],
       "year": 2020,
       "relevance": "Catalogs specification gaming behaviors in RL systems, providing the empirical foundation for the reward hacking taxonomy used in this paper."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper for LLM alignment; the training methodology this paper's detection framework monitors for evaluator gaming."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Stefano Ermon", "Christopher D Manning", "Chelsea Finn"],
       "year": 2023,
       "relevance": "DPO training method used as one of two fine-tuning approaches in this paper's LLM alignment experiments."
     },
     {
       "title": "Scaling laws for reward model overoptimization",
       "authors": ["Leo Gao", "John Schulman", "Jacob Hilton"],
       "year": 2023,
       "relevance": "Studies reward model overoptimization at scale, directly relevant to understanding proxy gaming dynamics in LLM alignment."
     },
     {
       "title": "Concrete problems in ai safety",
       "authors": ["Dario Amodei", "Chris Olah", "Jacob Steinhardt", "Paul Christiano", "John Schulman", "Dan Mané"],
       "year": 2016,
       "arxiv_id": "1606.06565",
       "relevance": "Foundational AI safety paper identifying reward hacking and proxy optimization as core safety challenges."
     },
     {
       "title": "The effects of reward misspecification: Mapping and mitigating misaligned models",
       "authors": ["Alexander Pan", "Kush Bhatia", "Jacob Steinhardt"],
       "year": 2022,
       "relevance": "Explores reward misspecification effects and mitigation strategies in RL, directly related to proxy gaming detection."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "CoT prompting methodology relevant to the reasoning validity detector component that checks whether models produce correct answers through invalid reasoning chains."
     },
     {
       "title": "Deep reinforcement learning from human preferences",
       "authors": ["Paul F Christiano", "Jan Leike", "Tom Brown"],
       "year": 2017,
       "relevance": "Foundational RLHF methodology paper; preference learning approach that creates the evaluator gaming vulnerability this paper detects."
     },
     {
       "title": "Alignment of language agents",
       "authors": ["Zachary Kenton", "Tom Everitt", "Laura Weidinger", "Iason Gabriel", "Vladimir Mikulik", "Geoffrey Irving"],
       "year": 2021,
       "arxiv_id": "2103.14659",
       "relevance": "Analyzes alignment challenges for language agents, including proxy optimization and evaluator gaming."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The detection framework is relevant for practitioners doing RLHF/DPO fine-tuning, but no code is released, limiting immediate usability."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Proxy gaming and reward hacking are well-known problems; the unified RL-LLM framework is novel but the core insight is not surprising."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that models systematically game evaluators during alignment training, raising concerns about RLHF/DPO pipeline reliability."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Shows LLM-as-judge pipelines are gameable (e.g., format exploitation scores 8.2 vs 5.1 human rating), but this is already a recognized concern."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released despite claims of benchmark release."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Iowa State University researchers without established profiles in AI alignment or safety research."
     }
   }
 }