ai-research-survey

scan.json (29564B)

---

 {
   "paper": {
     "title": "User Feedback Alignment for LLM-powered Exploration in Large-scale Recommendation Systems",
     "authors": [
       "Jianling Wang",
       "Yifan Liu",
       "Yinghao Sun",
       "Xuejian Ma",
       "Yueqi Wang",
       "He Ma",
       "Zhengyang Su",
       "Minmin Chen",
       "Mingyan Gao",
       "Onkar Dalal",
       "Ed H. Chi",
       "Lichan Hong",
       "Ningren Han",
       "Haokai Lu"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2504.05522",
     "doi": "10.48550/arXiv.2504.05522"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "A decomposed dual-LLM approach for recommendation exploration—separating novelty generation from user-preference alignment—simultaneously improves novelty and user satisfaction on a commercial short-form video platform serving billions of users. Direct RLHF on the novelty model failed (format collapse to 2% compliance and reward hacking), motivating the decoupled design with inference-time best-of-n selection via an alignment model trained on collective user feedback. Pointwise labeling slightly outperformed pairwise labeling and was 2x faster to train. Live A/B experiments showed increased positive playback rate, completion rate, and unique user-cluster engagement pairs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The training data and user feedback signals are proprietary YouTube/Google data. No dataset is released or made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details are provided. The paper mentions using Gemini but gives no setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The system is a proprietary production deployment and the paper provides no guidance for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported on the live experiment results in Figure 4. The offline training curves in Figure 3 also lack error bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'significantly improved user satisfaction' and 'significant gains' but no statistical significance tests (p-values, t-tests, etc.) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4(a) shows percentage improvements in novel impression ratio and positive feedback ratio relative to the hierarchical contextual bandit baseline. Figure 3 reports specific ppb rate improvements (+0.09%, +0.20%) for NDCG and F1 lifts."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The novelty model uses '<8k training examples' (Section 3) with no justification for this number. No power analysis or sample size justification is provided for the live experiments either."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or any spread measure is reported across experimental runs for either offline or live experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 lists multiple baselines: hierarchical contextual bandit (Song et al., 2022), neural linear bandit (Su et al., 2024), two-tower model (Yang et al., 2020), transformer-based sequential model, and the baseline LLM-powered exploration without alignment (Wang et al., 2024c)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent work (Su et al. 2024, Wang et al. 2024c) and current production models. The baselines span both exploration-oriented and exploitation-oriented approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares pairwise vs. pointwise labeling strategies (Figures 4b-d) and evaluates alignment models trained for 50k vs. 100k steps (Section 5.2), both of which serve as ablations of design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: novel impression ratio, positive playback rate, completion rate, unique engaged user-cluster (UEUC), F1@K, and NDCG@K."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Live A/B experiments on a platform serving billions of users constitute a large-scale user study. Real users evaluate the system's recommendations through their engagement behavior (playback, completion, likes, shares)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states: 'Offline evaluation is done on a holdout set of interest cluster sequences, the novel interest transitions and user's feedback scores.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4(a) breaks results down by model category (exploration-oriented vs. exploitation-oriented). Section 5.3 notes that UEUC is 'higher for more active users,' providing a user-segment breakdown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 extensively discusses failed approaches: RLHF caused format collapse (99%→2% correct format) and reward hacking (predicting high-reward words like 'cat', 'BTS'), and scaling training data yielded neutral results."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 reports that increasing training examples yielded 'neutral results,' that RLHF 'always resulted in the model quickly collapsing,' and that training beyond the convergence point (100k steps) led to worse live performance than the 50k-step checkpoint."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'significant gains in both user satisfaction (measured by watch activity and active user counts) and exploration diversity.' Section 5.3 and Figure 4 show improvements in positive playback rate, completion rate, and UEUC, supporting these claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('improves user satisfaction,' 'enhances exploration efficiency') are supported by live A/B experiments comparing treatment (user-aligned model) against control groups, which is an appropriate causal identification strategy."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title frames the approach as for 'Large-scale Recommendation Systems' generally, but all experiments are on a single commercial short-form video platform (YouTube Shorts). No evidence is presented for other recommendation domains, content types, or platforms."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. No consideration of confounding factors such as novelty effects, seasonal user behavior changes, or whether the improvement is due to the dual-LLM architecture vs. simply more compute at inference time."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures positive playback rate and completion rate but frames these as 'user satisfaction' without discussing whether engagement metrics actually capture satisfaction. Users may watch more without being more satisfied, and no discussion of this proxy gap appears."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 'we employed Gemini (Team et al., 2024)' with a citation to Gemini 1.5. No specific version, model size, or snapshot date is given. Just 'Gemini' is insufficient per the schema requirements."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides the actual prompt text used for the novelty model, including specific cluster examples and generation instructions ('With less than 30 words, generate a new and different short-form video cluster...')."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'high temperature setting' for sampling and K=2 for cluster sequence length, but does not report the actual temperature value, learning rate, or other training hyperparameters. Section 5.2 mentions 50k and 100k training steps but no other details."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The system uses a recommendation pipeline with two LLMs feeding into a traditional recommender, not agentic scaffolding with tools, retry logic, or memory management."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes preprocessing steps: normalizing skewed feedback scores, filtering cluster transition pairs with insufficient user feedback, and rounding scores to fixed intervals to account for margin of error."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 3 discusses limitations of the baseline approach (hierarchical planning) but the proposed method's limitations are not addressed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed for the proposed method. The paper does not address potential confounds in the A/B testing or limitations of the offline evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show or what settings the approach has not been tested in."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "All data is proprietary YouTube/Google user interaction data. No raw data is available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection: per-query logging of users' cluster sequences and feedback (playback, like, share, skip) on LLM predictions, then aggregation across users for each cluster transition pair."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper says the platform serves 'billions of users' but does not describe how users are selected for the live experiments, whether all users or a subset are included, or any sampling criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While Section 4.1 describes preprocessing steps (normalize, filter, round), exact thresholds and criteria are not given. The number of examples retained at each stage is not reported, and the filtering criteria for 'little user feedback' are unspecified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present. All authors are Google employees but no explicit funding statement is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Google DeepMind, YouTube, and Google Labs. The connection between authors and the evaluated platform is transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are employed by Google, which owns YouTube, the platform being evaluated. Google has a direct financial interest in demonstrating that its recommendation system improvements work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper fine-tunes Gemini on proprietary recommendation data and evaluates on proprietary holdout sets, not on any public benchmark testing model knowledge. Contamination of public benchmarks is not a concern here."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the evaluation uses proprietary holdout data and live A/B tests, not public benchmarks where pre-training contamination is a concern."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No public benchmarks are used. The evaluation is entirely on proprietary data and live traffic, making benchmark contamination inapplicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the live A/B experiments. The experiments appear to be standard industry A/B tests without academic pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned, despite the experiments being conducted on billions of real users on a commercial platform."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No user demographics are reported. The paper mentions 'billions of users' and that UEUC is 'higher for more active users' but provides no demographic characterization."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for experiment participants are stated. It is unclear whether all platform users were included or if a subset was selected."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'experiment and control' groups (Figure 4) implying randomization, but the randomization procedure (how users were assigned to arms, stratification, etc.) is not described."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Users presumably did not know they were in an experiment, but this is not explicitly stated or discussed."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition data is reported. The number of users who started vs. completed the experiment period is not mentioned."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 notes there is 'no latency impact' since inference happens offline, and mentions '5 times more predictions,' but no actual inference costs (compute time, API costs, tokens consumed) are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training steps (50k, 100k) are mentioned but no GPU hours, hardware specifications, total compute budget, or training time are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results appear to be from single runs of the models."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs for offline evaluation or model training is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 mentions offline evaluation guided hyperparameter tuning (e.g., 'score normalization strategy') but no search budget, number of configurations tried, or search method is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 justifies selecting the 50k-step checkpoint because 'F1 converged in offline evaluation as shown in Figure 3.' The 100k-step arm was deployed as a comparison arm and showed worse live performance, validating the selection."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Google employees evaluate a system deployed on Google's YouTube platform. The paper does not acknowledge or discuss the inherent bias of evaluating one's own system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The proposed method uses 5x more inference compute (5 independent samplings) than the baseline novelty model, but performance is not explicitly compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses that F1@K is more appropriate than NDCG for their top-n selection task: 'F1@K focuses on the model's ability to identify the top-K most relevant clusters, which is more crucial for our top-n selection task. Memorizing the exact rankings is unnecessary.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation compares the complete system (novelty LLM + alignment LLM + recommender) against other production systems. No model-level comparisons with different scaffolding are made — the system IS the thing being tested."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the holdout set's temporal relationship to training data could introduce leakage. The paper does not describe how the holdout set was constructed temporally."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the aggregated user feedback signals used for training the alignment model leak information about the holdout evaluation data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and holdout cluster sequences are independent. User cluster sequences may share structural patterns across the split."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are described."
       }
     }
   },
   "claims": [
     {
       "claim": "The decomposed dual-LLM approach (novelty + alignment models) simultaneously improves recommendation novelty and user satisfaction, achieving a more optimal point on the novelty-quality operating curve.",
       "evidence": "Figure 4(a) shows the proposed method in the upper-right quadrant (highest novelty AND highest quality improvement relative to hierarchical contextual bandit baseline), outperforming all existing production models. Section 5.3 discusses this.",
       "supported": "moderate"
     },
     {
       "claim": "Direct RLHF on the novelty model causes catastrophic format collapse and reward hacking.",
       "evidence": "Section 3 reports that after 5k RLHF steps, 'the LLM's chance of predicting in the correct format drops from 99+% to 2%' and the model learned to predict high-reward words like 'cat', 'BTS', 'toys'.",
       "supported": "moderate"
     },
     {
       "claim": "Inference-time scaling with best-of-n selection from the alignment model improves exploration efficiency.",
       "evidence": "Section 4.2 describes sampling 5x more predictions and selecting top-k using the alignment model. The live results in Figure 4 show improvements, but no ablation isolates the inference-scaling contribution from the alignment model itself.",
       "supported": "weak"
     },
     {
       "claim": "Pointwise labeling slightly outperforms pairwise labeling for the alignment model.",
       "evidence": "Figures 4(b), (c), (d) show both approaches improve user satisfaction and exploration, 'with the pointwise model slightly outperforming the pairwise model' (Section 5.3). Pointwise training is also 2x faster.",
       "supported": "moderate"
     },
     {
       "claim": "The alignment model trained with collective user feedback effectively predicts user preferences over novel interest clusters.",
       "evidence": "Figure 3 shows offline NDCG and F1 lift over random improving consistently during training. Figure 2 shows qualitative examples of alignment model predictions matching intuitive user preferences.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All 14 authors are Google employees (Google DeepMind, YouTube, Google Labs) evaluating a system deployed on YouTube. No independent evaluation or external validation is provided."
     },
     {
       "flag": "Redacted experimental details",
       "detail": "Exact dates are redacted in Figure 4, and specific metric values in the live experiments are shown only as relative percentages. This makes independent verification of the claimed improvements impossible."
     },
     {
       "flag": "No error bars or statistical tests on live experiments",
       "detail": "The word 'significant' is used multiple times to describe live experiment results, but no statistical significance tests, confidence intervals, or error bars are reported for any of the A/B test comparisons."
     },
     {
       "flag": "No limitations section for proposed method",
       "detail": "While Section 3 discusses limitations of the baseline approach, the proposed dual-LLM method's limitations (e.g., dependency on sufficient collective feedback, cold-start for new cluster transitions, scalability of K) are not discussed."
     },
     {
       "flag": "Inference-time scaling contribution not isolated",
       "detail": "The paper claims inference-time scaling (best-of-n) is effective, but no ablation separates its contribution from the alignment model's contribution. The improvement could be entirely from better scoring rather than repeated sampling."
     }
   ],
   "cited_papers": [
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper; the current work builds on and identifies limitations of this approach for structured recommendation tasks."
     },
     {
       "title": "Large language monkeys: Scaling inference compute with repeated sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Introduces the inference-time scaling approach via repeated sampling that this paper adapts for recommendation."
     },
     {
       "title": "LLMs for user interest exploration in large-scale recommendation systems",
       "authors": ["Jianling Wang", "Haokai Lu", "Yifan Liu"],
       "year": 2024,
       "relevance": "The base hierarchical planning system this paper extends with user feedback alignment; describes the deployed LLM-powered exploration paradigm."
     },
     {
       "title": "A survey on large language models for recommendation",
       "authors": ["Likang Wu", "Zhi Zheng", "Zhaopeng Qiu"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs applied to recommendation systems, providing context for the broader research area."
     },
     {
       "title": "TallRec: An effective and efficient tuning framework to align large language model with recommendation",
       "authors": ["Keqin Bao", "Jizhi Zhang", "Yang Zhang"],
       "year": 2023,
       "relevance": "Fine-tuning framework for aligning LLMs with recommendation tasks, directly related to the alignment challenge addressed in this paper."
     },
     {
       "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context",
       "authors": ["Gemini Team"],
       "year": 2024,
       "arxiv_id": "2403.05530",
       "relevance": "The base model used for both the novelty and alignment LLMs in this paper's experiments."
     },
     {
       "title": "Large language models are zero-shot rankers for recommender systems",
       "authors": ["Yupeng Hou", "Junjie Zhang", "Zihan Lin"],
       "year": 2024,
       "relevance": "Demonstrates LLM capabilities as zero-shot rankers in recommendation, relevant to understanding LLM reasoning in recommendation tasks."
     },
     {
       "title": "Better generalization with semantic IDs: A case study in ranking for recommendations",
       "authors": ["Anima Singh", "Trung Vu", "Nikhil Mehta"],
       "year": 2024,
       "relevance": "Fine-tuning LLMs for recommendation ranking with semantic IDs, related approach to aligning LLMs with recommendation objectives."
     },
     {
       "title": "Long-term value of exploration: Measurements, findings and algorithms",
       "authors": ["Yi Su", "Xiangyu Wang", "Elaine Ya Le"],
       "year": 2024,
       "relevance": "Addresses long-term value of exploration in recommendation systems, directly related to the exploration-exploitation tradeoff this paper tackles."
     },
     {
       "title": "Large language models as data augmenters for cold-start item recommendation",
       "authors": ["Jianling Wang", "Haokai Lu", "James Caverlee"],
       "year": 2024,
       "relevance": "Uses LLMs as data augmentation tools for recommendation systems, an alternative paradigm to the direct LLM planning approach in this paper."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Describes a deployed production system on YouTube with concrete architectural patterns, but the approach requires proprietary infrastructure and data that most practitioners cannot replicate."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that RLHF catastrophically fails for structured recommendation tasks (format collapse to 2%) is moderately surprising given RLHF's success elsewhere."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised; the paper focuses on recommendation quality optimization."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; straightforward system improvement paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or reproducible artifacts are released; the system is entirely proprietary."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Google DeepMind and YouTube authors working on YouTube's recommendation system with Gemini; high brand recognition across all dimensions."
     }
   }
 }