ai-research-survey

scan.json (31619B)

---

 {
   "paper": {
     "title": "Modular Pluralism: Pluralistic Alignment via Multi-LLM Collaboration",
     "authors": [
       "Shangbin Feng",
       "Taylor Sorensen",
       "Yuhan Liu",
       "Jillian Fisher",
       "Chan Young Park",
       "Yejin Choi",
       "Yulia Tsvetkov"
     ],
     "year": 2024,
     "venue": "Conference on Empirical Methods in Natural Language Processing",
     "arxiv_id": "2406.15951",
     "doi": "10.48550/arXiv.2406.15951"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Modular Pluralism augments black-box LLMs with a pool of smaller community LMs fine-tuned on community-specific corpora, improving pluralistic alignment across Overton (up to 50.3% better value coverage), steerable (up to 23.8% on balanced accuracy), and distributional (14.9% J-S distance reduction) objectives. The modular design allows patching underrepresented communities by adding new community LMs, with 5-7% improvements for Asian/African countries. Aligned LLMs benefit more than unaligned ones, and larger models show greater improvements (87.9% for 70B vs 17.2% average for 7B).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 states: 'Code and data are publicly available at https://github.com/BunsenFeng/modular_pluralism.' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (Value Kaleidoscope, OpinionQA, MoralChoice, GlobalOpinionQA) and states code and data are publicly available at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Implementation details are provided (LoRA, 16-bit precision, model checkpoints) but no requirements.txt, Dockerfile, or dependency specification is mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides implementation details in Appendix B (hyperparameters, model checkpoints, dataset sampling) but no step-by-step reproduction instructions or scripts to replicate experiments are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3, Figures 2-4, and appendix tables are reported as point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims ('outperforms baselines by up to 23.8%') but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere. The only inter-rater statistic is Fleiss' Kappa for human annotators."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context throughout (e.g., 'improving over baselines by up to 23.8% and 21.8% on balanced accuracy and Macro-F1 scores'), and full baseline and method numbers are provided in all tables, allowing effect size computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are stated (3,132 situations from VK, 22,378 from OpinionQA, 28,763 from GlobalOpinionQA) but no power analysis or sample size justification is provided. The 100 pairs for human evaluation and 5 annotators are given without justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. All tables show single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are consistently compared: vanilla (direct prompting), prompting (pluralism-inducing prompt prefix), and MoE (mixture-of-experts routing to most fitting community LM). Section 3 describes all baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are appropriate for this novel framework: vanilla LLM prompting, instruction-based pluralism induction, and MoE-style routing are all reasonable contemporary approaches. No prior work exists for this exact problem formulation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 includes component analyses: Table 4 compares perspective vs. cultural vs. mixed community LM pools, Figure 6 tests adding individual community LMs for patching, Table 7 analyzes model size effects, and Figure 5 measures message faithfulness."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Accuracy, Balanced Accuracy, Macro-F1 for steerable tasks; Jensen-Shannon distance for distributional tasks; NLI-based value coverage for Overton; plus human evaluation and GPT-4-as-judge evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 (task 2) describes human evaluation: 5 annotators evaluate 100 response pairs for Overton pluralism, with Fleiss' Kappa of 0.4678 reported. Results shown in Figure 3."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit dev/test split is described. Datasets are randomly sampled and evaluated directly. For LLaMA2-70B, 20% is randomly sampled 'due to computing constraints' but this is not a held-out test set. No mention of using a separate validation set for prompt or method tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks down OpinionQA results by 8 socio-political categories (party, ideology, religion, race, education, income, region, sex). Table 3 breaks down GlobalOpinionQA by 7 countries. Figure 4 separates low- and high-ambiguity MoralChoice scenarios."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows only positive working examples in the appendix (Figures 7-10). While Table 4 shows cultural LMs hurt OpinionQA performance, no qualitative failure analysis or error examples are provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows cultural community LMs hurt US-centric OpinionQA performance. Table 7 shows -10.6% on Overton for 7B model. Table 8 shows some models where prompting outperforms Modular Pluralism. These negative findings are discussed honestly."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about improving Overton (68.5%), steerable (26.6% and 10.4%), and distributional (10.9%) pluralism are supported by results in Figures 2-4 and Tables 1-3. The claim about patching underrepresented communities is supported by Figure 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('Modular Pluralism improves/advances pluralistic alignment') are supported by controlled comparisons: same base LLMs evaluated with different methods (vanilla, prompting, MoE, their approach). The component analyses in Table 4 and Figure 6 provide additional causal evidence by varying specific system components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests across 6 LLMs, 4 datasets, and 6 tasks, and explicitly acknowledges limitations about Western-centric community corpora, specific community definitions, and specific evaluation schemes. The Limitations section bounds scope to perspective- and culture-informed communities."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether improvements could be explained by alternative factors such as increased prompt length/context (community LM comments add substantial text), extra compute, or simply providing more diverse text as context. The entropy analysis (Table 6) partially addresses one confound but is insufficient."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "NLI entailment scores are used as a proxy for 'value coverage,' J-S distance for 'distributional alignment,' and classification accuracy for 'steerability,' but the paper does not discuss the gap between these proxies and the actual pluralism constructs they claim to measure."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B specifies exact model checkpoint names: meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-13b-chat-hf, davinci-002, gpt-3.5-turbo, mistralai/Mistral-7B-Instruct-v0.1, google/gemma-7b, etc."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided in Section 2 ('Please comment on a given situation with the help of the following passages,' 'Which of the following comments best reflect <attribute>?') and Appendix B ('Make sure your response reflects diverse values and perspectives,' 'Please evaluate which of the two responses better reflects pluralistic values')."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports: block size 128, batch size 64, learning rate 1e-6, weight decay 1e-2, 1 training epoch, 16-bit precision, LoRA with default hyperparameters, greedy decoding by default, temperature τ=1 for sampling, max 512 new tokens."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The multi-LLM collaboration is a straightforward pipeline (community LMs generate comments, then the main LLM processes them) without tool use, retry logic, or agent loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes dataset sampling procedures (3,132 situations from VK, 22,378 from OpinionQA, 28,763 from GlobalOpinionQA), evaluation protocols for each task, and community LM training data sources (Feng et al. 2023 perspective corpora, CultureBank)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section discusses three specific limitations: narrow community definitions, computational overhead, and evaluation scope. An 'Ethics Statement' section adds further caveats about dual-use risks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this study: community LMs only cover perspective- and culture-informed communities, computational costs of running 6 additional 7B models, reliance on 4 datasets with specific evaluation schemes, and challenges of collecting community-representative corpora including intersectional communities."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what was not tested: 'we mainly considered perspective-informed and culture-informed communities, while pluralistic alignment could be equally important for other definitions of community.' West-centric data bias is acknowledged, and evaluation is bounded to values, cultures, and perspectives."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data are publicly available at https://github.com/BunsenFeng/modular_pluralism' and uses publicly available datasets (VK, OpinionQA, MoralChoice, GlobalOpinionQA)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Community LM training corpora are described as 'six perspective-laden corpora in Feng et al. (2023) featuring left/center/right-learning news and social media documents.' Cultural LMs use CultureBank (Shi et al. 2024) partitioned by continent. Evaluation dataset sources and sampling are described in Appendix B."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Five human annotators are used for evaluation (Figure 3) but their recruitment method, qualifications, and selection criteria are not described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: community corpora → LoRA fine-tuning → community LMs → comment generation → LLM processing (summarization/selection/aggregation depending on pluralism mode). Appendix B details each evaluation task's data pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists NSF CAREER Grant IIS2142739, NSF grants IIS2125201 and IIS2203097, DARPA ITM program FA8650-23-C-7316, Office of Naval Research N00014-24-1-2207, and gift funding from OpenAI."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Washington and New York University. Authors are not affiliated with companies whose products are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI provides 'gift funding' while ChatGPT (gpt-3.5-turbo/davinci-002) is one of the six LLMs evaluated. OpenAI has a financial interest in positive evaluations of its products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The OpenAI funding is disclosed in acknowledgements but no formal conflict-of-interest declaration exists."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the six LLMs used (LLaMA2, ChatGPT, LLaMA3, Gemma). These models are evaluated on public benchmarks (VK, OpinionQA, MoralChoice, GlobalOpinionQA) that could have been in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation dataset examples (survey questions from OpinionQA, GlobalOpinionQA, MoralChoice scenarios, VK situations) appeared in any model's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All four evaluation datasets (VK, OpinionQA, MoralChoice, GlobalOpinionQA) were publicly available before or around the models' training periods. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not conduct a human subjects study. The 5 annotators serve as evaluators of system outputs for quality assessment, not as study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Annotators evaluate system outputs rather than being studied themselves."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotators are evaluators, not participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotators are used for evaluation only."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental human subjects study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental human subjects study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotators evaluate outputs."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "A theoretical compute overhead estimate is provided ('adds only (6*7)/405=10.4% compute') but no measured inference cost, latency, API costs, or tokens consumed are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training details are given (1 epoch, LoRA fine-tuning) but total GPU hours, training time, or total API spend are not quantified. The 20% subsample for LLaMA2-70B 'due to computing constraints' suggests significant cost but this is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single runs. Community LM fine-tuning and all evaluations show single-run numbers."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Fixed hyperparameters are used (learning rate 1e-6, etc.) but no search budget, search method, or justification for why these values were chosen is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses default LoRA hyperparameters and fixed prompts without explaining how these were selected or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across 6 models, 4 datasets, 6 tasks, and multiple settings, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (vanilla, prompting, MoE) and their own method, but do not acknowledge the bias of evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Modular Pluralism uses substantially more compute than baselines (6 extra 7B models) but performance is not reported as a function of matched compute budgets. The theoretical 10.4% overhead estimate does not substitute for actual compute-controlled comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses NLI-based value coverage as a proxy for Overton pluralism, J-S distance for distributional pluralism, and classification accuracy for steerability, but does not discuss whether these benchmarks and metrics actually measure the claimed pluralism constructs."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently compares different methods (vanilla, prompting, MoE, Modular Pluralism) using the same base LLMs, isolating the framework effect. Results are reported for 6 different LLMs, and the scaffold/framework is the controlled independent variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The evaluation datasets (VK, OpinionQA, MoralChoice, GlobalOpinionQA) were published before or around the training of the models used, but this is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. Community LM comments are provided as context to the LLM, but whether this introduces feature leakage for the specific tasks is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data for models overlaps with evaluation data. The community LMs are fine-tuned on news/social media data that may overlap with the evaluation datasets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, temporal splits, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "Modular Pluralism improves Overton pluralism value coverage by up to 50.3% over the strongest baseline.",
       "evidence": "Figure 2 shows NLI-based value coverage: aligned ChatGPT improves from 22.8% (MoE) to 29.0% (Ours), and aligned LLaMA2-13B from 23.0% (MoE) to 29.4%. Confirmed by alternative NLI model WANLI in Table 5.",
       "supported": "moderate"
     },
     {
       "claim": "Modular Pluralism achieves 45.8% and 16.5% higher win rates than the strongest baseline in human and GPT-4 Overton evaluation.",
       "evidence": "Figure 3 shows win/tie/lose rates against MoE, prompting, and vanilla baselines with ChatGPT. Against prompting (strongest baseline), win rates are 62.9% (human) and 46.0% (GPT-4).",
       "supported": "moderate"
     },
     {
       "claim": "Modular Pluralism improves steerability by up to 23.8% on balanced accuracy for value classification.",
       "evidence": "Table 1 shows three-way Value Kaleidoscope results: aligned ChatGPT achieves 68.7% BAcc vs 55.5% for aligned prompting, a 23.8% improvement. Results consistent across binary/three-way settings and both models.",
       "supported": "moderate"
     },
     {
       "claim": "Modular Pluralism reduces distributional distance to human populations by at least 10.9% on GlobalOpinionQA.",
       "evidence": "Table 3 shows J-S distance reductions across 7 countries. Unaligned LLaMA2-13B: 0.295 (MoE) to 0.274 (Ours), a 7.1% reduction. Unaligned ChatGPT: 0.335 (prompting) to 0.274 (Ours), an 18.2% reduction. Average reduction 14.9%.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs are generally faithful to community LM inputs with 51.2% average coverage rate.",
       "evidence": "Figure 5 shows NLI-based entailment analysis across 5 LLMs and 6 community LMs. Social media community LMs have higher coverage (57.7%) than news (44.7%). LLMs also add 33.8% new content.",
       "supported": "moderate"
     },
     {
       "claim": "Adding new community LMs can patch underrepresented communities, improving J-S distance by 5.2-6.7%.",
       "evidence": "Figure 6 shows adding an Asian culture community LM improves Japan alignment (from ~0.30 to ~0.285) and an African culture LM improves Nigeria/Kenya alignment, without degrading other communities.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All results across 6 tasks, 6 models, and 4 datasets are reported as point estimates from apparent single runs. No confidence intervals, significance tests, or variance measures are provided despite many comparative claims."
     },
     {
       "flag": "Non-independent funder",
       "detail": "OpenAI provides gift funding while ChatGPT is one of the primary evaluation models. The conflict is not explicitly acknowledged beyond listing OpenAI in acknowledgements."
     },
     {
       "flag": "Small human evaluation sample",
       "detail": "Human evaluation uses only 5 annotators on 100 response pairs with moderate agreement (Fleiss' Kappa 0.4678). No annotator demographics or recruitment details are provided."
     },
     {
       "flag": "No contamination analysis",
       "detail": "All evaluation datasets were publicly available around or before the models' training periods, yet no contamination analysis is performed. This could differentially affect baselines and the proposed method."
     },
     {
       "flag": "Compute confound unaddressed",
       "detail": "Modular Pluralism uses 6 additional 7B community LMs at inference time, providing substantially more context to the main LLM. The improvement could partially be attributed to longer/richer context rather than the specific community specialization. No ablation separates these effects."
     }
   ],
   "cited_papers": [
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Foundational alignment approach using AI feedback, directly relevant to alignment methodology evaluation."
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2024,
       "relevance": "Major alignment method (DPO) that this paper builds upon and contrasts with for pluralistic alignment."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT/RLHF foundational paper for LLM alignment with human preferences."
     },
     {
       "title": "MaxMin-RLHF: Towards Equitable Alignment of Large Language Models with Diverse Human Preferences",
       "authors": ["Souradip Chakraborty", "Jiahao Qiu", "Hui Yuan"],
       "year": 2024,
       "arxiv_id": "2402.08925",
       "relevance": "Directly addresses equitable alignment with diverse preferences, a key comparison point for pluralistic alignment methods."
     },
     {
       "title": "A Roadmap to Pluralistic Alignment",
       "authors": ["Taylor Sorensen", "Jared Moore", "Jillian Fisher"],
       "year": 2024,
       "arxiv_id": "2402.05070",
       "relevance": "Defines the three pluralism objectives (Overton, steerable, distributional) that this paper operationalizes and evaluates."
     },
     {
       "title": "Knowledge Card: Filling LLMs' Knowledge Gaps with Plug-in Specialized Language Models",
       "authors": ["Shangbin Feng", "Weijia Shi", "Yuyang Bai"],
       "year": 2024,
       "relevance": "Prior multi-LLM collaboration framework that Modular Pluralism builds upon for decoding-time collaboration."
     },
     {
       "title": "DExperts: Decoding-time Controlled Text Generation with Experts and Anti-experts",
       "authors": ["Alisa Liu", "Maarten Sap", "Ximing Lu"],
       "year": 2021,
       "relevance": "Decoding-time control method relevant to understanding controlled generation approaches for LLM alignment."
     },
     {
       "title": "Whose Opinions Do Language Models Reflect?",
       "authors": ["Shibani Santurkar", "Esin Durmus", "Faisal Ladhak"],
       "year": 2023,
       "relevance": "Demonstrates that LLMs reflect narrow demographic opinions, motivating pluralistic alignment and providing the OpinionQA dataset."
     },
     {
       "title": "Evaluating the Moral Beliefs Encoded in LLMs",
       "authors": ["Nino Scherrer", "Claudia Shi", "Amir Feder"],
       "year": 2024,
       "relevance": "Provides the MoralChoice dataset used for distributional pluralism evaluation, assessing moral reasoning in LLMs."
     },
     {
       "title": "Towards Measuring the Representation of Subjective Global Opinions in Language Models",
       "authors": ["Esin Durmus", "Karina Nyugen", "Thomas I Liao"],
       "year": 2023,
       "arxiv_id": "2306.16388",
       "relevance": "Provides the GlobalOpinionQA dataset and demonstrates LLM alignment gaps with global opinion distributions."
     },
     {
       "title": "Personalized Soups: Personalized Large Language Model Alignment via Post-hoc Parameter Merging",
       "authors": ["Joel Jang", "Seungone Kim", "Bill Yuchen Lin"],
       "year": 2023,
       "arxiv_id": "2310.11564",
       "relevance": "Alternative approach to personalized/pluralistic alignment through parameter merging, requiring white-box access unlike Modular Pluralism."
     },
     {
       "title": "Proximal Policy Optimization Algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "Core RL algorithm used in RLHF alignment pipelines that this paper's approach provides an alternative to."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Framework could be adopted by practitioners deploying LLMs to diverse user populations, though requires training community LMs as a prerequisite."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Builds on known issues with alignment monoculture; the idea that smaller specialized models can complement large LLMs is incremental rather than surprising."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Ethics statement mentions dual-use risk of representing hateful communities, but this is not the paper's focus."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to any specific company or widely-held belief."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data released at GitHub (https://github.com/BunsenFeng/modular_pluralism), though running the full system requires multiple model fine-tuning steps."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University of Washington and NYU affiliations with known researchers (Yejin Choi, Yulia Tsvetkov), but not a major industry lab."
     }
   }
 }