ai-research-survey

scan.json (27084B)

---

 {
   "paper": {
     "title": "Annotation alignment: Comparing LLM and human annotations of conversational safety",
     "authors": ["Rajiv Movva", "Pang Wei Koh", "Emma Pierson"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2406.06369"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or any repository URL is provided in the paper. No mention of code release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses the publicly available DICES dataset (Aroyo et al., 2023). The DICES-350 dataset is referenced as a public resource. However, the authors' own LLM annotations are not explicitly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper mentions API access (OpenAI API, Google Vertex AI API) but does not specify library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described in text but without sufficient detail for exact replication (e.g., exact API call parameters beyond temperature)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports confidence intervals: e.g., 'r = 0.95 (95% CI: [0.93, 0.96])' for split-half reliability, and 99% CIs for null distributions in RQ2 (e.g., 'r ∈ [0.44, 0.64] for the Latinx female group'). Bootstrap-based intervals are used throughout."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap resampling is used for significance testing: 'we check if r1 > r2 in at least 95% of 1000 bootstrap resamples' (Section 3, RQ1). Permutation tests with 5000 permutations are used for RQ2 (Section 3). Bonferroni correction is applied for multiple comparisons (footnote 3)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pearson correlations are reported as effect sizes throughout (e.g., r = 0.59 for GPT-4 vs. r = 0.51 for median annotator). Table 1 provides comprehensive correlation values. The paper contextualizes GPT-4's correlation as falling at the 81st percentile relative to individual annotators."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses statistical power: RQ2 shows 'the dataset is underpowered to detect demographic differences in annotator-LLM alignment' (Section 3). The authors demonstrate that 350 conversations are insufficient for subgroup comparisons by showing wide CIs. This is a power analysis by demonstration."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations of individual annotator correlations with GPT-4 are reported: 'across all annotators, the standard deviation of an individual annotator's correlation with GPT-4 is 0.107; for the raters within a group, the SD in correlation with GPT-4 ranges from 0.075 (Latinx female) to 0.132 (Asian male)' (Section 3, RQ2). Reliability checks in Appendix C report r >= 0.94 across repeated trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: five LLMs are compared (GPT-3.5, GPT-4, GPT-4o, Gemini 1.5 Pro, Llama 3.1 405B), and the median human annotator's correlation (r = 0.51) and the average of 3 random humans (r = 0.72) serve as human baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All models tested are contemporary as of 2024: GPT-4, GPT-4o, Gemini 1.5 Pro, and Llama 3.1 405B Instruct. These are leading models on recent leaderboards (Chiang et al., 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates prompt design choices: rating-only vs. analyze-rate prompts (Table 1), Likert vs. binary ratings (Appendix C), single vs. per-criterion ratings (Appendix C), and zero-shot vs. few-shot (Appendix C). Each design choice is evaluated on a held-out validation set."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pearson correlation on both Likert ratings and binarized ratings (Table 1), compares to the median annotator and percentile rank, and examines within-group standard deviation as additional measures."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is central to this paper. The entire study compares LLM annotations against 112 human annotators. The qualitative analysis of disagreements (Section 3, RQ1) involves manual categorization of 27+21 disagreement cases by the authors."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly uses DICES-990 as a validation set for prompt engineering and DICES-350 as a held-out test set. Footnote 7 states: 'prompt engineering is a form of hyperparameter tuning... here we engineered our prompt by validating on DICES-990, so that DICES-350 is a fully held-out test set.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (5 models), by prompt type (rating-only vs. analyze-rate), by rating type (Likert vs. binary) in Table 1, and by demographic group (10 race-gender subgroups) in Figure 2."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive qualitative analysis of failure cases is provided in Section 3 (RQ1) and Tables S2-S3: 27 cases where GPT-4 rates safe but humans rate unsafe, and 21 cases in the opposite direction, with manual categorization of error patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are prominently reported: RQ2 shows the dataset is underpowered for demographic alignment analysis; RQ3 shows GPT-4 cannot predict group disagreements; Appendix D describes three discarded approaches that did not work; Appendix C reports few-shot learning led to reduced correlation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT-4 achieves r = 0.59 (supported by Table 1), that larger datasets are needed to resolve demographic disparities (supported by RQ2 analysis showing wide CIs), that there is substantial idiosyncratic variation (supported by within-group SD analysis), and that GPT-4 cannot predict group disagreements (supported by RQ3 null results). All claims are directly supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to make correlational rather than causal claims. Statements like 'GPT-4 achieves a Pearson correlation of r = 0.59' and 'there is no evidence that GPT-4 can identify conversations...' are appropriately framed. The qualitative analysis identifies patterns but does not make causal claims about why disagreements occur."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds generalization: safety ratings depend on annotation guidelines and prompt design; the study uses only English conversations; the safety definition inherits DICES's five categories; race/gender are U.S.-centric and coarse. The authors note 'the results we observe here would change with different annotator guidelines or prompts.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations: disagreements may reflect normative differences rather than LLM errors (Section 3, RQ1); absence of significant demographic differences may be due to insufficient power rather than true absence (RQ2); alignment variation within groups suggests factors beyond demographics matter (RQ2); prompt sensitivity could explain results (Limitations)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model version IDs are provided in Appendix B: gpt-3.5-turbo-0125, gpt-4-0125-preview, gpt-4o-2024-05-23, google/gemini-1.5-pro-001, and meta/llama3-405b-instruct-maas."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Figure S1 (safety annotation prompt) and Figure S2 (disagreement prediction prompt). The actual text used is shown, including the analyze-rate structure and rating scale."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is reported as 0 for all models (Appendix B: 'All generations used a temperature of 0 to reduce non-determinism'). The binarization threshold is specified (>=3 for most models, >=2 for GPT-3.5)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study uses direct zero-shot prompting of LLMs via API calls."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: annotators with >80% disagreement with majority were removed (11 out of 123, leaving 112); binary aggregation of five safety criteria is described; the binarization threshold for Likert ratings is specified; Gemini's 16 blocked outputs are documented and handled by filling with middle value 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A substantial dedicated 'Limitations' section is present (spanning approximately one full page), discussing demographic scope, prompt sensitivity, safety definition, language scope, and annotator characteristics."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: the dataset includes only five coarse U.S.-centric race categories and two binary genders; DICES annotation guidelines were not released in full so the prompt may not perfectly match annotator instructions; the study did not systematically explore how prompt definitions of safety impact ratings; annotator characteristics beyond demographics were not captured."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Specific scope boundaries are stated: results apply only to English conversations; the safety conceptualization inherits DICES's five categories and may not generalize to other definitions; the RQ2 null result does not disprove meaningful demographic differences; results would change with different annotator guidelines or prompts; characteristics beyond race and gender may be more salient."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying DICES dataset is publicly available (Aroyo et al., 2023), enabling independent verification of the human annotation data. However, the LLM-generated annotations are not explicitly released as a separate dataset."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail in Section 2 and Appendix A: DICES-350 consists of 350 multi-turn conversations collected by crowdworkers interacting with LaMDA on sensitive topics, rated by 112 crowdworkers across five binary safety criteria. LLM annotations were generated via API calls with specified prompts and temperature settings."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the crowdworkers were recruited for DICES. It references the original DICES paper (Aroyo et al., 2023) for details, but does not provide recruitment methods itself. The potential bias introduced by crowdworker selection (e.g., platform used, selection criteria) is not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: 123 annotators → 11 removed for extreme disagreement → 112 remaining; 350 conversations rated; annotations aggregated across five binary criteria; LLM annotations generated with specified prompts; Gemini's 16/350 blocked outputs handled with middle value. Validation set (DICES-990) vs. test set (DICES-350) separation is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgments: RM supported by NSF DGE #2139899; PWK supported by Singapore NRF and AI Visiting Professorship Programme; EP supported by Google Research Scholar award, NSF CAREER #2142419, CIFAR Azrieli scholarship, LinkedIn Research Award, and Abby Joseph Cohen Faculty Fund."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Rajiv Movva (Cornell Tech), Pang Wei Koh (University of Washington), Emma Pierson (Cornell Tech). These are academic affiliations with no direct conflicts regarding the evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding sources (NSF, Singapore NRF, Google Research Scholar, CIFAR, LinkedIn) are generally independent of outcomes regarding GPT-4, Gemini, or Llama performance. The Google Research Scholar award to EP is a minor potential concern given that Gemini is a Google product, but Google is not the primary funder and the paper does not evaluate Gemini favorably over competitors."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate model capability on a benchmark. It studies annotation alignment between LLMs and humans on a subjective safety rating task. Training data contamination is not relevant because the task is subjective annotation, not factual knowledge recall."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The study evaluates subjective safety annotation rather than model knowledge on a benchmark. Whether the model has seen the DICES conversations during training does not invalidate the annotation alignment analysis in the same way benchmark contamination would."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. This is not a benchmark evaluation study. The task is subjective safety annotation where contamination with the specific conversations would not necessarily bias the results in a meaningful way (the model is not being tested on factual recall)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not recruit new human participants. It uses the existing public DICES dataset. The crowdworkers were part of the original DICES study, not this paper's research."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No new human participants were recruited. The paper uses an existing public dataset (DICES). Ethics review would have been the responsibility of the original DICES study authors."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No new human participants were recruited. The DICES dataset demographics (race, gender, age group) are described as part of the dataset description, but these are properties of the existing dataset, not a new human study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No new human participants were recruited. The authors describe their filtering of the existing DICES annotators (removing 11 with >80% majority disagreement), but this is data preprocessing, not human subjects research."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental study with human participants. The paper analyzes an existing dataset."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental study with human participants requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No new human participants were recruited. The annotator filtering (123 → 112) is data preprocessing on an existing dataset."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports per-model costs for annotating 350 conversations: approximately $0.10 for GPT-3.5, $0.60 for GPT-4o, $2 for GPT-4, $1 for Gemini-1.5, and $4 for Llama-3.1. Total cost for all experiments including prompt engineering was $200."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The total API spend is stated as $200 for all experiments (Appendix B). No GPU hours were needed as all computation was done via API calls."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 with analyze-rate prompt achieves Pearson correlation of r = 0.61 (Likert) and r = 0.59 (binary) with average annotator ratings, higher than the median human annotator's r = 0.51.",
       "evidence": "Table 1 reports correlations across 5 models and 2 prompt types. GPT-4 analyze-rate achieves the highest Likert correlation (0.61, significantly better than second-best in >95% of bootstraps). The binary correlation of 0.59 falls at the 81st percentile of human annotators (Section 3, RQ1).",
       "supported": "strong"
     },
     {
       "claim": "The DICES-350 dataset is underpowered to detect demographic differences in annotator-LLM alignment.",
       "evidence": "Figure 2 shows all true group-model correlations lie within their null 99% CIs. The CIs are wide (e.g., r ∈ [0.44, 0.64] for Latinx female group). Permutation tests with 5000 permutations and Bonferroni correction show no significant differences (Section 3, RQ2).",
       "supported": "strong"
     },
     {
       "claim": "There is substantial idiosyncratic variation in alignment with GPT-4 within demographic groups, suggesting race and gender do not fully capture differences in alignment.",
       "evidence": "The standard deviation of individual annotator correlations with GPT-4 is 0.107 overall; within-group SDs range from 0.075 to 0.132, often comparable to or exceeding the overall SD (Section 3, RQ2).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 cannot predict when one demographic group finds a conversation more unsafe than another.",
       "evidence": "Pearson correlations between GPT-4's disagreement Likert ratings and true group differences are near zero and not significant for all tested group pairs: (white, Black), (white, Asian), (white, Latinx). Additional checks in Appendix E confirm the null result (Section 3, RQ3).",
       "supported": "strong"
     },
     {
       "claim": "There are systematic patterns in GPT-4-human disagreements: GPT-4 rates sensitive advice as safe while annotators rate it unsafe, and GPT-4 flags subtle bias/stereotyping that annotators rate as safe.",
       "evidence": "Qualitative analysis of 27 safe-by-GPT/unsafe-by-humans cases (17/27 are sensitive advice) and 21 unsafe-by-GPT/safe-by-humans cases (15/21 involve bias/stereotyping). Examples in Tables S2 and S3 (Section 3, RQ1).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "GPT-4 and Llama 3.1 405B achieve higher correlation with average human safety ratings than the median individual annotator (r = 0.59-0.61 vs. r = 0.51) on the DICES-350 dataset, though both still fall well below the agreement among groups of human annotators (r = 0.72 for 3 random humans, r = 0.95 for split-halves). The dataset lacks statistical power to detect demographic differences in LLM alignment, and within-group variation in alignment often exceeds between-group variation. GPT-4 cannot predict when different demographic groups disagree about safety, suggesting current models do not capture pluralistic perspectives on harm.",
   "red_flags": [
     {
       "flag": "No code or LLM annotation data released",
       "detail": "Despite being a computational study with relatively simple methodology, no code repository or generated LLM annotation data is released, limiting reproducibility."
     },
     {
       "flag": "Potential contamination not discussed",
       "detail": "While contamination is less directly relevant for subjective annotation tasks than for benchmark evaluations, the models may have been trained on DICES conversations or similar content, which could influence their safety ratings. This is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "DICES Dataset: Diversity in Conversational AI Evaluation for Safety",
       "authors": ["Lora Aroyo", "Alex S. Taylor", "Mark Diaz", "Christopher M. Homan", "Alicia Parrish", "Greg Serapio-Garcia", "Vinodkumar Prabhakaran", "Ding Wang"],
       "year": 2023,
       "relevance": "Core dataset used for LLM safety annotation alignment study; key resource for evaluating diverse human perspectives on AI safety."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu"],
       "year": 2022,
       "relevance": "Foundational work on training AI with AI feedback for safety, directly relevant to LLM alignment methodology."
     },
     {
       "title": "Safety-Tuned LLaMAs: Lessons From Improving the Safety of Large Language Models that Follow Instructions",
       "authors": ["Federico Bianchi", "Mirac Suzgun", "Giuseppe Attanasio"],
       "year": 2024,
       "relevance": "Evaluation of LLM safety via adversarial prompts, complementary approach to annotation alignment studied in this paper."
     },
     {
       "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal",
       "authors": ["Mantas Mazeika", "Long Phan", "Xuwang Yin"],
       "year": 2024,
       "relevance": "Standardized safety benchmark for evaluating LLM robustness against adversarial attacks."
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng", "Ying Sheng"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Major LLM evaluation platform using human preferences; used as reference for model selection in this study."
     },
     {
       "title": "Can Large Language Models Be an Alternative to Human Evaluations?",
       "authors": ["Cheng-Han Chiang", "Hung-yi Lee"],
       "year": 2023,
       "relevance": "Directly relevant study on using LLMs as substitutes for human evaluation in NLP tasks."
     },
     {
       "title": "With Little Power Comes Great Responsibility",
       "authors": ["Dallas Card", "Peter Henderson", "Urvashi Khandelwal"],
       "year": 2020,
       "relevance": "Influential work on statistical power in NLP experiments, cited to support the underpowered finding in RQ2."
     },
     {
       "title": "A Roadmap to Pluralistic Alignment",
       "authors": ["Taylor Sorensen", "Jared Moore", "Jillian Fisher"],
       "year": 2024,
       "relevance": "Framework for pluralistic LLM alignment capturing diverse viewpoints, directly relevant to the survey's scope on alignment methodology."
     },
     {
       "title": "Large language models cannot replace human participants because they cannot portray identity groups",
       "authors": ["Angelina Wang", "Jamie Morgenstern", "John P. Dickerson"],
       "year": 2024,
       "relevance": "Empirical study on LLM limitations in representing diverse perspectives, directly relevant to annotation alignment findings."
     },
     {
       "title": "UltraFeedback: Boosting Language Models with High-quality Feedback",
       "authors": ["Ganqu Cui", "Lifan Yuan", "Ning Ding"],
       "year": 2023,
       "relevance": "Study on using LLM annotations for model training, relevant to the broader context of LLM-as-annotator methodology."
     },
     {
       "title": "The PRISM Alignment Project: What Participatory, Representative and Individualised Human Feedback Reveals About the Subjective and Multicultural Alignment of Large Language Models",
       "authors": ["Hannah Rose Kirk", "Alexander Whitefield", "Paul Röttger"],
       "year": 2024,
       "relevance": "Large-scale alignment study with individualized human feedback, complementary to the demographic alignment analysis in this paper."
     },
     {
       "title": "RTP-LX: Can LLMs Evaluate Toxicity in Multilingual Scenarios?",
       "authors": ["Adrian de Wynter", "Ishaan Watts", "Nektar Ege Altıntoprak"],
       "year": 2024,
       "relevance": "Study on LLM safety evaluation in multilingual settings, extending the monolingual English scope of the current paper."
     }
   ]
 }