ai-research-survey

scan.json (34265B)

---

 {
   "paper": {
     "title": "KorNAT: LLM Alignment Benchmark for Korean Social Values and Common Knowledge",
     "authors": [
       "Jiyoung Lee",
       "Minwoo Kim",
       "Seungho Kim",
       "Junghwan Kim",
       "Seunghyun Won",
       "Hwaran Lee",
       "Edward Choi"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2402.13605",
     "doi": "10.48550/arXiv.2402.13605"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "KorNAT is the first benchmark for measuring national alignment of LLMs with South Korea, comprising 4K social value questions (grounded in a 6,174-person survey) and 6K common knowledge questions (from Korean GED curriculum). Testing seven LLMs, only PaLM-2 and HyperCLOVA X exceeded reference scores in social value and common knowledge respectively, indicating most LLMs are insufficiently aligned with Korean values and knowledge. HyperCLOVA X (trained extensively on Korean data) dramatically outperformed others on Korean-specific subjects, while all models struggled with Mathematics and Science. A human evaluation validated that the proposed alignment metrics correlate with actual Korean population preferences.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository (e.g., GitHub link) is provided. The paper only links to a HuggingFace dataset page with samples and evaluation protocols."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Only 'small samples from the dataset are publicly available now' at the HuggingFace link. The full dataset was scheduled for release on AI Hub in December 2024. At time of publication, the full benchmark is not available for independent evaluation."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 specifies '2 NVIDIA A100 GPUs. We used CUDA 12.0, Pytorch 2.0.1, and huggingface 4.31.0.' This provides sufficient detail for environment recreation."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The paper describes the methodology but does not provide executable instructions for reproducing the evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "All results tables (Tables 1, 2, 3) report ± standard deviation across five prompt variations. For example, 'PaLM-2: 0.331±0.007' in SVA Final Adjustment."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Comparisons between models are based solely on comparing point estimates and standard deviations. Claims like 'PaLM-2 shows the highest social value alignment' are made without formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute alignment scores are reported alongside reference baselines (Best Score, All-Neutral), allowing computation of effect magnitudes. For example, PaLM-2's A-SVA of 0.532 vs. All-Neutral of 0.190 and Best of 0.626 provides full context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.6 provides a rigorous sampling theory justification: with 219 average responses per question, they achieve an error bound of <5.5% at 95% confidence. The derivation from Scheaffer et al. (2011) is fully worked out."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations across five different prompts are reported for all experiments. For example, Table 1 shows '0.331±0.007' for PaLM-2 SVA, and Table 3 shows per-subject standard deviations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes 'Best Score' (theoretical maximum achievable alignment) and 'All-Neutral' (naive baseline answering Neutral for all questions). Seven LLMs are compared against each other and these reference points."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The seven models tested (GPT-4, GPT-3.5-Turbo, Claude-1, PaLM-2, Gemini Pro, Llama-2 70B, HyperCLOVA X) were contemporary state-of-the-art models at the time of the study (2023-2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper proposes a benchmark instrument, not a system with removable components. The three response adjustment scenarios (No Adjustment, Age & Gender, Final) serve as variations of ground truth construction, not ablations of a method."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three social value alignment metrics (SVA, A-SVA, N-SVA) and accuracy per subject for common knowledge. Additionally, refrained/invalid response counts are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.3 describes a human evaluation with 107 participants comparing PaLM-2 and Llama-2 outputs. Participants selected which model's reasoning better aligned with their opinions across 100 questions. PaLM-2 was preferred in 94/100 questions."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated zero-shot on KorNAT with no fine-tuning or tuning on any portion of the dataset. The entire benchmark serves as a held-out test set. Results are averaged across 5 prompts rather than cherry-picking."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-subject breakdowns across 7 subjects for common knowledge. Table 1 shows three metrics under three adjustment conditions. Table 11 shows per-gender and per-age group breakdowns. Table 14 separates simple vs. complex questions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Table 4 analyze omitted responses (refrained and invalid), with examples in Appendix D.3. They discuss GPT-4's 13.93% refrained rate and Claude-1's 11.99% refrained rate on social value questions, and Gemini Pro's 'A/B/C/D' invalid response pattern."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central finding is essentially negative: most models fail to meet reference scores. Mathematics average was only 0.333 across models. Claude-1's alignment decreased with Cross-national Prompting (Table 2). Most models perform worse on Korean-specific subjects than English."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'only a few models met our reference score' — confirmed in Tables 1 and 3 (2/7 for social value, 3/7 for common knowledge). Claims about dataset size (4K+6K=10K), survey size (6,174 participants), and government approval are all substantiated in the paper body."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims, e.g., 'This suggests that the structure of prompts influences the social value alignment of LLMs' (Section 5.2.2) and 'models specifically trained for the Korean context are particularly effective at capturing Korean common knowledge' (Section 5.3). These are not backed by controlled causal designs — confounds like model size, training data volume, and architecture differences are not isolated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicitly scoped to South Korea: 'Our dataset primarily focuses on Korea in 2023' (Limitations). The title specifies 'Korean Social Values and Common Knowledge.' They note that 'nationally well-aligned model with one country may not be generalizable to another' (Broader Impacts)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers single interpretations for most findings. For HyperCLOVA X's superiority, they 'hypothesize' it stems from Korean linguistic capability and training data (Section 5.3) but don't seriously consider alternatives (model size, architecture, instruction tuning quality). The cross-national prompting results showing USA CP also improves alignment are noted but not explored for confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines 'national alignment' as a composite of social value alignment (agreement with survey distribution) and common knowledge alignment (accuracy on curriculum-based questions). The metrics directly measure what is claimed — there is no gap between measurement and framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D.1 specifies some models ('claude-instant-1.2', 'hcx-002'/'hcx-003', 'PaLM 2 Text Bison', 'gpt-4-1106-preview' for post-processing) but not all. GPT-3.5-Turbo lacks a snapshot version, 'instance version of gpt-4' is ambiguous, and Gemini Pro has no version identifier. Inconsistent specification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates with actual text are provided in Table 10 and Appendix D.2, including all five semantically similar prompt variations in both Korean and English translation. The fill values (question, answer candidates) come directly from the dataset."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for the evaluation experiments. For question generation with GPT-3.5-Turbo, they mention 'default decoding strategy' but do not state what the defaults were."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly with instruction, question, and choices in a zero-shot manner."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Extensive documentation of the data pipeline: keyword extraction (1,644 keywords), question generation and two rounds of human revision (Section 3.1.2), survey filtering via distractor and consistency checks (Section 3.1.3), response adjustment formulas (Section 3.1.4, Eqs. 1-3), and post-processing of model outputs (Appendix D.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section discusses temporal scope (Korea 2023), lack of universal values, demographic imbalances in survey recruitment, and MCQ format limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: the dataset focuses on Korea in 2023 and needs updating as values change; unable to recruit exactly equal demographic representation (smallest subgroup: 708 aged 60+); MCQ format doesn't capture free-form generation capabilities (citing Röttger et al., 2024)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what it does NOT cover: 'we did not address universally accepted social values' and 'we could not cover thorough free-form generation evaluation.' In Broader Impacts: 'nationally well-aligned model with one country may not be generalizable to another, especially those with significant cultural differences.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only 'small samples from the dataset are publicly available now' at HuggingFace. The full dataset and raw survey responses are not available for independent verification at time of publication."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in exceptional detail: Sections 3.1-3.2 and extensive appendices cover keyword extraction sources, question generation prompts, revision guidelines, survey interface (Figure 4), participant payment, filtering criteria, and textbook sources (Table 7)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Survey participants: 6,174 Korean citizens over age 19, recruited via an online survey platform with stratification by age and gender groups. Workers: 34 college graduates for first revision round, 7 selected for diligence for second round, 21 for knowledge dataset (with specific qualification criteria). Demographics in Table 5."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: 1,644 keywords → GPT-3.5-Turbo question generation → 2 rounds of human revision by 34 then 7 workers → survey of 6,174 participants → filtering via distractors/consistency → response adjustment (stratification + sampling + education/residence/income). Figure 2 provides a visual overview."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section discloses IITP grants (No.RS-2019-II190075, No.RS-2024-00338140) funded by the Korean government (MSIT), and the Open AI Dataset Project (AI-Hub) hosted by MSIT and NIA."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All affiliations are listed: KAIST AI, DATUMO Inc., Seoul National University Bundang Hospital, and NAVER AI Lab. Hwaran Lee's NAVER affiliation is disclosed, and NAVER's HyperCLOVA X is one of the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from the Korean government (IITP/MSIT/NIA), which has no direct financial interest in the performance of any particular model. The government's interest is in measuring LLM alignment with Korea, not promoting a specific model."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Hwaran Lee is from NAVER AI Lab, whose product HyperCLOVA X is evaluated and performs best on common knowledge. Authors from DATUMO Inc. (a data company) are also co-authors. These potential conflicts are not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the seven models tested. This is important because the social value questions were generated from news articles within the last 12 months (Aug 2022-Jul 2023), and some models may have trained on similar content."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any model's training data overlaps with the benchmark content. The questions were generated using GPT-3.5-Turbo from public news articles, and the common knowledge questions are based on published textbooks — both could appear in model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis is performed. While KorNAT is a newly created benchmark (reducing direct leakage risk), the source materials (Korean news articles, textbooks, GED reference materials) are publicly available and could be in model training data. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. The study design, survey questions, and analysis plan were not registered in advance on any pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "'This study has been approved by KAIST IRB (KH2024-020)' is stated in the Ethics Statement section."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive demographics in Table 5: breakdown by age × gender (10 groups), job (22 NACE categories), domestic area (7 regions), sexual orientation, education level (6 levels), annual income (10 brackets), religion (5 categories), and disability status."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Survey participants: Korean citizens over age 19, recruited with stratification by age and gender. Workers for question revision: 'all college graduates or above.' Workers for complex knowledge questions had stricter criteria: 'top 4% in Korean SAT, experience in education, or holding a college degree or higher in the relevant subject.'"
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation (Section 5.2.3), participants compared pairs of model outputs, but the paper does not describe how the order of presentation (Answer A vs. Answer B) was randomized or counterbalanced. For the main survey, the randomization of question presentation is not described beyond distractor/consistency questions appearing 'randomly with a probability of 10%.'"
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation (Section 5.2.3) presents model outputs as 'Answer A' and 'Answer B' (Figure 7), which suggests blinding, but the paper never explicitly states that evaluators were blinded to model identity."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports 6,174 final participants and describes filtering criteria (distractor questions, consistency checks), but does not report how many participants were initially recruited vs. how many were excluded by these quality controls. The pre-filtering count is never stated."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or evaluation latency are reported despite testing seven models including multiple API-based models (GPT-4, GPT-3.5-Turbo, Claude-1, PaLM-2, Gemini Pro) on 10K questions each across 5 prompts."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('2 NVIDIA A100 GPUs' in Appendix D.1) but no total compute time, API spend, or wall-clock time for the full evaluation is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported across 5 prompt variations, which is a form of sensitivity analysis, but random seed sensitivity is not explicitly tested. For API models with stochastic outputs, seed or temperature sensitivity is not addressed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "'We conducted experiments using five distinct yet semantically similar prompts' (Section 5.1). All results are reported as averages and standard deviations across these 5 runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The five prompts appear hand-crafted, but the selection process and number of candidate prompts considered is not disclosed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports averages across all 5 prompts rather than cherry-picking the best-performing prompt. This is a principled approach that avoids selection bias."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made (7 models × 3 metrics × 3 adjustments for social values; 7 models × 7 subjects for knowledge) without any multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Co-author Hwaran Lee is from NAVER AI Lab, and NAVER's HyperCLOVA X is evaluated in the benchmark. The Acknowledgements thank multiple Korean AI companies including NAVER for 'discussions regarding the benchmark planning process.' The potential bias of evaluating an affiliated product is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models with vastly different compute requirements (e.g., Llama-2-70B vs. GPT-4 vs. HyperCLOVA X) are compared without any discussion of compute budget normalization. Model sizes and training compute are not controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.3 conducts a human evaluation validating that the proposed SVA metric correlates with actual Korean population preferences (PaLM-2 preferred in 94/100 questions over Llama-2). The Limitations section discusses MCQ format limitations. The survey-based ground truth is grounded in sampling theory."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated via direct zero-shot prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The social value questions are based on news articles from Aug 2022–Jul 2023, and common knowledge questions from published textbooks. Models trained after these dates may have seen the source materials, but this temporal relationship is never analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The MCQ format with answer options could provide structural cues (e.g., longer correct answers, specific patterns in distractor design) that leak information about the correct answer."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Social value questions were generated from public news articles, and common knowledge questions from published textbooks — both widely available in pre-training corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection method (canary strings, membership inference, n-gram overlap analysis) is applied. The paper relies implicitly on the benchmark being newly created, without formal verification."
       }
     }
   },
   "claims": [
     {
       "claim": "Most current LLMs are not sufficiently aligned with South Korea in either social values or common knowledge.",
       "evidence": "Table 1 shows only PaLM-2 exceeded reference performance in SVA and A-SVA. Table 3 shows only HyperCLOVA X, PaLM-2, and Gemini Pro surpassed the 0.6 reference score for common knowledge, with HyperCLOVA X exceeding it by only 0.107.",
       "supported": "strong"
     },
     {
       "claim": "Models extensively trained on Korean data (HyperCLOVA X) are particularly effective at capturing Korean common knowledge.",
       "evidence": "Table 3 shows HyperCLOVA X achieves 0.707 total score, outperforming all others across most subjects (e.g., 0.783 in Korean vs. next-best 0.652), with analysis of unique correct answers in Appendix D.6 Table 13 showing questions requiring Korean cultural understanding.",
       "supported": "strong"
     },
     {
       "claim": "The proposed alignment metrics (SVA, A-SVA, N-SVA) effectively reflect social values as judged by Korean citizens.",
       "evidence": "Section 5.2.3 shows PaLM-2 (highest A-SVA) was preferred by human evaluators over Llama-2 (lowest A-SVA) in 94 out of 100 questions, with preference ratios predominantly 0.7–0.95 (Figure 3, 107 evaluators).",
       "supported": "strong"
     },
     {
       "claim": "Cross-national prompting with Korean context generally improves social value alignment.",
       "evidence": "Table 2 shows Korean CP improves SVA and A-SVA for GPT-3.5-Turbo, GPT-4, HyperCLOVA X, and Gemini Pro compared to baseline. However, Claude-1's alignment decreases with Korean CP (SVA drops from 0.286 to 0.227).",
       "supported": "moderate"
     },
     {
       "claim": "HyperCLOVA X tends to avoid engaging in topics with divided opinions (highest N-SVA, preference for Neutral).",
       "evidence": "Table 1 shows HyperCLOVA X achieves best N-SVA (0.414) while scoring lower in SVA (0.253) and A-SVA (0.318). Table 4 shows 59 refrained responses. The paper notes it is 'the only model to outperform All-Neutral' in N-SVA.",
       "supported": "moderate"
     },
     {
       "claim": "Korean fine-tuned small LMs tend to outperform their multilingual base models on national alignment.",
       "evidence": "Tables 17 and 18 (Appendix E) show Korean fine-tuned models (KoAlpaca, KoVicuna, KoAlpaca-Polyglot, KULLM-Polyglot, KoLlama-2) consistently outperform multilingual counterparts in both SVA and common knowledge. KoLlama-2 achieves highest total knowledge score (0.324) among small models.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Co-author Hwaran Lee is from NAVER AI Lab. NAVER's HyperCLOVA X is evaluated in the benchmark and achieves the best common knowledge score (0.707). The Acknowledgements thank NAVER and other Korean companies for 'discussions regarding the benchmark planning process,' but no conflict of interest statement appears in the paper."
     },
     {
       "flag": "Dataset not available for verification",
       "detail": "Only small samples are publicly available at HuggingFace. The full dataset was scheduled for release December 2024 on AI Hub. Independent verification and replication of results is impossible without the full benchmark. The survey ground truth distributions, which are the core contribution, are not available."
     },
     {
       "flag": "No contamination analysis despite public source materials",
       "detail": "Social value questions were generated from publicly available Korean news articles, and common knowledge questions from published textbooks and GED materials. These sources are likely present in the training data of tested models, yet no contamination analysis is performed. The strong performance of HyperCLOVA X (trained extensively on Korean content) could partly reflect memorization rather than genuine understanding."
     },
     {
       "flag": "GPT-4 as judge introduces potential bias",
       "detail": "GPT-4 (gpt-4-1106-preview) is used to map non-conforming model responses to answer choices (Appendix D.3). This introduces potential systematic bias: GPT-4 may interpret responses from certain models differently. GPT-4 itself is also one of the evaluated models, creating a circularity concern."
     }
   ],
   "cited_papers": [
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational work on RLHF alignment that KorNAT's social value alignment concept extends to national/cultural contexts."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "GPT-4 is one of the seven LLMs evaluated in the benchmark and is used as a judge for post-processing responses."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper motivating research on LLM alignment and capability evaluation."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Llama-2 is evaluated in KorNAT and serves as the least-aligned model in the human evaluation comparison."
     },
     {
       "title": "HyperCLOVA X technical report",
       "authors": ["Kang Min Yoo", "Jaegeun Han"],
       "year": 2024,
       "arxiv_id": "2404.01954",
       "relevance": "HyperCLOVA X (Korean-specialized LLM from NAVER) achieves the best common knowledge alignment in the benchmark evaluation."
     },
     {
       "title": "Whose opinions do language models reflect?",
       "authors": ["Shibani Santurkar", "Esin Durmus", "Faisal Ladhak"],
       "year": 2023,
       "relevance": "Closely related work on measuring whose values LLMs reflect; KorNAT extends this to Korean-specific national alignment."
     },
     {
       "title": "Towards measuring the representation of subjective global opinions in language models",
       "authors": ["Esin Durmus", "Karina Nyugen"],
       "year": 2023,
       "arxiv_id": "2306.16388",
       "relevance": "Measures global opinion representation in LLMs; KorNAT's cross-national prompting builds on their method."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2021,
       "relevance": "MMLU benchmark for LLM knowledge evaluation; KorNAT's common knowledge dataset is compared against MMLU Korean in Appendix C.4."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Benchmark for measuring LLM truthfulness; related to KorNAT's approach of testing factual knowledge alignment."
     },
     {
       "title": "Aligning AI with shared human values",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2020,
       "relevance": "Foundational dataset for measuring LLM alignment with human values; KorNAT extends this framework to nation-specific values."
     },
     {
       "title": "A general language assistant as a laboratory for alignment",
       "authors": ["Amanda Askell", "Yuntao Bai", "Anna Chen"],
       "year": 2021,
       "arxiv_id": "2112.00861",
       "relevance": "Early work on LLM alignment methods; KorNAT provides a benchmark for evaluating alignment outcomes in Korean contexts."
     },
     {
       "title": "Political compass or spinning arrow? Towards more meaningful evaluations for values and opinions in large language models",
       "authors": ["Paul Röttger", "Valentin Hofmann"],
       "year": 2024,
       "arxiv_id": "2402.16786",
       "relevance": "Critiques MCQ-based value evaluation of LLMs; cited in KorNAT's Limitations as motivation for acknowledging MCQ format constraints."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Useful for researchers building Korean-targeted LLMs but not immediately actionable as a tool or technique for general practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that most LLMs are poorly aligned with non-English cultures is expected; the Korean-specific detail adds nuance but doesn't challenge conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised; the paper focuses on cultural alignment measurement, not adversarial risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; straightforward benchmark construction and evaluation."
     },
     "demo_ability": {
       "score": 1,
       "justification": "HuggingFace link with sample data exists, but the full benchmark is not publicly available for immediate use."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4, PaLM-2, Gemini) but the paper itself is from Korean academic/industry labs without global brand recognition."
     }
   }
 }