ai-research-survey

scan.json (31091B)

---

 {
   "paper": {
     "title": "Sample-Efficient Human Evaluation of Large Language Models via Maximum Discrepancy Competition",
     "authors": [
       "Kehua Feng",
       "Keyan Ding",
       "Hongzhi Tan",
       "Kede Ma",
       "Zhihua Wang",
       "Shuangquan Guo",
       "Yuzhou Cheng",
       "Ge Sun",
       "Guozhou Zheng",
       "Qiang Zhang",
       "Huajun Chen"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2404.08008",
     "doi": "10.48550/arXiv.2404.08008"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is available at https://github.com/weiji-Feng/MAD-Eval.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly mentions code availability but does not state that the 120K evolved instruction pool or human annotation data is released. Seed datasets (GSM8K, CAMEL, AlpacaEval, CodeAlpaca) are public, but the novel evolved instructions and annotation results are not explicitly made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C mentions 'two NVIDIA GeForce RTX 4090 GPUs' and the use of vLLM, but no requirements.txt, Dockerfile, conda environment, or detailed library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and the experimental setup is described in detail (Appendix B-D), the paper does not include step-by-step reproduction instructions with commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All Elo ratings in Tables 1-8 are reported as point estimates without confidence intervals or error bars, despite using 1000 bootstrap resamples for aggregation. The bootstrap uncertainty is not shown."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Spearman's ρ with p-values is reported for ranking comparisons: e.g., 'Spearman's ρ = 0.965 with p = 5.93 × 10^−12' (Section 4.6), 'ρ = 0.993 with p = 2.17 × 10^−13' (Table 4)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Spearman's ρ values (0.965, 0.993, 0.989, 0.986) serve as effect sizes quantifying agreement with gold-standard rankings. Elo rating differences between models are also reported, providing magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 13 evaluators were chosen, why K=10 instructions per pair, or why 8 LLMs were selected. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite performing 1000 bootstrap resamples for Elo ratings, the variance or standard deviation of the bootstrap distributions is not reported. Tables show only point-estimate Elo ratings. For random sampling (Table 5), mean ρ = 0.791 with std = 0.069 is reported, but the main method's variance is absent."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 compares MAD Competition against 5 alternative sampling strategies: random sampling, KL divergence, cross-entropy (Boubdir et al., 2023), Anchor Points (Vivek et al., 2023), and DiffUse (Ashury-Tahan et al., 2024). Section 4.3 compares against Chatbot Arena, AlpacaEval-2.0, and CompassRank."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DiffUse (2024), Anchor Points (2023), and the current Chatbot Arena leaderboard. These represent contemporary state-of-the-art approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 presents ablation studies on three key dimensions: semantic discrepancy measure (Table 3), sample size K (Figure 4), and diversity weight λ."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Elo ratings, Spearman's ρ rank correlations, per-task rankings, pairwise win rates (Figure 6), and inter-annotator agreement (83.39%)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution is a human evaluation methodology. Thirteen STEM-trained postgraduates perform 3-AFC pairwise preference tasks on 1,120 comparison pairs (Appendix D.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no separation between development and test data. The hyperparameters K and λ are tuned on the same evaluation data used for the main results. The validation against external leaderboards (Chatbot Arena, GSM8K) serves as external validation but not as a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-task breakdowns across four tasks (Understanding, Reasoning, Writing, Coding) with separate Elo ratings and rankings for each. Figure 6 shows per-task pairwise win rate heatmaps."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.4 presents extensive case studies of failure cases, including five counterexample categories for GPT-4-Turbo (Tables 27-31): algorithm explanation, instruction comprehension, code constraint, instruction adherence. Table 9 summarizes model weaknesses."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports where alternative methods fail (Table 2, 8), discusses discrepancies between their rankings and established leaderboards (Section 4.3), and shows that without diversity (λ=0), instructions cluster around narrow themes (Section 4.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the method 'recovers gold-standard model rankings with a handful of MAD-selected instructions.' This is supported by Spearman's ρ = 0.965 with Chatbot Arena (Table 7), ρ = 0.993 with MATH500 (Table 4), and ρ = 0.986 with Chatbot Arena Conversations (Table 5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes implicit causal claims about MAD Competition promoting sample efficiency. These are supported by ablation studies (Section 4.5) that systematically vary components (discrepancy measure, K, λ) and demonstrate their contribution to performance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested settings. The abstract specifies 'eight widely used LLMs across four tasks.' The Limitations section acknowledges that the method 'may become computationally prohibitive' at larger scales and that 'underlying data distributions may differ between our curated instruction set and the Chatbot Arena benchmark.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 compares against 5 alternative sampling strategies and analyzes why they fail (KL divergence concentrates on poetry, cross-entropy favors academic tasks). Section 4.3 discusses why leaderboard discrepancies arise (e.g., WizardLM-13B's strong instruction-following on AlpacaEval)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that 'underlying data distributions may differ between our curated instruction set and the Chatbot Arena benchmark, this single comparison may not fully attest to real-world robustness' (Section 4.6). It validates against multiple external gold standards to assess the proxy-outcome gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C specifies exact model versions: 'GPT-4-1106-preview', 'GPT-3.5-Turbo-1106', 'Gemini-1.0-Pro', 'WizardLM-13B-V1.2', 'Vicuna-13B-V1.5'. The 20-model experiment (Table 7) also uses versioned names like 'GPT-4o-2024-05-13'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in the appendix: instruction evolution prompts (Tables 10-13), LLM-based evaluator prompts (Tables 14-17), and the GPT-4-Turbo semantic similarity prompt (Table 20)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C: temperature = 0.7, top-p = 1.0, max sequence length = 2,048 (1,024 for Qwen). Appendix A: Elo parameters τ = 400, η = 4. Section 4.5: λ = 1.0, K = 10."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method directly queries LLMs for responses and applies MAD selection."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix B describe the instruction pool construction in detail: 3K seed instructions per task from established datasets, 10 rounds of automated evolution using 3 closed-source models, yielding 30K evolved instructions per task (120K total)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the Conclusion, discussing three specific limitations: brute-force search scalability, variable cognitive load on evaluators, and O(N²K) scaling for many models."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: brute-force search 'may become computationally prohibitive as pools scale to millions,' variable cognitive load means 'certain pairs of responses may be inherently harder to judge,' and O(N²K) scaling 'may strain annotation budgets.' These are specific to the proposed method."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific scope boundaries: 'this single comparison may not fully attest to real-world robustness' (Section 4.6), acknowledges that 'developing fully automated pipelines remains an open challenge,' and notes cognitive load and scalability limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw human annotation data (individual annotator judgments for 1,120 comparisons) is not stated as publicly available. Only code is explicitly released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes the instruction pool construction (seed collection, 10 rounds of evolution). Appendix D describes the human evaluation process in detail: evaluator selection, briefing, GUI design, annotation protocol."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 describes evaluator selection criteria: graduate-level STEM training, language proficiency requirements (NCEE ≥125, CET-6 ≥500, or native English), disciplinary foundation, and Python proficiency. Appendix D.4 states 'Thirteen graduate students with strong STEM backgrounds are recruited.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: seed instructions → instruction evolution (Appendix B) → LLM response generation → MAD selection (Algorithm 1) → human 3-AFC annotation (Appendix D) → Elo rating with bootstrapping (Section 3.3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists funding: National Natural Science Foundation of China (62301480 and 62302433), Zhejiang Provincial programs (2025C01097 and 2024C01135), and Hangzhou West Lake Pearl Project (TD2023017)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Zhejiang University, ZJU-Hangzhou Global Scientific and Technological Innovation Center, Shanghai Electric Group Co., City University of Hong Kong, and associated research centers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Chinese government research grants (NSFC, provincial programs) which have no direct financial stake in the ranking outcomes of specific LLMs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the 8 evaluated LLMs (GPT-4-Turbo, GPT-3.5-Turbo, Gemini-Pro, or the open-source models)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The instruction pool is evolved from GSM8K, CAMEL, AlpacaEval, and CodeAlpaca — all publicly available before the models' training cutoffs. No discussion of whether evolved instructions or their source benchmarks overlap with model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), CAMEL, AlpacaEval, and CodeAlpaca were all published before the evaluated models' training periods. The evolved instructions derive from these public benchmarks. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned. Appendix D.2 describes informed consent procedures but does not reference any ethics review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Evaluators are described only as 'Thirteen graduate students with strong STEM backgrounds' (Appendix D.4). Selection criteria are detailed (D.1) but actual demographics (gender, age, years of experience, geographic distribution) are not reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 provides detailed inclusion criteria: language proficiency (NCEE English ≥125 and CET-6 ≥500, or native English), high-school mastery of STEM subjects, professional Python proficiency, and commitment to 2+ hour sessions."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an annotation study, not an experimental study with treatment/control conditions. All evaluators perform the same 3-AFC task; there is no randomization to different conditions."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows the evaluation GUI where responses are presented as 'A' and 'B' without revealing model identities. Evaluators select 'A win,' 'Tie,' or 'B win,' with no information about which model generated which response."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports 13 evaluators were recruited (Appendix D.4) but does not mention whether all completed their assigned annotations or if any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or per-comparison costs are reported for running 8 LLMs on 120K instructions across 4 tasks, or for the human annotation effort."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two NVIDIA GeForce RTX 4090 GPUs' (Appendix C) for open-source models but does not report total GPU hours, API spend, training time, or annotation costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Model responses are generated with temperature 0.7 (stochastic), but the paper does not report whether different random seeds produce different MAD selections or rankings. For random sampling, 3 seeds are tested (Table 5), but MAD Competition's seed sensitivity is not assessed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states '1000 bootstrap datasets' for Elo aggregation (Section 3.3), 'at least five different students' per annotation pair (Appendix D.4), and 'three trials with seeds 657, 216, and 849' for random sampling comparison (Section 4.6)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 explicitly reports the configurations tested: λ ∈ {0, 0.5, 1.0, 1.5, 2.0} for diversity weight, K from 1 to 9 for sample size, and 3 semantic discrepancy measures (BERTScore, GPT-4-Turbo, text-embedding-ada-002)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 justifies K=10 (correlations exceed 0.95 for K≥5 and reach 1.0 for K≥8) and λ=1.0 (intermediate value balances discrepancy vs. diversity; λ≤0.5 clusters around narrow themes, λ=2.0 inflates ties)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports multiple Spearman's ρ correlations with p-values across different experimental conditions (Tables 4, 5, 7) but does not apply any correction for multiple comparisons (Bonferroni, Holm, etc.)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose MAD Competition and compare it against their own implementations of 5 alternative sampling strategies. They do not acknowledge or discuss the potential bias of evaluating their own system's implementations of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "DiffUse uses 630 instructions per experiment while MAD Competition uses 210, yet the paper does not present a formal compute-budget-vs-performance analysis. Table 5 compares at matched sample sizes (1K comparisons) for one experiment, but this is not systematic."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses whether the evaluation actually measures what is claimed by comparing against multiple gold standards (Chatbot Arena, AlpacaEval-2.0, CompassRank) and analyzing why discrepancies arise (e.g., AlpacaEval's emphasis on instruction-following vs. human preferences)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. LLMs are directly queried for responses without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The instruction pool is evolved from GSM8K (2021), CAMEL, AlpacaEval, and CodeAlpaca — all published before the models' training periods. No discussion of whether models may have seen these benchmark problems or their variants during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information. For reasoning tasks, evolved instructions include exemplar answers (Appendix B), but it's unclear if this information affects the evaluation."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evolved instructions share structural similarities with model training data, or whether the 10-round evolution process produces near-duplicates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, temporal splits, or decontamination pipelines are employed."
       }
     }
   },
   "claims": [
     {
       "claim": "MAD Competition recovers gold-standard model rankings with only K=10 instructions per model pair",
       "evidence": "Table 2 shows MAD Competition best matches the GSM8K golden ranking among 6 sampling strategies. Table 4 shows Spearman's ρ = 0.993 (p = 2.17e-13) with MATH500. Table 5 shows ρ = 0.986 with Chatbot Arena Conversations using 1K MAD-selected comparisons vs. ρ = 0.791 for random 1K.",
       "supported": "strong"
     },
     {
       "claim": "Reliable rankings emerge from as few as 5 MAD-selected instructions per model pair",
       "evidence": "Figure 4 shows Spearman's ρ exceeds 0.95 for K ≥ 5 and reaches 1.0 for K ≥ 8 across all four tasks.",
       "supported": "moderate"
     },
     {
       "claim": "MAD Competition significantly outperforms random sampling and four alternative strategies in ranking fidelity",
       "evidence": "Tables 2 and 8 show MAD Competition produces the smallest ranking discrepancies from gold standards. Random, KL, cross-entropy, Anchor Points, and DiffUse all show notable ranking errors. Section 4.6: MAD ρ = 0.986 vs. random mean ρ = 0.791 (std = 0.069).",
       "supported": "strong"
     },
     {
       "claim": "LLM-based evaluators (GPT-4o) can substitute for human evaluators with Spearman's ρ > 0.95",
       "evidence": "Table 6 shows Spearman's ρ between GPT-4o and human rankings exceeds 0.95 across all four tasks and the overall ranking.",
       "supported": "moderate"
     },
     {
       "claim": "The method scales to 20 LLMs with strong agreement (ρ = 0.965) with Chatbot Arena",
       "evidence": "Table 7 shows the 20-model ranking with Spearman's ρ = 0.965 (p = 5.93e-12) against Chatbot Arena. However, this uses LLM-based evaluation (GPT-4o), not human evaluation.",
       "supported": "moderate"
     },
     {
       "claim": "The diversity term in MAD Competition prevents narrow clustering and ensures broader task coverage",
       "evidence": "Section 4.5: λ ≤ 0.5 causes clustering around a few themes (e.g., poetry), while λ = 1.0 produces diverse instruction sets. Tables 18-19 compare selected instructions with and without diversity.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces MAD Competition, a sample-efficient evaluation method that adaptively selects maximally discriminative instructions for pairwise LLM comparison. With only 280 human comparisons (K=10 per model pair for 8 models), the method achieves near-perfect rank correlation (ρ = 0.965-0.993) with established leaderboards including Chatbot Arena. The diversity-aware instruction selection explicitly prevents narrow clustering and outperforms five alternative sampling strategies. The method extends to 20 models and generalizes across external datasets (MATH500, Chatbot Arena Conversations).",
   "red_flags": [
     {
       "flag": "Circular evaluation design for reasoning task",
       "detail": "The mathematical reasoning instruction pool is derived from GSM8K via instruction evolution, and the 'gold standard' for validating rankings is GSM8K accuracy itself. This circularity may inflate the reported agreement between MAD-selected rankings and the gold standard."
     },
     {
       "flag": "Complete absence of contamination analysis",
       "detail": "All seed datasets (GSM8K, CAMEL, AlpacaEval, CodeAlpaca) were publicly available before the evaluated models' training periods. The evolved instructions derive from these. No analysis of whether models have seen these benchmarks or similar problems in training, which could undermine the validity of the discrepancy-based selection."
     },
     {
       "flag": "Small evaluator pool",
       "detail": "Only 13 STEM postgraduates from Chinese universities serve as evaluators for all 1,120 comparisons, with inter-annotator agreement of 83.39%. This narrow demographic may not represent the broader user population, and the small pool limits generalizability of the human preference signal."
     },
     {
       "flag": "No uncertainty quantification on main results",
       "detail": "Despite performing 1000 bootstrap resamples for Elo aggregation, no confidence intervals, error bars, or standard deviations are reported in any results table. The reader cannot assess the stability of the reported rankings."
     },
     {
       "flag": "Scaling claim based on LLM-based evaluation, not human evaluation",
       "detail": "The 20-model experiment (Table 7) uses GPT-4o as judge, not human evaluators. The scaling claim is predicated on LLM-based evaluation, which introduces the very biases the paper argues human evaluation should address."
     }
   ],
   "cited_papers": [
     {
       "title": "Chatbot Arena: An open platform for evaluating LLMs by human preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng", "Ying Sheng"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Major LLM evaluation platform using Elo-based human preference ranking; serves as gold-standard comparison for the proposed method."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Foundational work on using LLMs as evaluation judges and MT-Bench as a conversation benchmark."
     },
     {
       "title": "Humans or LLMs as the judge? A study on judgement biases",
       "authors": ["Guiming Hardy Chen", "Shunian Chen", "Ziche Liu"],
       "year": 2024,
       "arxiv_id": "2402.10669",
       "relevance": "Documents biases in LLM-based evaluation including position, format, verbosity, and self-enhancement bias."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Yupeng Chang", "Xu Wang", "Jindong Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM evaluation methods spanning benchmarks, automated evaluation, and human assessment."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the primary models evaluated and used as evaluation infrastructure in this study."
     },
     {
       "title": "AlpacaFarm: A simulation framework for methods that learn from human feedback",
       "authors": ["Yann Dubois", "Xuechen Li", "Rohan Taori"],
       "year": 2023,
       "arxiv_id": "2305.14387",
       "relevance": "Framework using GPT-4 as LLM-as-judge for evaluating instruction-following; source of evaluation methodology."
     },
     {
       "title": "PandaLM: An automatic evaluation benchmark for LLM instruction tuning optimization",
       "authors": ["Yidong Wang", "Zhuohao Yu", "Zhengran Zeng"],
       "year": 2023,
       "arxiv_id": "2306.05087",
       "relevance": "Open-source LLM-based pairwise comparison evaluator, alternative to proprietary model judges."
     },
     {
       "title": "Prometheus: Inducing fine-grained evaluation capability in language models",
       "authors": ["Seungone Kim", "Jamin Shin", "Yejin Cho"],
       "year": 2023,
       "relevance": "Open-source LLM evaluator with fine-grained scoring rubrics for customizable evaluation criteria."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "RLHF methodology for training instruction-following LLMs; foundational work for the evaluated models."
     },
     {
       "title": "Label-efficient model selection for text generation",
       "authors": ["Shir Ashury-Tahan", "Ariel Gera", "Benjamin Sznajder"],
       "year": 2024,
       "arxiv_id": "2402.07891",
       "relevance": "DiffUse method for label-efficient LLM evaluation using embedding-difference clustering; baseline comparison."
     },
     {
       "title": "Anchor points: Benchmarking models with much fewer examples",
       "authors": ["Rajan Vivek", "Kawin Ethayarajh", "Diyi Yang"],
       "year": 2023,
       "arxiv_id": "2309.08638",
       "relevance": "K-Medoids-based sample selection for efficient LLM benchmarking; baseline comparison method."
     },
     {
       "title": "An in-depth look at Gemini's language abilities",
       "authors": ["Syeda Nahida Akter", "Zichun Yu", "Aashiq Muhamed"],
       "year": 2023,
       "arxiv_id": "2312.11444",
       "relevance": "Detailed evaluation of Gemini's capabilities including HumanEval pass@1 rates referenced in this paper."
     }
   ]
 }