ai-research-survey

scan.json (20958B)

---

 {
   "paper": {
     "title": "From Crowdsourced Data to High-Quality Benchmarks: Arena-Hard and BenchBuilder Pipeline",
     "authors": ["Tianle Li", "Wei-Lin Chiang", "Evan Frick", "Lisa Dunlap", "Tianhao Wu", "Banghua Zhu", "Joseph E. González", "Ion Stoica"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2406.11939"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/lmarena/arena-hard-auto (Section 1, footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Arena-Hard-Auto benchmark of 500 prompts is released as part of the open-source repository. The source data (Chatbot Arena, WildChat-1M) are publicly available datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section found in the paper. Only mentions of specific models and APIs used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The pipeline is described at a high level but specific commands or scripts to replicate results are not included."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are computed via 100 rounds of bootstrapping on judgment results (Section 6.1). Confidence intervals shown in Figure 5 and used throughout the metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal significance tests (p-values, t-tests, etc.) are used. Comparisons between benchmarks rely on the proposed metrics (separability, agreement) without statistical tests of whether differences are significant."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are provided in context: e.g., '3x higher separation' compared to MT-Bench, specific percentage differences in separability (87.4% vs 22.6%), agreement (90.9% vs 26.6%), and Brier scores (0.069 vs 0.09)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 500 prompts were chosen, why 250 clusters were sampled, or why 20 models were used for evaluation. These are stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Variance is captured through bootstrapped confidence intervals (100 rounds of bootstrapping). Confidence intervals are shown in Figure 5 and used in the separability metric."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons against MT-Bench, AlpacaEval 2.0 LC, and Chatbot Arena (Table 1). Random baselines also compared (Table 2, Appendix Table 7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MT-Bench (2023), AlpacaEval 2.0 LC (2024), and Chatbot Arena are all contemporary and widely-used LLM benchmarks at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Several ablation-like analyses: quality score threshold effects (Figure 3), random vs. curated baselines (Table 2, Table 7), different annotators (Llama-3-70B vs GPT-4-Turbo, Table 8), different judges (Table 4), and style control effects (Table 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Separability, Confidence Agreement, Spearman Correlation, Kendall Tau Correlation, and Brier Score (Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Chatbot Arena human preference rankings serve as the ground truth against which Arena-Hard-Auto is validated. The entire framework is evaluated by how well it aligns with human judgments."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark construction paper, not a model training paper. The concept of held-out test sets does not apply in the traditional sense."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Topic cluster analysis provided (Figure 4), per-model breakdowns in multiple tables, per-quality-score breakdowns (Figure 3), and per-judge breakdowns (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Stylistic biases (length bias, self-bias) are discussed as failure modes in Sections 6.5 and 6.6. Table 5 shows how style manipulation can game the benchmark without style control."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Self-bias of GPT-4-Turbo judge is reported (Section 6.6, Appendix Table 10) showing it favors OpenAI models. Claude-3-Opus and Llama-3-70B perform worse as judges (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '3x higher separation' supported by Table 1 (87.4% vs 22.6%), '98.6% correlation' supported by Table 3/Appendix Table 9, '$20 cost' stated in Section 4.3/Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about the quality score filtering improving benchmark quality. This is supported by controlled comparisons: curated vs. random baselines (Table 2, Table 7), and ablations across quality score thresholds (Figure 3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section (Section 7) explicitly states the benchmark 'currently lacks evaluation for multi-turn and non-English interactions' and acknowledges the seven qualities 'may not fully capture the range of possible attributes, potentially skewing towards prompts in technical domains.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses stylistic bias as an alternative explanation for benchmark scores (Section 6.5), self-bias of judges (Section 6.6), and tests whether results hold under style control (Table 3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions provided throughout: gpt-4-0314, gpt-4-1106-preview, gpt-4-turbo-2024-04-09, claude-3-opus-20240229, claude-3-sonnet-20240229, gemini-1.5-pro-0514, llama-3-70b-instruct, etc. (footnote 4, Table 4)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates for the judge are stated to be in Section C (Appendix). Style control system prompts are provided in Table 6 (e.g., 'You are a helpful assistant who thoroughly explains things with as much detail as possible.')."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other sampling parameters reported for LLM API calls used in judging or annotation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The pipeline is a data curation workflow, not an agent system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data pipeline documented: 200,000 initial prompts from Chatbot Arena, filtering of duplicates/multi-turn/non-English, clustering into 4,000 topics, quality scoring, threshold filtering (score <6 removed, cluster mean <5 removed), sampling 2 per cluster from 250 clusters (Section 4.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated Section 7 'Limitations' discusses biases in the pipeline, lack of multi-turn and non-English coverage, and potential skew toward technical domains."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats identified: seven quality criteria may skew toward technical domains, lack of multi-turn data in crowdsourced sources, primary language proficiency of authors limiting non-English evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what is NOT covered: multi-turn interactions, non-English evaluation. The benchmark is bounded to single-turn English queries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark prompts are released via the GitHub repository. Source datasets (Chatbot Arena conversations, WildChat-1M) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sourced from Chatbot Arena (200,000 prompts) and WildChat-1M (150,000 queries). Collection via crowdsourced live platforms described in Sections 4.1-4.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study. Data comes from existing crowdsourced platforms (Chatbot Arena, WildChat). Human preferences used as ground truth are from the existing Chatbot Arena platform."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: embedding with text-embedding-3-small, UMAP dimensionality reduction, HDBSCAN clustering into 4,000 topics, LLM quality scoring on 7 criteria, threshold filtering, balanced sampling across clusters (Section 4.1-4.2, Figure 2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed with UC Berkeley affiliation. Authors are also affiliated with the LMSYS Chatbot Arena project, which is the source of the benchmark data."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. The authors operate the Chatbot Arena platform whose data and rankings are used as ground truth, creating a potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs on the benchmark but does not state training data cutoff dates for any of the models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Arena-Hard-Auto prompts (sourced from Chatbot Arena) could overlap with training data of the evaluated models. Chatbot Arena conversations are public and could be in training sets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 discusses benchmark leakage as a motivation. The paper argues BenchBuilder enables 'continuous benchmark updates' to address contamination risk of static benchmarks. However, contamination of Arena-Hard-Auto itself is not directly tested."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study. Human preference data comes from existing Chatbot Arena platform."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation cost per model reported as $20 (Table 1, abstract). Pipeline annotation cost reported as ~$500 with GPT-4-Turbo or ~$45 with Llama-3-70B (Section 4.3)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "API costs are reported but total computational budget (GPU hours for embeddings, clustering, etc.) is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Arena-Hard-Auto achieves 98.6% correlation with Chatbot Arena human preference rankings.",
       "evidence": "Table 3 shows 98.6% Confidence Agreement and 98.6% Spearman Correlation with style-controlled Chatbot Arena English Hard Prompts ranking. Table 1 shows 93.2% Spearman and 90.9% Confidence Agreement with overall Chatbot Arena.",
       "supported": "strong"
     },
     {
       "claim": "Arena-Hard-Auto provides 3x higher separation of model performances compared to MT-Bench.",
       "evidence": "Table 1: Arena-Hard-Auto separability 87.4% vs MT-Bench 22.6% (3.87x). Figure 5 visually demonstrates the difference.",
       "supported": "strong"
     },
     {
       "claim": "Arena-Hard-Auto evaluation costs only $20 per model.",
       "evidence": "Table 1 lists eval cost per model as $20. Section 4.3 provides cost breakdown for the pipeline annotation step.",
       "supported": "strong"
     },
     {
       "claim": "Style control effectively neutralizes gaming via response length or markdown manipulation.",
       "evidence": "Table 5 shows that with style control, the 'detail' variant of Llama-3.1-70B no longer outperforms the base model, whereas without style control it gains ~9 points.",
       "supported": "strong"
     },
     {
       "claim": "BenchBuilder generalizes to different data sources (WildChat).",
       "evidence": "Table 2 shows Wild-Hard-Auto (from WildChat) achieves 86.7% separability and 88.6% confidence agreement vs 75.6% and 36.4% for random baseline.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces BenchBuilder, an automated pipeline for curating LLM benchmarks from crowdsourced data, and Arena-Hard-Auto, a 500-prompt benchmark. Arena-Hard-Auto achieves 87.4% separability (vs 22.6% for MT-Bench) and up to 98.6% agreement with human preferences at $20 per model evaluation. The paper also proposes style control methods that effectively mitigate length and formatting biases in LLM-as-a-judge evaluation, and demonstrates the pipeline generalizes across data sources (Chatbot Arena, WildChat).",
   "red_flags": [
     {
       "flag": "Authors evaluate their own platform",
       "detail": "The authors operate LMSYS Chatbot Arena, whose rankings serve as ground truth for validating Arena-Hard-Auto. This circular dependency means the benchmark is validated against a system the authors control. High agreement could partly reflect shared biases in data sourcing and evaluation methodology."
     },
     {
       "flag": "Contamination of Arena-Hard-Auto not directly tested",
       "detail": "While benchmark leakage is discussed as motivation, the paper does not test whether Arena-Hard-Auto prompts (sourced from public Chatbot Arena conversations) appear in training data of the evaluated models. Given Chatbot Arena data is publicly scraped, this is a real risk."
     },
     {
       "flag": "No hyperparameters for LLM calls",
       "detail": "Temperature, top-p, and other sampling parameters for the judge and annotator LLM calls are not reported, despite these being known to significantly affect output."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-judge evaluation methodology that Arena-Hard-Auto builds upon."
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng", "Ying Sheng"],
       "year": 2024,
       "relevance": "The crowdsourced human evaluation platform whose data and rankings serve as ground truth for this work."
     },
     {
       "title": "Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators",
       "authors": ["Yann Dubois", "Balázs Galambosi", "Percy Liang", "Tatsunori B. Hashimoto"],
       "year": 2024,
       "arxiv_id": "2404.04475",
       "relevance": "Key baseline benchmark and introduces length bias control methods that Arena-Hard-Auto extends."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Prominent task-based LLM benchmark for code, relevant to the survey's scope of AI coding evaluation."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Contemporary code generation benchmark relevant to evaluating LLM programming capabilities."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Live benchmark addressing contamination in code evaluation, directly relevant to benchmark methodology quality."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu"],
       "year": 2023,
       "relevance": "Benchmark for evaluating LLMs in agentic settings, relevant to the survey's coverage of agent evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, foundational to LLM code evaluation methodology."
     },
     {
       "title": "NLP Evaluation in Trouble: On the Need to Measure LLM Data Contamination for Each Benchmark",
       "authors": ["Oscar Sainz"],
       "year": 2023,
       "relevance": "Directly addresses benchmark contamination measurement, a key methodological concern in the survey."
     },
     {
       "title": "WildBench: Benchmarking LLMs with Challenging Tasks from Real Users in the Wild",
       "authors": ["Bill Yuchen Lin"],
       "year": 2024,
       "arxiv_id": "2406.04770",
       "relevance": "Contemporary benchmark using real user queries for LLM evaluation, comparable approach to Arena-Hard-Auto."
     }
   ]
 }