ai-research-survey

scan.json (22429B)

---

 {
   "paper": {
     "title": "Copilot Arena: A Platform for Code LLM Evaluation in the Wild",
     "authors": ["Wayne Chi", "Valerie Chen", "Anastasios Nikolas Angelopoulos", "Wei-Lin Chiang", "Aditya Mittal", "Naman Jain", "Tianjun Zhang", "Ion Stoica", "Chris Donahue", "Ameet Talwalkar"],
     "year": 2025,
     "venue": "",
     "arxiv_id": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repo provided: https://github.com/lm-sys/copilot-arena, and VSCode extension at https://lmarena.ai/copilot."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we will release a curated dataset' and 'Upon publication, we will also open-source more diverse examples' — future promises count as NO. Privacy concerns limit full data release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the system but does not give instructions for replicating the experiments or analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are computed via bootstrapping for the BT model rankings (Section 4.1, Table 4, Table 5). Lower and upper bounds are reported for each model's beta estimate."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Spearman rank correlations are reported but no formal significance tests (p-values) are provided for claims that rankings differ between platforms or that win-rate changes are 'significant' — the paper uses a percentile threshold (top 90th) rather than a statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Spearman rank correlation coefficients are reported (rs = 0.62, 0.48, ≤0.1) for leaderboard comparisons. BT coefficients with confidence intervals provide magnitude context. Win-rate differences are quantified."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports 11k+ pairwise judgments and 4.5M suggestions but does not justify whether this sample size is sufficient for the claims made, nor is a power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap confidence intervals across 100 rounds are reported in Tables 4 and 5, showing spread of BT coefficient estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Copilot Arena rankings against multiple existing evaluations: Chatbot Arena (general and coding), LiveBench, LiveCodeBench, and BigCodeBench (Section 4.2, Figure 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent and maintained benchmarks: LiveBench, LiveCodeBench, BigCodeBench, and Chatbot Arena — all contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablations: style-controlled BT model (Appendix D, Table 5), Snip-It vs non-Snip-It prompting (Table 2), and win-rate analysis across data partitions (task type, context length, FiM, language) in Section 5.2."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: BT coefficients, win-rates, Spearman rank correlation, pass@1 for prompting evaluation (Table 2), and pairwise win-rate matrices."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire platform is built around human evaluation — users provide pairwise preference judgments on code completions in their IDE. Over 11k judgments collected."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a live preference platform, not a train/test paradigm. There is no model training on the collected data in this paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns by task category, programming language, context length, FiM vs completion-only, and natural language (Section 5, Figures 6 and 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2.3 shows examples of errors without Snip-It. The paper also discusses completion order bias (86% first-completion preference) in Appendix C and discusses where smaller models underperform."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that models explicitly trained for infilling 'do not experience large changes to win-rate' (Section 5.2), which is a surprising negative result. Also reports low correlation with static benchmarks (rs ≤ 0.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about rankings differing from existing evaluations are supported by Spearman correlations in Section 4.2. Claims about consistency across programming languages and variation by task are supported by Figure 7 and Section 5.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims differences are attributable to 'the more realistic distribution of data and tasks' (abstract) — this is a causal claim from observational data without controlling for confounds like user population differences, model API differences, or selection effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section (Section 7) explicitly states 'it is unclear to what extent our results encapsulate all real-world use cases' and notes the platform doesn't perfectly mirror real-world tools like GitHub Copilot."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 considers confounding variables like response length and latency (style-controlled BT ablation in Appendix D). The paper also discusses completion order bias as an alternative explanation for preferences. Limitations section acknowledges platform differences."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 in Appendix B provides full model names with version identifiers: claude-3-5-sonnet-20240620, gpt-4o-2024-08-06, gpt-4o-mini-2024-07-18, codestral-2405, gemini-1.5-flash-002, gemini-1.5-pro-002, etc."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates with examples are provided in Appendix A.2 (PSM, SPM, Mask, IPF formats) including the actual prompt text used for code completion, and task detection prompts in Appendix C."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, or other sampling parameters used when calling the LLM APIs for code completion. The 0.5 second delay and 8000 token input limit are mentioned but not model generation hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The system is a code completion platform, not an agentic scaffolding setup. Models are called directly for single-turn completions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents data preprocessing: natural language detection method (lingua detector, confidence > 0.7), programming language detection (file extension), task clustering pipeline (multi-step LLM classification), and filtering criteria for completion bias analysis (outlier removal)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Discussion' begins with a dedicated 'Limitations' subsection with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: pairwise completions and slower latency don't perfectly mirror real-world platforms, unable to include GitHub Copilot because its model is not available via API, privacy considerations limit data release, completion order bias (86% first-completion selection)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'this should not be treated as the sole defining metric of model quality, but instead an informative one' and acknowledges inability to include GitHub Copilot and that results may not encapsulate all real-world use cases."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is not released due to privacy concerns. The paper promises future curated releases with PII removal but does not provide data access at time of publication."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 2-3 describe data collection: VSCode extension deployed to real users, pairwise completion interface, voting mechanism, privacy settings (Appendix B). Collection period and scale (4.5M suggestions, 11k+ votes) are stated."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how users were recruited to install the Copilot Arena VSCode extension, whether this introduces selection bias, or what channels were used for recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from user interaction to analysis is documented: code context capture → model routing → completion generation → user voting → BT analysis. Data filtering for analysis is described (Appendix C: language detection, task clustering, outlier removal for bias analysis)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Acknowledgments' lists NSF grants (IIS1705121, IIS1838017, IIS2046613, IIS2112471) and funding from Sony AI, Meta, Morgan Stanley, Amazon, Google, and Scribe."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (Carnegie Mellon University and UC Berkeley) are clearly listed. No authors appear affiliated with the model providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funders include Google, Amazon, and Meta — companies whose models (Gemini, GPT-4o via partnership) are evaluated in the leaderboard. This creates a potential conflict of interest that is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates models on live user data, not on a benchmark that could be in training data. Contamination is not a concern for a live preference platform."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — user code contexts are live and novel, not a pre-existing benchmark dataset."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a live evaluation platform, not a static benchmark. The paper actually positions itself as an alternative that avoids contamination issues."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite collecting data from human users and their code."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No user demographics are reported — no information on experience level, geographic distribution, or professional background of the developers using the platform."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for participants are stated. Anyone who installs the VSCode extension can participate."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the model routing/sampling strategy: models are randomly paired for each comparison, with a latency-optimized sampling strategy described."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Users are blinded to model identities during evaluation — they see two anonymous completions and only learn model identities after voting (Appendix B.1: 'After the user votes, we reveal the model pair')."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on how many users installed the extension vs. how many actually provided votes, or user dropout/retention over time."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, per-completion costs, or latency measurements are reported for running the platform, despite serving 4.5 million suggestions across 10 models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or infrastructure costs are stated for operating the platform."
       }
     }
   },
   "claims": [
     {
       "claim": "Model rankings from Copilot Arena differ from those of existing evaluations, with highest correlation (rs = 0.62) with Chatbot Arena coding and low correlation (rs ≤ 0.1) with static benchmarks.",
       "evidence": "Section 4.2 and Figure 5 show Spearman rank correlations between Copilot Arena and other leaderboards across 10 models.",
       "supported": "moderate"
     },
     {
       "claim": "Downstream task type significantly affects relative model win-rates (31.1% of possible changes), while programming language has little effect (6.7% of possible changes).",
       "evidence": "Section 5.2 and Figure 7 show win-rate difference analysis across data partitions with a top-90th-percentile threshold.",
       "supported": "moderate"
     },
     {
       "claim": "Models explicitly trained for infilling do not experience large changes to win-rate on FiM vs non-FiM tasks.",
       "evidence": "Section 5.2 and Table 6 show DeepSeek Coder performance remains consistent when using Chat API with prompting scheme vs native FiM.",
       "supported": "moderate"
     },
     {
       "claim": "Smaller models tend to overperform on data similar to static benchmarks compared to Copilot Arena's realistic distribution.",
       "evidence": "Section 4.2 and 5.2 discuss GPT-4o mini and Qwen-2.5 Coder performing worse on frontend/backend tasks, longer contexts, and non-Python settings in Copilot Arena.",
       "supported": "moderate"
     },
     {
       "claim": "The Snip-It prompting method improves pass@1 for most models and prompt templates.",
       "evidence": "Table 2 in Appendix A.2.2 shows pass@1 comparisons across 9 models and 4 prompt templates with and without Snip-It.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Copilot Arena is a VSCode-integrated platform for evaluating code LLMs via live pairwise preference judgments, collecting 11k+ votes across 10 models. Rankings differ substantially from static benchmarks (Spearman rs ≤ 0.1) but correlate moderately with Chatbot Arena coding (rs = 0.62). Task type is the strongest factor affecting model rankings, while programming language has minimal effect. Code-specific models like DeepSeek Coder are competitive with general-purpose SOTA models, and smaller models tend to overperform on static benchmark-like data compared to realistic IDE usage.",
   "red_flags": [
     {
       "flag": "Completion order bias",
       "detail": "Users selected the first completion 86% of the time. While the paper investigates this and finds non-trivial decision times, such a strong positional bias could significantly affect rankings if model assignment to positions is not perfectly balanced."
     },
     {
       "flag": "No IRB approval for human subjects research",
       "detail": "The platform collects code and preference data from users but does not mention IRB or ethics review, despite this constituting human subjects research."
     },
     {
       "flag": "Funders include evaluated model providers",
       "detail": "Google and Amazon are listed as funders, and Google's Gemini models are evaluated in the leaderboard. This conflict is not acknowledged."
     },
     {
       "flag": "Unknown user population",
       "detail": "No demographics or recruitment methods described. Self-selected users who install a research VSCode extension may not be representative of professional developers, limiting generalizability claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-judge with MT-bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang"],
       "year": 2023,
       "relevance": "Foundational work on arena-based LLM evaluation via human preferences that Copilot Arena builds upon."
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["Wei-Lin Chiang"],
       "year": 2024,
       "arxiv_id": "2403.04132",
       "relevance": "Primary comparison platform for human preference evaluation of LLMs."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Live coding benchmark designed to reduce contamination, used as baseline comparison."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Code generation benchmark used as baseline comparison for model rankings."
     },
     {
       "title": "AI Agents That Matter",
       "authors": ["Sayash Kapoor", "Benedikt Stroebl"],
       "year": 2024,
       "arxiv_id": "2407.01502",
       "relevance": "Discusses evaluation methodology concerns for AI agents, directly relevant to survey scope."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Major study on AI coding assistant productivity impact."
     },
     {
       "title": "The RealHumanEval: Evaluating Large Language Models' Abilities to Support Programmers",
       "authors": ["Hussein Mozannar", "Valerie Chen"],
       "year": 2024,
       "arxiv_id": "2404.02806",
       "relevance": "Human-centric evaluation of LLMs for programming support, directly comparable methodology."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark — foundational code generation evaluation that Copilot Arena aims to complement."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2023,
       "relevance": "Major real-world coding benchmark for evaluating LLM software engineering capabilities."
     },
     {
       "title": "The Productivity Effects of Generative AI: Evidence from a Field Experiment with GitHub Copilot",
       "authors": ["Kevin Zheyuan Cui", "Mert Demirer"],
       "year": 2024,
       "relevance": "Field experiment on coding assistant productivity, complementary evaluation methodology."
     },
     {
       "title": "LiveBench: A Challenging, Contamination-Free LLM Benchmark",
       "authors": ["Colin White", "Samuel Dooley"],
       "year": 2024,
       "arxiv_id": "2406.19314",
       "relevance": "Live benchmark designed to avoid contamination, used as baseline comparison."
     }
   ]
 }