ai-research-survey

scan.json (29084B)

---

 {
   "paper": {
     "title": "Remote Labor Index: Measuring AI Automation of Remote Work",
     "authors": [
       "Mantas Mazeika",
       "Alice Gatti",
       "Cristina Menghini",
       "Udari Madhushani Sehwag",
       "Shivam Singhal",
       "Yury Orlovskiy",
       "Steven Basart",
       "Manasi Sharma",
       "Denis Peskoff",
       "Elaine Lau",
       "Jaehyuk Lim",
       "Lachlan Carroll",
       "Alice Blair",
       "Vinaya Sivakumar",
       "Sumana Basu",
       "Brad Kenstler",
       "Yuntao Ma",
       "Julian Michael",
       "Xiaoke Li",
       "Oliver Ingebretsen",
       "Aditya Mehta",
       "Jean Mottola",
       "John Teichmann",
       "Kevin Yu",
       "Zaina Shaik",
       "Adam Khoja",
       "Richard Ren",
       "Jason Hausenloy",
       "Long Phan",
       "Ye Htet",
       "Ankit Aich",
       "Tahseen Rabbani",
       "Vivswan Shah",
       "Andriy Novykov",
       "Felix Binder",
       "Kirill Chugunov",
       "Luis Ramirez",
       "Matias Geralnik",
       "Hernán Mesura",
       "Dean Lee",
       "Ed-Yeremai Hernandez Cardona",
       "Annette Diamond",
       "Summer Yue",
       "Alexandr Wang",
       "Bing Liu",
       "Ernesto Hernandez",
       "Dan Hendrycks"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.26787"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation platform code is open-sourced (Section 3.4: 'The code for the evaluation platform is open-sourced'). However, the benchmark itself is largely private (230/240 projects in private test set)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Only 10 of 240 projects are publicly released. The private test set of 230 projects is withheld to prevent contamination (Section 3.2). The benchmark data is largely not available for independent verification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper describes agent scaffolds at a high level but does not specify reproducible environment configurations."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The private test set prevents full reproduction, and the paper does not provide scripts or commands to replicate the evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 shows 95% confidence intervals for Elo scores computed via 100 bootstrap samples (Section B.2: 'We use 100 bootstrap samples to compute 95% confidence intervals')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used to compare model automation rates. Differences between models (e.g., 2.5% vs 2.1%) are presented without any test of whether they are statistically distinguishable."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported in context: automation rates as percentages, dollars earned, and Elo differences with the interpretation that '400 corresponds to 10:1 odds of winning' (Section 3.3)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark size of 240 projects is not justified via power analysis or other formal reasoning. No discussion of whether 240 projects is sufficient to distinguish models at the observed ~1-2.5% automation rates."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported for automation rates. Elo scores have bootstrap CIs, but automation rates are reported as single point estimates without any spread measure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The human gold-standard deliverable serves as the baseline. Multiple AI agents are compared against each other and against this human baseline (Section 3.3, Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include frontier agents as of 2025: GPT-5, Claude Sonnet 4.5, Grok 4, Gemini 2.5 Pro, Manus, and ChatGPT agent. These are current state-of-the-art systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not systematically vary benchmark components (e.g., task complexity, category) to understand which factors drive the low automation rates."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: automation rate, Elo score, dollars earned, and autoflation (Section 3.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation is manual. Trained human evaluators assess AI deliverables against human gold standards using a 3-point scale, with majority voting across 3 independent evaluations (Section 3.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "230 of 240 projects are maintained as a private test set, with only 10 released publicly (Section 3.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as aggregate automation rates and Elo scores. No per-category breakdown (e.g., by Upwork category) is provided in the main tables or appendix, despite 23 categories being represented."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides qualitative analysis of failure modes with specific examples: truncated videos, child-like drawings, inconsistent 3D renderings, robotic voiceovers. Table 2 gives failure category frequencies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is essentially reporting negative results — all agents perform near the floor (max 2.5% automation rate). GPT-5 CUA underperformed CLI (Appendix A.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'highest-performing agent achieving an automation rate of 2.5%' which matches Table 1 (Manus at 2.5%). The claim of 'near the floor' is supported by all models being below 3%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims about AI capability limitations ('current systems remain far from capable'), but the low scores could be partly due to scaffold/tooling limitations, evaluation criteria strictness, or format compatibility issues rather than fundamental capability gaps. This is not adequately addressed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 5) explicitly states RLI 'does not represent several types of remote work' and lists excluded categories (tutoring, team projects, client interaction). The title specifies 'Remote Work' rather than all work."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for low automation rates. Scaffold limitations are briefly mentioned (Appendix A.3) but not explored as a systematic confound. Format compatibility, evaluation strictness, and prompt engineering quality are not discussed as factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit that automation rate measures whether deliverables would be 'accepted by a reasonable client as the commissioned work' (Section 3.3) — a specific proxy for economic automation. Section 5 discusses the gap between task-specific automation and general labor automation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'Claude Sonnet 4.5', 'GPT-5', 'Gemini 2.5 Pro', 'Grok 4'. No API versions, snapshot dates, or exact model identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix B.6 for the base prompt, OpenHands extensions, computer-use agent prompt, and Claude-specific extensions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, session timeout details beyond 'default: 1 hour') are reported for the model API calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Agent scaffolds are described: OpenHands CLI environment, Scale AI computer-use scaffold with MCP tools (mouse, keyboard, bash, file editor), and integrated agents (Manus, ChatGPT agent). Available tools including gpt-image-1, tts-1, and veo-3.0 are listed (Appendix B.6)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Appendix C.2 document the extensive filtering pipeline: 550 initial projects → multiple rounds of review, cleaning, and standardization → 240 final projects. Filtering criteria are stated in detail."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Discussion' contains a 'Limitations' subsection discussing excluded work types and cost reporting limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: exclusion of interactive work (tutoring), team work (project management), work requiring time to evaluate (SEO), and the fact that costs may be underestimated due to inflation (Section 5)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states: 'an AI obtaining 100% automation rate on RLI may still underperform humans on types of work that we do not evaluate.' The excluded categories are enumerated in Appendix C.2."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The private test set of 230 projects is not available. Only 10 projects are released publicly. Individual evaluator annotations are not released. Independent verification of the results is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the two-stage collection process: freelance platform sourcing (207 projects from Upwork) and long-tail sourcing (40 additional projects). The pipeline is visualized in Figure 5."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes recruiting 358 freelancers with verified Upwork accounts: 'on average, they had 2,341 hours worked, 89 prior jobs, and $23,364 in total earnings.' Freelancers were paid $15-$200 per project (average $41)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 and Section 3.2 document the pipeline: 550 collected → cleaning → spot check → final filter → 240 final tasks. Appendix C.2 lists filtering criteria and specific stages."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section listing grants or sponsors. The work is affiliated with Scale AI and CAIS but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with Scale AI and Center for AI Safety are clearly listed. Some authors are noted as having done work while at Scale AI or CAIS."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Scale AI is a data labeling and AI evaluation company with a direct commercial interest in AI benchmark results. Multiple authors are Scale AI employees. The funder has a stake in demonstrating the value of AI evaluation services and human labor for AI tasks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is provided. Scale AI co-founder Alexandr Wang is a senior author, and Scale AI has obvious financial interests in the AI evaluation space."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states: 'None of the project descriptions in RLI are searchable' and describes contamination prevention measures including a blocklist of domains for long-tail data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes multiple contamination prevention strategies: private test set of 230 projects, non-searchable project descriptions, domain blocklist for online deliverables. The benchmark is designed to be contamination-resistant."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper. While freelancers provided work samples and evaluators annotated deliverables, neither group is studied as human subjects — they are data providers and annotators respectively."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Freelancers sold work samples; evaluators performed annotation work."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Freelancer statistics (hours worked, prior jobs, earnings) are reported as data source quality indicators, not participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 12 shows the distribution of model running costs: 'average cost of generating AI deliverables was $2.34' with max $29.51."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Figure 12 shows API cost distribution. Session timeout is stated as 1 hour default (Appendix B.6). The total human labor cost of the benchmark is stated: 'over 6,000 hours of real work valued at over $140,000' (Section 3.1)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of random seed sensitivity. Each model appears to have been run once per project with no analysis of result variability across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of generation runs per model per project is not explicitly stated. It appears each model generated one deliverable per project, but this is not confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper mentions tuning prompts ('we tune prompts and provide standardized tooling scaffolds', Section 4.1) but does not report how many prompt variants were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "For GPT-5, two scaffolds are compared and the better one (CLI) is reported in main tables. However, the selection process for prompt tuning and scaffold configuration is not documented."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Scale AI developed the computer-use scaffold and the evaluation platform, and is evaluating models on their own benchmark. No discussion of self-comparison bias or author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models are given different scaffolds and tools but performance is not analyzed as a function of compute budget. Some models may have spent more tokens/compute than others, which is not controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Figure 6 extensively discuss what RLI measures vs. what prior benchmarks measure. The paper compares RLI's task distribution to actual Upwork distribution, arguing it better captures real remote labor than prior benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different models use different scaffolds (OpenHands CLI, Scale AI CUA, integrated agents) but this confound is not systematically addressed. Only GPT-5 is tested on two scaffolds. The paper acknowledges 'more vertical integration of model scaffolds will yield stronger performance' (Appendix A.3) but does not control for scaffold differences when comparing models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses private projects sourced from freelancers' past work, with non-searchable descriptions and a domain blocklist. This inherently addresses temporal leakage since the projects are not in public training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation compatibility prompt or format specifications could leak information about what constitutes a successful deliverable."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether projects from the same freelancer or category share structural similarities that could affect results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Concrete prevention methods are used: private test set, non-searchable project descriptions, and domain blocklist for online deliverables (Section 3.2)."
       }
     }
   },
   "claims": [
     {
       "claim": "The best-performing AI agent (Manus) achieves an automation rate of only 2.5% on RLI",
       "evidence": "Table 1 shows Manus at 2.5%, Grok 4 and Sonnet 4.5 at 2.1%, GPT-5 at 1.7%, ChatGPT agent at 1.3%, Gemini 2.5 Pro at 0.8%",
       "supported": "strong"
     },
     {
       "claim": "RLI is substantially more complex than prior benchmarks, with median completion time of 11.5 hours exceeding previous benchmarks by more than 2x",
       "evidence": "Figure 4 shows completion time distribution; Figure 6 compares against HCAST and GDPval, showing RLI matches Upwork distribution",
       "supported": "strong"
     },
     {
       "claim": "Inter-annotator agreement is 94.4% for automation rate evaluation",
       "evidence": "Section 3.4 states this figure. Section B.5 reports auditing of false positives and estimation of false negative rate ≤5.8% with 95% confidence",
       "supported": "strong"
     },
     {
       "claim": "AI agents are steadily improving as measured by Elo scores, with newer models achieving higher scores",
       "evidence": "Figure 8 shows Elo progression from Gemini 2.5 Pro (~412) to Manus (~510), with 95% CIs from bootstrap",
       "supported": "moderate"
     },
     {
       "claim": "RLI captures the true diversity and complexity of remote labor markets better than prior benchmarks",
       "evidence": "Figure 6 shows RLI's project type distribution is more diverse than HCAST and GDPval; Figure 14 shows more unique file types; completion time matches Upwork distribution",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The Remote Labor Index (RLI) benchmark of 240 real-world freelance projects shows that frontier AI agents achieve at most 2.5% automation rate, with the best model (Manus) earning only $1,720 of $143,991 in potential project value. Common failure modes include corrupted files (17.6%), incomplete deliverables (35.7%), poor quality (45.6%), and inconsistencies (14.8%). AI successes cluster in audio editing, image generation, data visualization, and writing — domains where current models already have strong skills — while complex multimodal work like architecture, 3D modeling, and video production remains far beyond current capabilities.",
   "red_flags": [
     {
       "flag": "Company evaluating benchmark relevance to its own business",
       "detail": "Scale AI is a data labeling and AI evaluation company. The benchmark's conclusion — that human evaluation is needed because AI cannot automate most work — directly supports Scale AI's business model. Co-founder Alexandr Wang is a senior author. This conflict of interest is not disclosed or discussed."
     },
     {
       "flag": "Scaffold confound across model comparisons",
       "detail": "Different models use different scaffolds (OpenHands, Scale AI CUA, integrated agents) but are compared as if differences reflect model capability. Only GPT-5 is tested on two scaffolds, showing scaffold choice matters (CLI outperforms CUA). The other models' rankings could partly reflect scaffold quality."
     },
     {
       "flag": "No per-category breakdown despite 23 categories",
       "detail": "The paper reports only aggregate automation rates. With 240 projects across 23 categories, per-category results would reveal whether low scores are uniform or driven by specific hard categories. The qualitative analysis hints at category-dependent performance but this is not quantified."
     },
     {
       "flag": "Single-run results",
       "detail": "Each model appears to have generated one deliverable per project. LLM outputs are stochastic; a single run could underperform or overperform. No variance analysis is provided for automation rates."
     },
     {
       "flag": "Private benchmark limits independent verification",
       "detail": "230 of 240 projects are private. While this protects against contamination, it also means the results cannot be independently verified. The benchmark design, task selection, and evaluation quality must be taken on trust."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major software engineering benchmark that RLI positions against as more limited in scope"
     },
     {
       "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?",
       "authors": ["Samuel Miserendino", "Michele Wang", "Tejal Patwardhan"],
       "year": 2025,
       "arxiv_id": "2502.12115",
       "relevance": "Economically-grounded software engineering benchmark that directly inspired RLI's approach to measuring economic value"
     },
     {
       "title": "GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks",
       "authors": ["Tejal Patwardhan", "Rachel Dias"],
       "year": 2025,
       "arxiv_id": "2510.04374",
       "relevance": "Most similar prior work: shows AI near human parity on specific cross-profession tasks but limited to task-level evaluation, not end-to-end projects"
     },
     {
       "title": "HCAST: Human-Calibrated Autonomy Software Tasks",
       "authors": ["David Rein", "Joel Becker", "Amy Deng"],
       "year": 2025,
       "arxiv_id": "2503.17354",
       "relevance": "Software autonomy benchmark compared against in terms of completion time and task diversity"
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Multi-environment agent benchmark evaluating autonomous agent capabilities"
     },
     {
       "title": "MLE-Bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Domain-specific ML engineering benchmark for AI agents"
     },
     {
       "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
       "authors": ["Tianbao Xie", "Danyang Zhang"],
       "year": 2024,
       "relevance": "Computer-use benchmark for multimodal agents in realistic environments"
     },
     {
       "title": "RE-Bench: Evaluating Frontier AI R&D Capabilities of Language Model Agents Against Human Experts",
       "authors": ["Hjalmar Wijk", "Tao Lin", "Joel Becker"],
       "year": 2024,
       "arxiv_id": "2411.15114",
       "relevance": "AI R&D capability benchmark comparing agent performance to human expert baselines"
     },
     {
       "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
       "authors": ["Giulio Starace", "Oliver Jaffe", "Dane Sherburn"],
       "year": 2025,
       "arxiv_id": "2504.01848",
       "relevance": "Measures AI agents' ability to replicate research papers — related to end-to-end task automation"
     },
     {
       "title": "The AI Productivity Index (APEX)",
       "authors": ["Bertie Vidgen", "Abby Fennelly"],
       "year": 2025,
       "arxiv_id": "2509.25721",
       "relevance": "AI productivity benchmark measuring economically valuable task completion"
     },
     {
       "title": "A Definition of AGI",
       "authors": ["Dan Hendrycks", "Dawn Song", "Christian Szegedy"],
       "year": 2025,
       "relevance": "Defines AGI in terms of cognitive skills, directly informs RLI's cognitive skills analysis of failure modes"
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng"],
       "year": 2024,
       "relevance": "Elo methodology for LLM comparison that RLI adapts for its pairwise evaluation system"
     }
   ]
 }