ai-research-survey

scan.json (30523B)

---

 {
   "paper": {
     "title": "HAI-Eval: Measuring Human-AI Synergy in Collaborative Coding",
     "authors": ["Hanjun Luo", "Chiming Ni", "Jiaheng Wen", "Zhimu Huang", "Yiran Wang", "Bingduo Liao", "Sylvia Chung", "Yingbin Jin", "Xinfeng Li", "Wenyuan Xu", "XiaoFeng Wang", "Hanan Salam"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2512.04111",
     "doi": "10.48550/arXiv.2512.04111"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "rct"],
   "key_findings": "HAI-Eval introduces 'collaboration-necessary' coding tasks where standalone LLMs achieve near-zero pass rates (best pass@10: 4.22%) and unaided humans achieve only 18.89%, but human-AI collaboration raises performance to 31.11%. The within-subject study with 45 participants reveals that high-performing developers use AI not just for implementation but for strategic co-reasoning, with 51% adopting fundamentally different approaches suggested by AI. LLM performance is uniformly poor across all difficulty levels and professional tracks, suggesting the bottleneck is higher-order reasoning rather than algorithmic complexity.",
   "claims": [
     {
       "claim": "SOTA LLMs achieve near-zero pass rates on HAI-Eval, with the best model achieving only 4.22% overall pass@10",
       "evidence": "Table 1 shows all five tested LLMs achieving 0-0.67% pass@1 under C0 and 0-2.89% under C1, with Claude-Sonnet-4 achieving best pass@10 of 4.22% under C1.",
       "supported": "strong"
     },
     {
       "claim": "Human-AI collaboration significantly improves performance from 18.89% to 31.11% overall pass@1",
       "evidence": "Table 2 shows CH=18.89% vs C2=31.11% averaged across difficulty levels. The paper states this is 'statistically significant' but does not report the specific test or p-value.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs are emerging as co-reasoning partners, not just implementation tools",
       "evidence": "Based on self-reported participant feedback (Figure 4, Table 10): 80% used AI for brainstorming, 51% adopted a fundamentally different approach proposed by AI. Case study in Appendix L illustrates one example.",
       "supported": "weak"
     },
     {
       "claim": "Collaborative performance remains stable across difficulty levels while standalone human performance degrades sharply",
       "evidence": "Table 2 shows C2 overall pass@1: Easy 43.33%, Medium 26.67%, Hard 23.33% vs CH: Easy 36.67%, Medium 13.33%, Hard 6.67%.",
       "supported": "moderate"
     },
     {
       "claim": "LLM failure is uniform across difficulty levels and professional tracks, suggesting a fundamental reasoning bottleneck",
       "evidence": "Tables 6 and 7 in Appendix K show consistently near-zero pass rates across all difficulty levels and tracks for all models.",
       "supported": "strong"
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'Our benchmark and interactive demo will be openly accessible' — future tense with no URL provided. No repository link appears anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes releasing '450 manually-curated static tasks' and states 'the dataset is released as a standalone resource,' but provides no download URL or repository link. The abstract uses future tense ('will be openly accessible')."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes devcontainer files and Docker-based environments conceptually but provides no actual environment specification files, requirements.txt, or dependency versions for reproducing the evaluation setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the evaluation pipeline architecture but does not include commands or procedures to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 1, 2, 6, 7, 8) report only point estimates for pass rates, partial pass rates, completion times, and token usage. No confidence intervals or error bars on the primary performance metrics. SD is only reported for Likert-scale feedback items (Tables 3, 9)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'a statistically significant improvement' (Section 5.2) and states 'Statistical comparisons use appropriate tests with averaged results' (Section 5.1) but never specifies which tests were used, reports no p-values, and provides no test statistics."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context throughout: e.g., pass@1 rises from 18.89% (CH) to 31.11% (C2); Tables 1, 2, 6, 7 include absolute values and delta columns showing improvement magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The study uses 45 participants (15 per track) and 450 task instances for LLMs (10 per template). No power analysis or justification for why these specific sample sizes were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main experimental results (pass rates across conditions) are reported as single aggregate numbers without standard deviations, interquartile ranges, or any spread measure. SD appears only for Likert-scale feedback in Tables 3 and 9."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four conditions serve as baselines: CH (human-only), C0 (fully autonomous AI), C1 (minimally-intervened AI), enabling comparison against C2 (human-AI collaboration)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Five SOTA models evaluated as of July 2025: Claude-Sonnet-4, Claude-Sonnet-3.7, GPT-4.1, GPT-4o, and Gemini-2.5-Pro. All are current at time of study."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The four conditions (CH, C0, C1, C2) function as an ablation: C0→C1 isolates procedural failures, CH→C2 measures AI contribution, C1→C2 measures human reasoning contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four aggregated metrics are used: Overall Pass, Partial Pass, Completion Time, and Token Usage. Pass@1 and pass@10 are also reported for LLM conditions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire study includes human evaluation: 45 participants solve tasks under different conditions. Additionally, two independent expert reviewers validated task instances (Appendix D), and results are validated through participant feedback and independent expert review."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Hidden test cases are used for final scoring: 'Each task includes a set of visible unit tests to assist users during implementation. Final scoring, however, is determined by a comprehensive suite of hidden test cases executed on the backend' (Section 4.3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by difficulty level (Easy/Medium/Hard in Table 2), by professional track (SDE/MLE/DS in Table 8), and by model in Tables 6-7."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o's universal 0% pass rate is discussed. Template validation failures are reported (5 of 45 templates required revision or replacement). Appendix L provides a detailed case study of failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o achieves 0% pass rate across all conditions. Two templates were rejected during validation. The paper reports that minimal intervention (C1) provides only marginal improvement over fully autonomous (C0), showing the intervention protocol has limited impact."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (0.67% standalone LLM pass rate, 18.89% unaided human pass rate, 31.11% collaborative pass rate) are directly supported by Tables 1 and 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims 'human-AI collaboration significantly improves performance.' The within-subject design with full counterbalancing (Latin Square, randomized task sequences, 24-hour interval between sessions) provides adequate causal identification for this claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A explicitly lists limitations: Python only, Copilot-supported models only, East Asian university students only. Section 5.1 acknowledges 'generalizability to industry professionals and other ethnic groups is limited.' The title 'Collaborative Coding' is reasonably bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses order effects (Part 3 of post-test questionnaire, Figure 11), cognitive fatigue mitigation (24-hour interval), and learning effects. They analyze whether improvement is due to tool familiarity vs genuine collaboration (finding expert users are minimally influenced by order)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully defines what it measures (pass rates, completion time, token usage) and frames these as measures of 'synergy' within the specific benchmark context. Section 4.5 explicitly defines metrics and their derivation. The claims match the granularity of measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'Claude-Sonnet-4', 'GPT-4.1', 'GPT-4o', 'Claude-Sonnet-3.7', 'Gemini-2.5-Pro'. No snapshot dates or API versions are provided. The paper notes evaluations were conducted 'as of July 26, 2025' but does not specify exact model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C provides example prompts for the task generation agent (GPT-4.1). Appendix F provides summarized task descriptions noting 'The task description below is a short summary of the original text.' The actual full task READMEs sent to Copilot during evaluation are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1 reports GPT-4.1 hyperparameters: temperature=0.7, top_p=0.9, max_tokens=8192. For LLM evaluation: 'Copilot does not permit hyperparameter customization, all models are evaluated with default settings' (Section 5.1)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates models through GitHub Copilot as a black box. The authors cannot describe Copilot's internal scaffolding. NA per schema guidelines for evaluating third-party tools."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The task generation pipeline is documented in detail: template creation → two-stage validation (fidelity check + generalization test) → expert review (Appendix D). The agentic task system's four tools and their invocation sequence are described in Section 4.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A 'Limitation & Future Work' provides substantive discussion of limitations across multiple dimensions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: Python-only implementation, Copilot-constrained model selection (excluding DeepSeek, Llama, Qwen, o3, GPT-5), homogeneous participant demographics (East Asian students), and discrete pass/fail conversion impacting interpretability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A explicitly lists what was NOT tested: other programming languages, models via direct APIs, industry developers, diverse ethnic backgrounds, granular scoring beyond pass/fail."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (participant solutions, interaction logs, LLM outputs) is made available. Only aggregated results are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix J details the full experimental protocol: informed consent, pre-test questionnaire, task procedure (Steps 1-4 with figures), post-test questionnaire. Data collection mechanisms (shell script submission, backend API, operational logs) are described."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1: 'We recruit 45 participants through personal contacts and advertisements posted on university forums.' Appendix H.1 provides detailed selection criteria. Appendix I describes the verification process including GitHub profile and LinkedIn checks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: template creation → validation (2-stage protocol, Appendix D) → task generation (Section 4.2) → evaluation (hidden test cases, auto-calibrated baselines) → aggregated metrics (Section 4.5). Template pass rates reported (88.9% first-attempt, Appendix D.3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information appears anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed on the first page: NYU Abu Dhabi, NTU, UIUC, Harvard, Zhejiang University, UESTC, Beijing University of Technology, Hong Kong Polytechnic University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper uses GitHub Copilot extensively but does not disclose any relationship with Microsoft/GitHub."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five models evaluated. The paper only notes evaluations were conducted 'as of July 26, 2025.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The dynamic task generation inherently mitigates train/test overlap, but the paper never explicitly discusses this as a contamination mitigation. The 450 static task instances for LLMs are not assessed for potential training data overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the dynamic generation approach and the Equation 2 validation protocol ensure tasks are not trivially solvable, the paper never frames this as addressing benchmark contamination. No decontamination analysis is performed on the static 450-instance dataset."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any registry is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "The informed consent form (Appendix J.1) states: 'this study has been reviewed and approved by the Institutional Review Board (IRB).'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H.2 reports demographics: 28 males, 17 females, ages 19-26 (mean 21.4), education levels (24 undergrad, 15 master's, 6 PhD), 84.4% daily LLM users, mean 1.47 internships. Figure 5 provides visual representations."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H.1 lists detailed criteria: age ≥18, CS-related major, ≥2 years programming experience, VS Code proficiency, AI assistant usage ≥3 times/week, Python proficiency, English proficiency, track-relevant expertise."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1: 'Each participant's task sequence is randomly selected from all balanced permutations of the four conditions. A Latin Square design ensures that every problem appears equally across conditions and is completed by different participants.'"
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not feasible in this design: participants necessarily know whether Copilot is enabled (CH) or not (C2), as the presence of the AI assistant is the experimental manipulation."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports recruiting and studying 45 participants but does not explicitly state whether any participants dropped out or were excluded after beginning the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token usage is reported as a primary metric in Tables 2 and 8 (e.g., C2 uses 2.04-2.31M tokens across difficulty levels). Completion times are also reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated: no API costs, no total token expenditure across all experiments, no hardware specifications for the evaluation server or Docker environments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported from single runs (pass@1) and 10 attempts (pass@10) but with no analysis of seed sensitivity or result variation across repeated evaluations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 450 task instances (10 per template) for LLM evaluation. pass@1 and pass@10 are reported, implying the number of attempts. C0 allows 'up to 25 attempts' via Copilot's retry mechanism."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'Copilot does not permit hyperparameter customization, all models are evaluated with default settings' (Section 5.1). No hyperparameter search was conducted, and this is justified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All models use default settings with no configuration selection. The baseline model (Claude-Sonnet-4) is justified by 'market prevalence' (citing Menlo Ventures 2025)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Five models are compared across multiple conditions, difficulty levels, and tracks (Tables 1, 6, 7). The paper states 'Statistical comparisons use appropriate tests' but provides no details on multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluate models on it. No discussion of potential bias from evaluating on their own benchmark or whether independent evaluation would yield different results."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Token usage varies substantially across conditions (C0: 0.42-0.58M vs C2: 2.04-2.31M) but the relationship between compute budget and performance is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3 and 4 extensively discuss what the benchmark measures (ecological validity, collaboration-necessary tasks) grounded in distributed cognition theory. Participant feedback validates task realism (mean 4.07/5, Table 3). The paper discusses limitations of what it captures vs real-world development."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates Copilot as a bundled product — the scaffold IS the thing being tested. All models are evaluated through the same Copilot interface, so scaffold is controlled within the study. NA per schema."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses newly generated tasks, which inherently mitigates temporal leakage. However, the paper never explicitly discusses temporal leakage as a concern or explains that dynamic generation serves this purpose."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation provides visible test cases during solving and hidden test cases for scoring. The paper does not discuss whether visible test cases could leak information about the hidden test cases or final evaluation criteria."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 450 static instances are generated from 45 templates (10 per template). Instances from the same template share structural similarities. The paper does not discuss whether this non-independence affects aggregate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No explicit leakage detection method is applied. The template validation protocol (Equation 2) ensures tasks are hard but does not specifically test for data leakage."
       }
     }
   },
   "red_flags": [
     {
       "flag": "Statistical significance claimed without evidence",
       "detail": "The paper claims results are 'statistically significant' (Section 5.2) and mentions 'appropriate tests' (Section 5.1) but never reports which tests were used, p-values, or test statistics for any comparison."
     },
     {
       "flag": "Homogeneous participant sample",
       "detail": "All 45 participants are East Asian university students/recent graduates aged 19-26. The paper acknowledges this but still draws broad conclusions about 'human-AI synergy' and 'developer competencies in the AI era.' The recruitment via 'personal contacts and university forums' introduces selection bias."
     },
     {
       "flag": "No artifacts released despite claims",
       "detail": "The paper uses present tense ('we release a reproducible toolkit') and future tense ('will be openly accessible') for artifact release but provides no URLs, repository links, or DOIs for any code, data, or benchmark materials."
     },
     {
       "flag": "Circular benchmark validation",
       "detail": "The benchmark is designed to be hard for LLMs (Equation 2 requires <5% pass rate). Templates that LLMs could solve were rejected. Then the paper's main finding is that LLMs perform poorly on the benchmark. The 'collaboration-necessary' design creates a ceiling that ensures the desired result."
     },
     {
       "flag": "Co-reasoning claims based on self-report",
       "detail": "The central qualitative finding — that LLMs serve as 'co-reasoning partners' — is based primarily on self-reported participant feedback and a single case study (Appendix L). Behavioral log analysis is mentioned but results are not systematically reported."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models solve real-world software engineering problems?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "relevance": "Key benchmark for evaluating LLM coding capabilities on real-world software engineering tasks, directly compared against HAI-Eval's approach."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, foundational benchmark that HAI-Eval positions against."
     },
     {
       "title": "The effects of generative ai on high-skilled work: Evidence from three field experiments with software developers",
       "authors": ["Zheyuan Kevin Cui", "Mert Demirer", "Sonia Jaffe", "Leon Musolff", "Sida Peng", "Tobias Salz"],
       "year": 2025,
       "relevance": "Enterprise RCT measuring AI impact on developer productivity, directly relevant to user study methodology comparison."
     },
     {
       "title": "Measuring the impact of early-2025 ai on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Controlled study finding negative productivity impacts from AI tools, providing counterpoint to productivity gain claims."
     },
     {
       "title": "Experience with github copilot for developer productivity at zoominfo",
       "authors": ["Gal Bakal", "Ali Dasdan", "Yaniv Katz", "Michael Kaufman", "Guy Levin"],
       "year": 2025,
       "arxiv_id": "2501.13282",
       "relevance": "Enterprise study of Copilot productivity impact, relevant to AI-assisted coding evaluation methodology."
     },
     {
       "title": "How much does ai impact development speed? an enterprise-based randomized controlled trial",
       "authors": ["Elise Paradis", "Kate Grey", "Quinn Madison"],
       "year": 2025,
       "relevance": "Enterprise RCT on AI impact on development speed, directly relevant comparison for human-AI collaboration study design."
     },
     {
       "title": "Reading between the lines: Modeling user behavior and costs in ai-assisted programming",
       "authors": ["Hussein Mozannar", "Gagan Bansal", "Adam Fourney", "Eric Horvitz"],
       "year": 2024,
       "relevance": "Framework for modeling human behavior in AI-assisted programming, relevant to understanding developer-AI interaction patterns."
     },
     {
       "title": "Agent-as-a-judge: Evaluate agents with agents",
       "authors": ["Mingchen Zhuge"],
       "year": 2024,
       "arxiv_id": "2410.10934",
       "relevance": "Agent-based evaluation methodology, relevant to evolving approaches for evaluating AI coding systems."
     },
     {
       "title": "Collaborative gym: A framework for enabling and evaluating human-agent collaboration",
       "authors": ["Yijia Shao", "Vinay Samuel", "Yucheng Jiang", "John Yang", "Diyi Yang"],
       "year": 2024,
       "arxiv_id": "2412.15701",
       "relevance": "Framework for evaluating human-agent collaboration, directly relevant to HAI-Eval's approach of measuring collaboration value."
     },
     {
       "title": "MLE-Bench: A comprehensive benchmark for evaluating large language models in machine learning engineering tasks",
       "authors": ["Dong He"],
       "year": 2024,
       "arxiv_id": "2405.16672",
       "relevance": "Benchmark for ML engineering tasks that HAI-Eval extends beyond by requiring human-AI collaboration."
     },
     {
       "title": "LiveCodeBench: A comprehensive benchmark for general-purpose language agents",
       "authors": ["Zihan Zheng"],
       "year": 2024,
       "arxiv_id": "2406.01869",
       "relevance": "Evolving code benchmark focused on algorithmic complexity, positioned as insufficient for measuring collaboration."
     },
     {
       "title": "Sea change in software development: Economic and productivity analysis of the ai-powered developer lifecycle",
       "authors": ["Thomas Dohmke", "Marco Iansiti", "Greg Richards"],
       "year": 2023,
       "arxiv_id": "2306.15033",
       "relevance": "Productivity analysis of AI-powered development, relevant to understanding the economic impact HAI-Eval aims to measure."
     }
   ]
 }