ai-research-survey

scan.json (27376B)

---

 {
   "paper": {
     "title": "GitTaskBench: A Benchmark for Code Agents Solving Real-World Tasks Through Code Repository Leveraging",
     "authors": ["Ziyi Ni", "Huacan Wang", "Shuo Zhang", "Shuo Lu", "Ziyang He", "Wang You", "Zhenheng Tang", "Yuntao Du", "Bill Sun", "Hongzhang Liu", "Sen Hu", "Ronghao Chen", "Bo Li", "Xin Li", "Chen Hu", "Binxing Jiao", "Daxin Jiang", "Pin Lyu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.18993"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GitTaskBench evaluates code agents on 54 real-world tasks across 7 domains requiring repository leveraging. The best system (OpenHands+Claude 3.7) solves only 48.15% of tasks, with environment setup errors accounting for 65% of failures. The paper introduces an alpha-value metric integrating task success, cost, and human labor rates to assess economic viability. Open-source models significantly underperform closed ones, and agents excel on textual tasks versus multimodal ones.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/QuantaAlpha/GitTaskBench, and a project page at https://gittaskbench.github.io/."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark tasks, evaluation scripts, and repositories are released via the GitHub repository. 'All tasks and their detailed success criteria are openly available in the official GitHub repository.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Execution configurations are detailed in Appendix B: specific Docker containers for OpenHands and SWE-Agent, Python 3.12 Conda for Aider, and hardware specs (9 vCPUs, 33 GB RAM, 2965 GiB SSD). Framework versions pinned in Table 8."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'all tasks can be automatically assessed with a single shell command' and provides detailed framework configurations. The benchmark is open-sourced with evaluation scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Results in Table 3 are point estimates only, despite being averaged over two runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims like 'OpenHands achieves the best overall performance' and comparisons between frameworks/models are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance differences are reported with baseline context, e.g., 'ECR 72.22%, TPR 48.15%' for the best system, with all other systems' scores shown for comparison. The alpha metric provides economic effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 54 tasks were chosen. No power analysis or discussion of whether 54 tasks is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'all reported results are averaged over two independent runs' but does not report standard deviation, variance, or any spread measure across those runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple agent frameworks (Aider, OpenHands, SWE-Agent) and multiple LLMs are compared against each other. Table 1 also compares GitTaskBench against prior benchmarks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include current state-of-the-art models: GPT-4.1, Claude 3.7, Gemini 2.5 Pro, DeepSeek V3, and Qwen3. Frameworks are pinned to April 2025 releases."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Sensitivity analysis on timeout and max_iteration hyperparameters (Table 7) shows their impact on performance, functioning as an ablation of configuration choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three main metrics: Execution Completion Rate (ECR), Task Pass Rate (TPR), and the alpha-value economic metric. Token usage and cost are also reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Five raters independently compare each agent output with human-produced groundtruth and assign quality levels for the Q factor. 'Five raters independently compare each agent output with the groundtruth and assign it to one of five levels.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 54 tasks are used for evaluation. There is no held-out test set or dev/test split. The benchmark is a single evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides domain-specific performance breakdowns across Image Processing, Video, Speech, Physiological Signals, Security, Web Scraping, and Office Document Processing."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive error analysis in the Error Analysis section and Appendix F, with five error types classified and detailed case studies of failures (Listings 4-13)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that agents struggle with multimodal tasks, that replacing humans is not always cost-effective (negative alpha scores), and that environment setup causes 65% of failures. Gemini 2.5 Pro 'underwhelms in think mode.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 48.15% best TPR (Table 3), environment setup failures (Figure 8b, 65%), and the alpha metric are all supported by results. The updated 62.96% figure for RepoMaster+Claude 3.5 is mentioned in the abstract and leaderboard."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'likely due to its robust code execution capabilities and more proactive and explorative strategies' for OpenHands' superiority, without controlled experiments isolating these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to evaluate 'real-world tasks' and agent capabilities generally, but tests only 54 tasks across 18 repositories — all Python-based, with specific selection criteria (≥50 stars). The title and framing suggest broader applicability than the tested scope supports."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations for performance differences. For example, framework differences in resource allocation, prompt design, or runtime behavior could explain results, but these are not systematically analyzed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper distinguishes between ECR (execution completion) and TPR (task pass), and introduces the alpha metric to bridge the gap between technical performance and practical economic value. The proxy/outcome relationships are explicit."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'GPT-4o-2024-08-06', 'Claude-3-5-sonnet-20241022', 'Claude-3-7-sonnet-20250219'. Table 8 lists exact framework versions (OpenHands 0.33.0, SWE-Agent v1.0.1-61-gaa4e8ea1, Aider v0.84.1.dev-21-gb2592267)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix C (Listings 1-2), including the core objective, workflow guidelines, and key constraints used for agent task execution."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature (0.5), top-p (1.0), max response length (4096 tokens), timeout (600s for OpenHands), and other framework-specific parameters are listed in Table 8 and Appendix B."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party agent frameworks (OpenHands, SWE-Agent, Aider) as black boxes. The authors cannot be expected to describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The task and repository selection process is documented with criteria (Python-based, ≥50 stars, active in past 5 years), completeness verification procedures, and the four-step construction pipeline (Figure 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G contains a 'Limitations & Future Work' section discussing scope limitations and planned expansions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations in Appendix G are generic future work plans ('will further expand the benchmark', 'will broaden model coverage') rather than specific threats to validity of current results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It acknowledges the benchmark 'focuses on practical, user-facing tasks' but does not bound the generalizability of its findings to the 54 Python-only tasks tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark, evaluation scripts, and task definitions are open-sourced. Agent execution traces (events folders, batch_results.jsonl) appear to be available through the repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The four-step curation pipeline is described in detail: task/repository selection with criteria, completeness verification by experts, execution framework design, and evaluation framework development (Figure 4)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Five PhD students in computer science performed completeness verification and human evaluation. Human completion times are reported (average 1.34 hours). However, evaluator selection criteria beyond 'PhD student' are not described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from repository selection to evaluation is documented with specific criteria at each stage (Figure 4), including selection criteria, verification steps, and evaluation metrics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UCAS, CASIA, BUPT, NUS, StepFun, HKUST, SDU, PINAI, USYD, PKU, USTC. StepFun is a company, and several authors are affiliated with it."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "StepFun (an AI company) has authors on the paper and corresponding authors affiliated with it. StepFun has a potential interest in code agent benchmarking outcomes. No independence statement is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models, despite using pre-trained LLMs on benchmark tasks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 18 GitHub repositories used as tasks could appear in any model's training data. Popular repositories with ≥50 stars are likely in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark tasks involve well-known open-source repositories (SpeechBrain, Scrapy, etc.) that are almost certainly in LLM training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study. The five PhD raters are evaluators of agent outputs, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study; PhD students served as benchmark validators and output evaluators."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "API costs are reported per framework-model combination in Table 3 (e.g., $29.8 for OpenHands+Claude 3.7). Token pricing is detailed in Table 11."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specs provided (9 vCPUs, 33 GB RAM, 2965 GiB SSD). API costs, token counts, and per-task costs are reported. The alpha metric explicitly incorporates operational costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over two runs but no seed sensitivity analysis or variance across seeds is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "'All reported results are averaged over two independent runs under identical settings.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The sensitivity analysis in Table 7 tests different timeout and max_iteration values, but no systematic hyperparameter search budget is reported. It's unclear how the default configurations were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses default framework settings except for explicitly stated changes (e.g., timeout=600s for OpenHands). Configuration choices are documented and justified in Appendix B."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite many pairwise comparisons across models and frameworks."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the benchmark and evaluate systems on it without acknowledging potential bias in benchmark design favoring certain capabilities. RepoMaster (from the same research group based on author overlap) achieves the top score."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports both performance (ECR/TPR) and cost ($) for every configuration. The alpha metric explicitly relates performance to cost. The sensitivity analysis shows timeout vs. performance tradeoffs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 54 tasks actually measure 'real-world' code agent capability. The tasks are curated by the authors with no external validation of representativeness."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates multiple frameworks (OpenHands, SWE-Agent, Aider) with the same LLMs, allowing readers to see scaffold effects. Table 3 shows the same model performing differently across frameworks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The repositories used (e.g., SpeechBrain, Scrapy, DeOldify) are well-established open-source projects. Models trained on GitHub data have likely seen these repositories' code, READMEs, and usage patterns. This is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether providing repository READMEs and code in the prompt constitutes feature leakage — models may have memorized repository usage patterns from training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Some tasks share the same repository (e.g., NeuroKit has 3 tasks, SpeechBrain has 5). No discussion of whether these non-independent tasks inflate performance estimates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used despite evaluating models on well-known public repositories."
       }
     }
   },
   "claims": [
     {
       "claim": "OpenHands+Claude 3.7 achieves the best performance with 72.22% ECR and 48.15% TPR",
       "evidence": "Table 3 shows comparative results across all framework-model combinations. OpenHands+Claude 3.7 has the highest ECR and TPR.",
       "supported": "strong"
     },
     {
       "claim": "Environment setup errors account for 65% of all failures",
       "evidence": "Figure 8(b) shows error distribution: E1 (Environment-Setup) at 65.04%, followed by E2 (Workflow Planning) at 11%, E4 (Runtime) at 10%, E3 (Repository Comprehension) at 9%, E5 (Instructions) at 5%.",
       "supported": "strong"
     },
     {
       "claim": "Agents perform notably better on purely textual tasks compared to multimodal, model-based tasks",
       "evidence": "Figure 5 shows domain-specific performance. Office document processing tasks have higher success rates than image/speech processing tasks. The paper explains this is because textual tasks typically require 'reading simple wrapper scripts that import the library API.'",
       "supported": "moderate"
     },
     {
       "claim": "Replacing humans with agents is not always cost-effective",
       "evidence": "Alpha scores in Table 10 show negative values for several repositories (e.g., AnimeGANv3 at -11.134 for Claude 3.5, DeScratch at -3.967). Low-MV image tasks often produce negative alpha when agent cost exceeds $1-2.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek V3 delivers the highest overall benefit and best cost-performance ratio",
       "evidence": "Table 10 and Figure 7 show DeepSeek V3's alpha scores. However, this is for the alpha metric only — DeepSeek V3 has lower TPR (26.85%) than GPT-4.1 (42.59%) or Claude 3.7 (48.15%) under OpenHands.",
       "supported": "moderate"
     },
     {
       "claim": "More generous timeout and max_iteration settings significantly boost performance",
       "evidence": "Table 7 shows ECR increasing from 18.52% to 50.00% as timeout increases from 120s to 1800s, and TPR from 33.33% to 37.04% as max_iteration increases from 30 to 100.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Self-comparison bias",
       "detail": "RepoMaster, which achieves the top score of 62.96% on the leaderboard (Figure 1), shares authors with this paper (Wang et al. 2025a lists Huacan Wang as first author). The benchmark may be designed in ways that favor the authors' own system."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The 18 repositories are well-known open-source projects (SpeechBrain 8k+ stars, Scrapy 53k+ stars) whose code, READMEs, and usage patterns are almost certainly in LLM training data. Models may be recalling solutions rather than reasoning about repositories."
     },
     {
       "flag": "Very small benchmark size",
       "detail": "54 tasks across 18 repositories is a small benchmark. With only 1-5 tasks per repository, individual task success/failure has outsized impact on domain-level conclusions. No justification is given for this sample size."
     },
     {
       "flag": "Only two runs per configuration",
       "detail": "Results are averaged over only two independent runs with no variance reported. Two runs is insufficient to estimate result stability, especially with stochastic LLM outputs at temperature 0.5."
     },
     {
       "flag": "No statistical tests despite comparative claims",
       "detail": "The paper makes numerous comparative claims (e.g., framework X outperforms Y, model A is more cost-efficient than B) without any statistical testing. With 54 tasks and two runs, observed differences could easily be within noise."
     },
     {
       "flag": "Market value estimates are subjective",
       "detail": "The alpha metric relies on 'estimated, prevailing market value' from freelance platforms (Fiverr, Upwork). These estimates (e.g., $5 for scratch detection, $150 for video action analysis) are highly variable and could be selected to favor certain conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, a foundational LLM code generation benchmark."
     },
     {
       "title": "Swe-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks (GitHub issue resolution)."
     },
     {
       "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?",
       "authors": ["Samuel Miserendino"],
       "year": 2025,
       "arxiv_id": "2502.12115",
       "relevance": "Benchmark linking software engineering agent performance to real-world economic value."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang"],
       "year": 2025,
       "arxiv_id": "2407.16741",
       "relevance": "Major open-source agent framework evaluated in this benchmark."
     },
     {
       "title": "Swe-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang"],
       "year": 2024,
       "relevance": "Key agent framework for automated software engineering, evaluated in the benchmark."
     },
     {
       "title": "MLE-Bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Jun Shern Chan"],
       "year": 2025,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark for ML engineering agents with Kaggle competition tasks."
     },
     {
       "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
       "authors": ["Giulio Starace"],
       "year": 2025,
       "relevance": "Benchmark for evaluating AI agents on research paper replication tasks."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Code generation benchmark with diverse function calls and complex task instructions."
     },
     {
       "title": "MLAgentBench: Evaluating language Agents on Machine Learning Experimentation",
       "authors": ["Qian Huang"],
       "year": 2023,
       "arxiv_id": "2310.03302",
       "relevance": "Benchmark for evaluating LLM agents on ML experimentation tasks."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian"],
       "year": 2023,
       "relevance": "Multi-agent framework for software development using LLM agents."
     },
     {
       "title": "RepoMaster: Autonomous Exploration and Understanding of GitHub Repositories for Complex Task Solving",
       "authors": ["Huacan Wang"],
       "year": 2025,
       "relevance": "Repository-aware agent framework achieving top GitTaskBench scores; shares authors with this paper."
     },
     {
       "title": "Livecodebench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-aware code evaluation benchmark."
     }
   ]
 }