ai-research-survey

scan.json (27080B)

---

 {
   "paper": {
     "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?",
     "authors": ["Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke"],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2502.12115",
     "doi": "10.48550/arXiv.2502.12115"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper open-sources a unified Docker image and public evaluation split at https://github.com/openai/SWELancer-Benchmark, mentioned in the abstract and throughout."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "SWE-Lancer Diamond (public evaluation split worth $500,800) is released. The full dataset is 'available upon request' — but the public split is a substantial release."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A5 specifies the Docker container setup: 'Microsoft Azure Standard D2as v4 virtual machine with 2 vCPUs and 8GB RAM', 64GB shared memory, 192GB max memory, pre-built Docker image with all libraries installed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The open-source evaluation harness and Docker image are provided. Appendix A5 describes the execution environment and scaffold details. The GitHub repo contains the eval harness."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 1 are single-run pass@1 with no confidence intervals or error bars. The paper acknowledges 'there can be significant variance between runs' (footnote 2) but does not report CIs for main results. Only the price analysis (Section 3.6) reports a 95% CI."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Claude 3.5 Sonnet 'outperforms' o1 by 9.7% on IC SWE tasks but provides no statistical significance test. All comparisons are raw point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as pass@1 percentages and dollar amounts with baselines, e.g., 'Claude 3.5 Sonnet scores 26.2% on IC SWE tasks' and '$208,050 out of $500,800', providing sufficient context to understand magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 1,488 tasks were selected or why the Diamond split is 502 tasks. No power analysis or discussion of whether this sample size is sufficient for the comparisons made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results are single-run pass@1. The paper states 'each is just a single rollout' (Appendix A5). The pass@k experiments use multiple runs but the main results table does not report variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple frontier models compared: GPT-4o, o1, Claude 3.5 Sonnet, plus open-source models Deepseek-R1 and Llama 3.3 70B in the appendix."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o, o1, and Claude 3.5 Sonnet were all frontier models at the time of evaluation. Open-source models also included."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations: user tool removal (Section 3.5), reasoning effort levels for o1 (Section 3.4), and pass@k with varying attempts (Section 3.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports pass@1, dollar earnings, earn rate, and pass@k. Also breaks down by IC SWE vs SWE Manager tasks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "100 professional software engineers reviewed tasks. End-to-end tests were triple-verified by experienced engineers. SWE Manager ground truth validated with 99% agreement from experienced engineers (Section 2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark maintains a $499,200 private holdout set (Appendix A11) separate from the Diamond public split, to mitigate contamination."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 3 break down pass rates by task type (Application Logic, UI/UX, Server-Side Logic) and nature of work (Bug Fixes, New Features, Maintenance). Price range breakdowns in Figure 8."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A9 provides detailed qualitative analysis of failure trajectories for o1, GPT-4o, and Claude 3.5 Sonnet on specific tasks. Section 3.7 discusses common failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Models fail the majority of tasks. The paper reports GPT-4o fails 90% of IC SWE tasks, open-source models perform poorly (3.8-5.9% on IC SWE Diamond), and user tool removal has minimal impact."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that models 'are still unable to solve the majority of tasks' are supported by Table 1 (best is 26.2% on IC SWE). The '1,400 freelance tasks' and '$1 million' claims match the dataset description."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are limited and justified: 'higher reasoning effort improves pass@1' is supported by controlled ablation varying only reasoning effort (Section 3.4). 'More attempts leads to consistent increase' is supported by pass@k (Section 3.3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 (Limitations) explicitly bounds generalization: tasks are from one repository (Expensify), one platform (Upwork), one language stack (JS/TS), and freelance tasks are 'more self-contained than full-time software engineering tasks.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for performance differences. For example, Claude 3.5 Sonnet outperforming o1 could be due to scaffold compatibility, but this is not explored. The scaffold confound is not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly maps task completion to monetary payout, with the payout being the actual freelancer payment. Section 3.6 carefully analyzes cost savings with explicit limitations. The paper acknowledges freelance tasks don't represent all SWE work."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 specifies 'gpt-4o-2024-08-06', 'o1' with high reasoning effort, and 'claude-3-5-sonnet-20240620' — exact API version strings provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for both IC SWE and SWE Manager tasks are provided in Appendix A8, including the exact text sent to models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A5 states: 'All rollouts are conducted with temperature 1.0', 'maximum number of tool calls is 100', 'maximum time allowed is 3 hours', and reasoning effort levels are specified for o1."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A5 describes the scaffold: 'a basic scaffold that lets them browse the local codebase, modify files, and execute terminal commands.' The user tool is described in Section 2.2. Docker container setup is detailed."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 documents the 5-step benchmark construction pipeline: repository selection, task selection with review criteria, task generation, E2E test development, and user tool creation. Appendix A6 details the curation criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 is a dedicated 'Limitations' section covering diversity of repositories, scope, modalities, environments, and contamination."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: single repository (Expensify), underrepresentation of infrastructure tasks, text-only evaluation missing screen recordings, models cannot ask clarifying questions, and specific contamination risks from public GitHub issues."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states: 'SWE-Lancer would also be less representative of \"zero to one\" SWE work', 'we remain cautious about extrapolating impact beyond' freelance engineering, infrastructure tasks are underrepresented."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Diamond evaluation split is open-sourced with the Docker image, including task descriptions, end-to-end tests, and codebase snapshots. Full dataset available upon request."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes collection: tasks sourced from Upwork postings for the Expensify repository, real tasks previously solved by paid contributors, reviewed by 100 professional software engineers."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '100 professional software engineers' and 'a team of ten experienced engineers' but does not describe how these engineers were recruited, their qualifications, or whether recruitment introduces bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 documents the full pipeline: repository selection → task selection with multi-engineer review → task generation → E2E test development with triple verification. Appendix A6 provides detailed curation criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding source disclosed. Acknowledgements mention OpenAI organizations sponsoring the work but no formal funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with OpenAI (footnote 1). The paper header states '1OpenAI'."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI funded and conducted this research. OpenAI has a financial interest in demonstrating LLM capabilities and benchmarking its own models (GPT-4o, o1). The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. OpenAI employees evaluating OpenAI models (GPT-4o, o1) alongside competitors have clear financial interests that are not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 (Appendix A2) lists training cutoff dates: GPT-4o (2023-10-01), o1 (2023-10-01), Claude 3.5 Sonnet (2024-04-01). Used for contamination analysis."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses contamination risk: 'tasks originate from public GitHub issues between 2023 and 2024. Depending on model cutoff date, contamination in training data is possible.' Table 4 analyzes pre/post-cutoff performance."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple measures: private holdout set (Appendix A11), no internet access during evaluation, GitHub remote removed, Table 4 shows 'no clear performance improvement for tasks predating the models' knowledge cutoffs.' The holdout set uses re-introduced bugs to resist contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as subjects. The software engineers are annotators/evaluators, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Engineers were hired to create/validate benchmark tasks, not studied as participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper describes annotators as 'professional software engineers' and 'experienced software engineers' but these are not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 reports API costs: GPT-4o 95% CI $194.31 ± $17.31 per run, o1 $1,623.76 ± $56.68 per run on 237 Diamond IC SWE tasks. Also reports cost savings ratios."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A5 specifies: Azure Standard D2as v4 VM (2 vCPUs, 8GB RAM), 3-hour max per task, 5 runs for price analysis. Section 3.6 reports total API costs across runs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results are single rollouts. The paper states 'each is just a single rollout' (Appendix A5). Pass@k uses multiple samples but the main Table 1 results are single-seed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'each is just a single rollout' for main results (Appendix A5), and '5 runs' for the price analysis (Section 3.6). Pass@k experiments state the number of samples."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The scaffold and prompt appear fixed, but no discussion of whether these were tuned or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the scaffold configuration, prompt template, or tool settings were selected. These likely involved some tuning but the process is not documented."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI authors evaluate OpenAI models alongside competitors but do not acknowledge the bias of evaluating their own systems. Scaffold may favor their own models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 explicitly reports o1 performance as a function of reasoning effort (compute). Figure 8 shows pass@1 by price range across compute levels."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 and the comparison table (A4) discuss construct validity: E2E tests vs unit tests, grader hacking susceptibility, real-world representativeness vs prior benchmarks. Section 4 discusses scope limitations."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All models use the same basic scaffold, but the paper does not discuss whether this scaffold favors certain models. Appendix A15 acknowledges that open-source models may be underelicited due to scaffold design, but the confound between scaffold and model is not systematically addressed for the main models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 in Appendix A2 analyzes performance split by whether tasks predate model training cutoffs, directly addressing temporal leakage. Results show 'no clear performance improvement for tasks predating the models' knowledge cutoffs.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states agents run with 'no Internet access', GitHub remote removed, future commits removed. The user tool provides no feedback about success. Models cannot access E2E tests during evaluation."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All tasks come from a single repository (Expensify). The paper does not discuss whether tasks share structural similarities or whether performance on one task could inform another due to shared codebase context."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A11 describes augmented holdout tasks where engineers 'reintroduce the bug in a distinct but equivalent way.' Section 3.1 removes internet access and Git remotes. Table 4 performs temporal split analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "Claude 3.5 Sonnet is the best performing model, earning $208,050 on SWE-Lancer Diamond and resolving 26.2% of IC SWE tasks.",
       "evidence": "Table 1 shows Claude 3.5 Sonnet at 26.2% pass@1 on IC SWE Diamond ($58k), 44.9% on SWE Manager Diamond ($150k), totaling $208k on the Diamond set.",
       "supported": "moderate"
     },
     {
       "claim": "Higher test-time compute (reasoning effort) improves performance, especially on harder tasks.",
       "evidence": "Section 3.4 and Figure 8: o1 improves from 9.3% (Low) to 16.5% (High) on IC SWE Diamond. Greater improvements on higher-priced tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Allowing more attempts (pass@k) leads to consistent performance increases, with o1 nearly tripling its solve rate with 6 additional attempts.",
       "evidence": "Section 3.3 and Figure 7: o1 pass@1 ~16.5% increases to ~45% at pass@7. GPT-4o pass@6 matches o1 pass@1.",
       "supported": "strong"
     },
     {
       "claim": "Using o1 before deferring to freelancers could reduce costs by up to 33.5% (pass@5).",
       "evidence": "Section 3.6: o1 pass@5 with freelancer fallback yields 33.5% cost savings over paying freelancers for all 237 Diamond IC SWE tasks.",
       "supported": "moderate"
     },
     {
       "claim": "SWE-Lancer's E2E tests are more resistant to cheating than unit tests used in prior benchmarks.",
       "evidence": "Appendix A7 provides a concrete example of o1 cheating on SWE-Bench Verified by renaming a variable. The paper argues E2E tests emulate human workflows and are harder to exploit, but no systematic grader hacking analysis on SWE-Lancer is provided.",
       "supported": "weak"
     },
     {
       "claim": "Models perform better on SWE Manager tasks than IC SWE tasks.",
       "evidence": "Table 1: All models show higher pass@1 on Manager tasks (37-47%) vs IC SWE tasks (8-26%). Consistent across Diamond and Full sets.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-Lancer introduces a benchmark of 1,488 freelance software engineering tasks valued at $1M, evaluating both coding (IC SWE) and managerial decision-making capabilities. Claude 3.5 Sonnet leads with 26.2% on IC SWE and 44.9% on manager tasks, earning ~$400K on the full set. Higher test-time compute and more attempts improve performance, but models still fail the majority of tasks. The benchmark's use of E2E tests and real monetary payouts provides a more realistic evaluation framework than prior work.",
   "red_flags": [
     {
       "flag": "Company evaluating its own models",
       "detail": "All authors are from OpenAI. While Claude 3.5 Sonnet outperforms OpenAI models, GPT-4o and o1 are heavily featured and the scaffold was designed by OpenAI. Appendix A15 acknowledges open-source models may be 'underelicited' due to scaffold design."
     },
     {
       "flag": "Single-run main results",
       "detail": "Main results in Table 1 are single rollouts at temperature 1.0. The paper acknowledges 'there can be significant variance between runs' but does not quantify this variance for the main results. At temperature 1.0, single-run variance could be substantial."
     },
     {
       "flag": "Single repository benchmark",
       "detail": "All 1,488 tasks come from one repository (Expensify) on one platform (Upwork). Despite claims of real-world representativeness, this limits generalizability. The paper acknowledges this in limitations but the title and framing ('$1 Million from Real-World Freelance Software Engineering') implies broader applicability."
     },
     {
       "flag": "No statistical tests for model comparisons",
       "detail": "Claims like 'Sonnet 3.5 outperforms o1 by 9.7%' are based on single-run point estimates without statistical tests. Given acknowledged variance, these differences may not be statistically significant."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and pass@k metric, foundational for code generation evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Primary predecessor benchmark for real-world software engineering evaluation."
     },
     {
       "title": "Introducing SWE-Bench Verified",
       "authors": ["Neil Chowdhury", "James Aung", "Chan Jun Shern"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Improved version of SWE-bench with human-verified test cases."
     },
     {
       "title": "SWE-bench Multimodal: Do AI systems generalize to visual software domains?",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alex L. Zhang"],
       "year": 2024,
       "arxiv_id": "2410.03859",
       "relevance": "Extends SWE-bench to frontend/visual programming tasks."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Y. Zhuo", "Minh Chien Vu", "Jenny Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Benchmark for code generation with complex multi-step instructions."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-aware code benchmark using temporal splits."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Landmark system for competitive programming code generation."
     },
     {
       "title": "Can LLMs generate novel research ideas? A large-scale human study with 100+ NLP researchers",
       "authors": ["Chenglei Si", "Diyi Yang", "Tatsunori Hashimoto"],
       "year": 2024,
       "arxiv_id": "2409.04109",
       "relevance": "Evaluates LLM capability for research ideation with human expert comparison."
     },
     {
       "title": "CodeELO: Benchmarking competition-level code generation of LLMs with human-comparable Elo ratings",
       "authors": ["Shanghaoran Quan", "Jiaxi Yang", "Bowen Yu"],
       "year": 2025,
       "arxiv_id": "2501.01257",
       "relevance": "Competition-level code generation benchmark using Elo rating system."
     },
     {
       "title": "SciCode: A research coding benchmark curated by scientists",
       "authors": ["Minyang Tian", "Luyu Gao", "Shizhuo Dylan Zhang"],
       "year": 2024,
       "arxiv_id": "2407.13168",
       "relevance": "Scientific programming benchmark evaluating LLM coding for research tasks."
     }
   ]
 }