ai-research-survey

scan.json (28533B)

---

 {
   "paper": {
     "title": "SWE-Effi: Re-Evaluating Software AI Agent System Effectiveness Under Resource Constraints",
     "authors": [
       "Zhiyu Fan",
       "Kirill Vasilevski",
       "Dayi Lin",
       "Boyuan Chen",
       "Yihao Chen",
       "Zhiqing Zhong",
       "Jie M. Zhang",
       "Pinjia He",
       "Ahmed E. Hassan"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2509.09853",
     "doi": "10.48550/arXiv.2509.09853"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-Effi introduces multi-dimensional effectiveness metrics (EuTB, EuITB, EuCTB, EuCB) to re-evaluate AI software engineering systems beyond resolve rate alone. The study finds that scaffold-model synergy matters more than scaffold design alone, with the same scaffold varying dramatically in effectiveness across models (e.g., SWE-Agent EuTB: 21.8% with Qwen3-32B vs 5.1% with GPT-4o-mini). Two systemic issues are identified: a 'Token Snowball' effect where naive memory accumulation causes linear context growth, and 'expensive failures' where unresolved attempts consume 4x+ more resources than successful ones. The 50-issue evaluation on SWE-bench-Verified is explicitly acknowledged as exploratory and not a definitive verdict.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a URL (https://centre-for-software-excellence.github.io/SWE-Effi/) and states 'releasing our code, data, and hosting a public leaderboard' (Section 6). Code release is indicated."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Leaderboard, data, and code at [URL]' and 'We will make our subset publicly available on HuggingFace' (Section 4). The SWE-bench-Verified base data is public, and they indicate their subset and collected metrics data are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment specifications are provided in the paper. Hardware and API provider details are mentioned but not a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. They describe the experimental setup conceptually but do not provide commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates. No confidence intervals, error bars, or uncertainty measures are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'performance drops significantly', '4x increase') without any statistical significance tests. All comparisons are raw number comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative effect sizes throughout, e.g., '18× the token cost' (Section 5.1), 'over 4 times more resources' (Section 5.4), 'nearly in half' (Section 5.2), with baseline context provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 50 issues is acknowledged as a limitation ('we limited the SWE-bench-Verified dataset down to the 50 samples from initial 500' due to cost, Section 6) but no power analysis or formal justification for 50 being sufficient is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported. Tables show means ('mean across samples') without any indication of variability across the 50 issues."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five scaffolds are compared against each other across three LLMs, providing 15 system configurations as mutual baselines (Tables 1, 2, 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Scaffolds were 'selected five popular, actively maintained open-source SWE scaffolds from the top of the SWE-bench leaderboard (as of May 2025)' (Section 4). All are recent and actively maintained."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the SWE-Effi metric framework or on any scaffold component. The study compares different systems but does not isolate the contribution of individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports five metrics: EuTB, EuITB, EuCTB, EuCB, and Resolve Rate (Table 1), plus detailed core metrics in Table 2 (CPU time, inference time, tokens, LLM requests)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to measuring resource efficiency of AI systems on automated benchmarks with pass/fail test suites."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on a subset of SWE-bench-Verified. No system was tuned or developed using these issues — all scaffolds were run with their default configurations on the test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by scaffold (5 scaffolds), by model (3 models), and by resolved vs unresolved attempts (Table 3). Multiple dimensions of per-category analysis are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 ('Failing is Far More Expensive Than Succeeding') extensively discusses failure behavior. Section 5.3 discusses the Token Snowball effect as a failure mode. Figure 1c shows SWE-Agent/GPT-4o-mini entering unproductive loops."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: SWE-Agent with GPT-4o-mini collapses to 5.1% EuTB (Section 5.1), Agentless-Mini with Llama-3.3-70B achieves only 4% resolve rate (Table 1), and the 'expensive failures' pattern shows systemic inefficiency."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims about scaffold-model synergy, Token Snowball effect, expensive failures, and token-vs-time trade-off are all supported by corresponding observations in Sections 5.1-5.4 with data from Tables 1-3 and Figures 1-2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 claims 'High-Quality Reasoning Minimizes Iterations and Saves Tokens' — this is a causal claim but Qwen3-32B differs from Llama-3.3-70B in many ways beyond reasoning quality. The study is observational with respect to model properties and cannot isolate reasoning quality as the causal factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('Re-Evaluating Software AI Agent System Effectiveness') and abstract ('any AI system should be more than correct') are broader than the evidence from 5 scaffolds, 3 budget-tier models, and 50 SWE-bench issues. Section 6 hedges ('initial insights') but the framing extends well beyond the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not systematically consider alternative explanations. For instance, the 'expensive failures' pattern could partly reflect budget caps forcing termination rather than inherent system behavior. The Qwen3-32B advantage could stem from recency or architecture rather than 'reasoning quality.' No confounds are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper precisely defines 'effectiveness' as AUC of resolve-rate vs. resource-consumption curves (Section 3.2) and is careful to note this measures efficiency on a specific benchmark subset, not general AI capability. The measurement matches the defined metric."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 specifies 'GPT-4o-mini-2024-07-18', 'Llama-3.3-70B-Instruct' (quantized to FP8), and 'Qwen3-32B'. The GPT model includes a snapshot date; the open-source models are specific named versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses five open-source scaffolds that contain prompts, but the actual prompts used are not provided in the paper or appendix. The reader must consult each scaffold's source code to find the prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states scaffolds were configured per 'official guidelines' with 'default iteration or generation limits' and SWE-Agent had a $1 budget cap (Section 4), but temperature, top-p, max tokens, and other LLM hyperparameters are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 and Section 4 describe each scaffold: SWE-Agent uses a ReAct loop with editing/testing tools, OpenHands adds IPython/browser/editing tools, Agentless uses hierarchical localization then patch generation, AutoCodeRover uses AST-based localization. Agentic vs procedural distinction is clearly made."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the subset selection: '50 issues randomly drawn from the well-respected SWE-bench-Verified dataset' using 'stratified sampling, preserving the original distribution of issues across different software projects.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion and Conclusion) contains substantive discussion of limitations: 'we had to limit the scope of our evaluation down to the 15 permutations... we limited the SWE-bench-Verified dataset down to the 50 samples from initial 500.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: the 50-sample limitation from 500, cost and time constraints limiting to 15 of many possible configurations, the fact that 'initial runs sometimes took upwards of two weeks to complete with several hundred dollars in API costs.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states 'this work is intended to provide an introduction and initial insights' and 'The findings we present are not intended to be a final, definitive verdict on these five scaffolds.' They explicitly aim to expand with community help."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper indicates all data is released at the project URL. Section 3.1.1 mentions 515,041 API call logs used for the regression model. Section 6 states they are 'releasing our code, data.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data collection: scaffolds were augmented with profiling code, parallel processing was disabled to ensure accurate time measurements, fine-grained metrics were collected for CPU time, inference time, and token usage."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is SWE-bench-Verified, a standard public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is described: select scaffolds and models → configure per official guidelines → instrument with profiling → run on stratified 50-issue subset → collect core metrics → derive effectiveness scores via AUC (Sections 3-4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Five authors are affiliated with Huawei/Huawei Canada but no funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Huawei, Huawei Canada, CUHK Shenzhen, King's College London, Queen's University. The affiliations are prominently displayed under author names."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Five of nine authors are at Huawei, a company with commercial interest in AI coding tools, though the paper evaluates open-source scaffolds rather than Huawei products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the three models (GPT-4o-mini, Llama-3.3-70B, Qwen3-32B). This is relevant because SWE-bench issues are from public GitHub repositories."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models may have seen SWE-bench solutions or related GitHub data during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench issues are derived from public GitHub repositories. All three models likely trained on GitHub data. This contamination risk is not discussed or addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's core contribution. Table 2 reports average CPU time, inference time, input/output tokens, and LLM requests per system. Appendix 7.1 provides per-token dollar costs. Table 3 breaks down costs by resolved vs unresolved."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states 'initial runs sometimes took upwards of two weeks to complete with several hundred dollars in API costs.' Section 3.1.1 notes 515,041 API call logs were collected. Per-token API costs are in Appendix 7.1."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each scaffold-model combination appears to be run once on the 50 issues. No multiple-seed experiments or seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state how many times each experiment was run. Results appear to be from single runs with no discussion of run count."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses default scaffold configurations but does not report the hyperparameter details or justify why defaults are optimal. No search budget is discussed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states 'We configured those scaffolds based on their official guidelines and adhered to their default iteration or generation limits' — the selection of default configurations is justified as providing an 'out-of-the-box' baseline."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so the question of correction for multiple comparisons does not arise."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not discuss potential bias in their metric framework design, such as whether the AUC-based effectiveness scores inherently favor certain scaffold architectures (e.g., lightweight procedural approaches) over others."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's central contribution. Performance (resolve rate) is explicitly plotted as a function of compute budget in Figures 5-7, and effectiveness metrics (EuTB, EuCB, etc.) formalize this relationship."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core argument questions SWE-bench's single-metric (resolve rate) evaluation paradigm. Section 1 argues 'current benchmarks create a disconnect from the realities of practical AI deployment' and proposes multi-dimensional metrics to address this validity gap."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Observation 1 (Section 5.1) is explicitly about the scaffold-model confound: 'SWE Scaffold Performance Is Highly Model-Dependent.' The factorial design (5 scaffolds × 3 models) directly addresses this by testing all combinations."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench issues are from public GitHub repositories that predate the models' training. No discussion of whether model training data includes these issues or their solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether scaffolds or models receive information during evaluation that would not be available in real deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 50 issues are from SWE-bench-Verified with stratified sampling by project, but no discussion of whether models' training data shares structure with these specific GitHub repositories."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are employed. No canary strings, membership inference, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "AI system effectiveness depends on scaffold-model synergy, not scaffold design alone — SWE-Agent EuTB drops from 21.8% with Qwen3-32B to 5.1% with GPT-4o-mini",
       "evidence": "Table 1 shows the full 15-system comparison. SWE-Agent with Qwen3-32B achieves 28% resolve rate with 35.5 API calls, vs 10% with GPT-4o-mini requiring 181 calls and 18x more tokens (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Reasoning models that consume more tokens per call can resolve issues with fewer total tokens by reducing the number of iterations",
       "evidence": "Table 2 shows AutoCodeRover with Qwen3-32B uses 14.7 API calls and 55.5K input tokens vs Llama-3.3-70B requiring 38.3 calls and 416.1K tokens (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "A 'Token Snowball' effect causes input token consumption to grow linearly with API calls due to naive memory accumulation strategies",
       "evidence": "Figure 1 shows the relationship between API calls and input tokens for one selected instance per scaffold. Linear growth is demonstrated (Section 5.3).",
       "supported": "moderate"
     },
     {
       "claim": "Unresolved attempts are far more expensive than resolved ones — SWE-Agent/GPT-4o-mini failures consume 4x+ more tokens and time than successes",
       "evidence": "Table 3 shows resolved vs unresolved resource consumption. SWE-Agent/GPT-4o-mini: 8.8M tokens and 658s for failures vs 1.8M tokens and 167s for successes. Pattern holds across most scaffold-model pairs (Section 5.4).",
       "supported": "strong"
     },
     {
       "claim": "Agentless leads in resolve rate (48% with Qwen3-32B) but at the cost of highest CPU time (727.9s average)",
       "evidence": "Tables 1 and 2 show Agentless/Qwen3-32B achieves 48% resolve rate with 83.1 API calls and 727.9s CPU time (Section 5.1).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny sample size",
       "detail": "Only 50 SWE-bench-Verified issues used. With 15 system configurations, some cells have very few resolved instances (e.g., Agentless-Mini/Llama achieves 4% = 2 issues). Results may be highly sensitive to the specific issues sampled."
     },
     {
       "flag": "Single run without variance",
       "detail": "Each scaffold-model configuration appears to be run once. No repeated runs, no variance reported, no error bars. With stochastic LLM outputs, results could vary substantially across runs."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('drops significantly', 'outperforming') are based on raw number comparisons. No statistical tests are performed to determine whether observed differences exceed random variation."
     },
     {
       "flag": "Missing frontier models",
       "detail": "The three selected models (GPT-4o-mini, Llama-3.3-70B, Qwen3-32B) are all budget-tier. The scaffolds were designed and optimized for frontier models (GPT-4, Claude). Evaluating only on weaker models may not reflect realistic scaffold performance."
     },
     {
       "flag": "No contamination analysis",
       "detail": "SWE-bench issues come from public GitHub repositories. All three evaluated models likely trained on GitHub data. The complete absence of contamination discussion is a significant gap for a benchmark evaluation paper."
     },
     {
       "flag": "Potential commercial interest",
       "detail": "Five of nine authors are at Huawei/Huawei Canada. While they evaluate open-source scaffolds rather than Huawei products, no funding or competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. R. Narasimhan"],
       "year": 2024,
       "relevance": "The foundational benchmark for evaluating AI systems on software engineering issue resolution, which SWE-Effi builds upon."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["J. Yang", "C. E. Jimenez", "A. Wettig", "K. Lieret", "S. Yao", "K. Narasimhan", "O. Press"],
       "year": 2024,
       "relevance": "Major agentic scaffold for SWE tasks using ReAct loop, evaluated in this paper's efficiency analysis."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["X. Wang", "B. Li", "Y. Song"],
       "year": 2025,
       "relevance": "Open-source agentic scaffold with extensive tooling evaluated in the SWE-Effi efficiency comparison."
     },
     {
       "title": "Agentless: Demystifying LLM-Based Software Engineering Agents",
       "authors": ["C. S. Xia", "Y. Deng", "S. Dunn", "L. Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Procedural (non-agentic) scaffold for SWE tasks that led in resolve rate but had highest CPU costs in the evaluation."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Y. Zhang", "H. Ruan", "Z. Fan", "A. Roychoudhury"],
       "year": 2024,
       "relevance": "AST-based procedural scaffold that showed consistently competitive efficiency across model pairings."
     },
     {
       "title": "SWE-RL: Advancing LLM Reasoning via Reinforcement Learning on Open Software Evolution",
       "authors": ["Y. Wei", "O. Duchenne", "J. Copet"],
       "year": 2025,
       "arxiv_id": "2502.18449",
       "relevance": "RL-based approach to SWE tasks that uses lightweight Agentless-Mini scaffold, illustrating efficiency needs for RL training."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu"],
       "year": 2023,
       "relevance": "Foundational reasoning-and-acting paradigm used by agentic SWE scaffolds like SWE-Agent and OpenHands."
     },
     {
       "title": "SWE-Rebench: An Automated Pipeline for Task Collection and Decontaminated Evaluation of Software Engineering Agents",
       "authors": ["I. Badertdinov", "A. Golubev", "M. Nekrashevich"],
       "year": 2025,
       "arxiv_id": "2505.20411",
       "relevance": "Extends SWE-bench with 21,000 new tasks and decontamination, addressing benchmark limitations relevant to evaluation methodology."
     },
     {
       "title": "SWT-BENCH: Testing and Validating Real-World Bug-Fixes with Code Agents",
       "authors": ["N. Mündler", "M. Müller", "J. He", "M. Vechev"],
       "year": 2024,
       "relevance": "Evaluates LLM agents on test generation for bug reproduction, complementing SWE-bench's issue resolution focus."
     },
     {
       "title": "Trae Agent: An LLM-Based Agent for Software Engineering with Test-Time Scaling",
       "authors": ["P. Gao", "Z. Tian", "X. Meng"],
       "year": 2025,
       "arxiv_id": "2507.23370",
       "relevance": "SWE agent using test-time compute scaling, directly relevant to the efficiency-vs-accuracy trade-off SWE-Effi measures."
     }
   ]
 }