ai-research-survey

scan.json (26373B)

---

 {
   "paper": {
     "title": "SWEnergy: An Empirical Study on Energy Efficiency in Agentic Issue Resolution Frameworks with SLMs",
     "authors": ["Arihant Tripathy", "Ch Pavan Harshit", "Karthik Vaidhyanathan"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.09543",
     "doi": "10.48550/arXiv.2512.09543"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Current agentic issue resolution frameworks (SWE-Agent, OpenHands, AutoCodeRover, Mini SWE Agent) fail when paired with SLMs, achieving near-zero resolution rates (4% max for AutoCodeRover+Qwen, 0% for all others) on SWE-bench Verified Mini. Framework architecture is the primary driver of energy consumption, with AutoCodeRover consuming 9.4x more energy than OpenHands. Energy correlates strongly with runtime (R=0.89) and output tokens (R=0.88), with 'chatty' ReAct-style agents entering unproductive loops that waste energy without progress.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Replication package with source code, logs, and measurements provided at https://github.com/sa4s-serc/swenergy (footnote 1, Section 3)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "SWE-bench Verified Mini is publicly available. The replication package includes raw logs and computed metrics in structured JSON format (Section 4.3)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (Intel Xeon w3-2435, 32GB RAM, RTX A2000 16GB) and FP16/32K context window stated, but no software dependency specifications (requirements.txt, Dockerfile, library versions) are described for the monitoring/measurement infrastructure."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "A replication package is mentioned but the paper does not include step-by-step reproduction instructions. No README with commands or 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports Min, P50, Mean, Max, and Std for all metrics across repetitions. Figure 2 shows box plots with interquartile ranges and whiskers."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite claims of differences between frameworks. Comparisons rely on descriptive statistics and correlation coefficients (R values in Section 5.2) without formal hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with context: '9.4x more energy' (abstract), specific kJ values with baselines (e.g., 216.21 kJ vs 23.05 kJ), and correlation coefficients (R=0.89, R=0.88) in Section 5."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 50 tasks is justified by SWE-bench Verified Mini's representativeness (Section 3.1) and 3 repetitions are performed, but no power analysis or formal sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported for all metrics in Table 1 across 3 repetitions. Box plots in Figure 2 show full distributions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four frameworks are compared against each other (SWE-Agent, OpenHands, AutoCodeRover, Mini SWE Agent), each serving as a baseline for the others (Section 3.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Three of the four frameworks ranked among the top three on SWE-bench Full leaderboard as of October 2025 (Section 3.1). Models selected from Berkeley Function-Calling leaderboard."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing third-party frameworks comparatively, not a single system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Eight metrics across three dimensions: resolution status, failure mode (effectiveness); wall-clock duration, token usage, LLM call count (efficiency); total energy, peak memory, cost (resource utilization). Section 3.3."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The study measures energy consumption and automated resolution rates of software agents. Human evaluation is irrelevant to these claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWE-bench Verified Mini is used as-is as the evaluation benchmark. No tuning or selection is performed on this dataset (Section 3.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-framework-per-model breakdowns for all metrics. Figure 5 shows per-framework failure mode breakdowns. Figure 4 shows per-framework token distributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis in Section 5.2 and 6.1: failure mode taxonomy (Figure 5), qualitative log analysis identifying Step Repetition, Context Loss, Follow Task Requirements failures, and 'false positive' completions (Lesson 2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is essentially a negative result: SLMs achieve near-zero resolution rates in all frameworks. The paper explicitly frames this as a finding rather than hiding it."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (near-zero resolution, 9.4x energy difference, framework architecture as primary driver, 4% for AutoCodeRover) are all supported by Table 1 and the analysis in Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The claim 'framework architecture is the primary driver of energy consumption' is supported by controlling for model and hardware across 4 frameworks. Section 5.2 provides correlation analysis and failure mode analysis linking architectural choices to energy patterns."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Threats to Validity) explicitly bounds findings: limited to 2 SLMs, single hardware configuration, 50-task subset. External validity subsection notes findings 'may not generalize to different types or scales of issues.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses alternative explanations: low energy could mask premature termination or false positives (Lesson 2), framework design vs model capability as competing explanations (Lesson 3). Section 7 considers OS background activity as a confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures energy (kJ), resolution rate, and tokens directly. Claims about 'energy efficiency' match the granularity of measurements. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified as 'Gemma-3 4B' and 'Qwen-3 1.7B Instruct' with parameter counts but no specific checkpoint IDs, version hashes, or snapshot dates. These are marketing names without precise version identification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses existing frameworks' default prompts but does not reproduce them. The replication package may contain them via the framework code, but the paper itself does not provide the actual prompt text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "32K context window and FP16 precision are stated (Section 4.1), along with 30-minute timeout. But LLM sampling parameters (temperature, top-p, etc.) are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Each framework's architecture is described in Section 3.1: SWE-Agent's ACI with ReAct prompting, OpenHands' sandboxed Docker with multi-agent support, AutoCodeRover's three-phase pipeline (fault localization → context retrieval → patch generation), Mini SWE Agent's minimal bash-only design."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the full experimental procedure: idle baseline measurement and subtraction (51.69W CPU, 2.70W GPU), monitoring script instrumentation, SWE-bench evaluation harness verification, and JSON storage format."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides substantive discussion across internal, construct, external, and conclusion validity categories."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats: '50 tasks are a small subset' (external), 'limited to two small, open-weight models' (external), 'single hardware configuration' (external), 'Gemini 2.5 Flash may influence pattern identification' (construct), 'dataset heavily skewed toward failure' (conclusion)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what results do NOT show: 'findings may not generalize to different types or scales of issues', 'trade-offs could differ substantially with larger models', 'energy profiles may vary on different CPU/GPU architectures.' Conclusion validity notes inability 'to draw firm conclusions about the energy cost of a successfully resolved issue.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3: 'All raw logs and computed metrics were stored in a structured JSON format to ensure transparency and facilitate downstream analysis.' Replication package provided at GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the experimental procedure in detail: isolated machine, RAPL/NVML instrumentation, idle baseline subtraction, 30-min timeout, 3 repetitions, SWE-bench evaluation harness for verification."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is SWE-bench Verified Mini, a standard public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from execution to analysis is documented: framework execution with monitoring → idle baseline subtraction → SWE-bench evaluation → structured JSON storage → statistical analysis (Section 4). Idle power values (51.69W CPU, 2.70W GPU) are explicitly stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section: 'This research was funded by the ANRF Prime Minister Early Career Research Grant (ANRF/ECRG/2024/003379/ENS).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with SERC, IIIT-Hyderabad. They evaluate third-party frameworks (SWE-Agent, OpenHands, AutoCodeRover), not their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funder is ANRF (Indian government research grant), which has no financial stake in the performance of any evaluated framework or model."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of training data cutoff dates for Gemma-3 4B or Qwen-3 1.7B Instruct. The models could have been trained on SWE-bench task repositories."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SWE-bench Verified Mini tasks or their solutions appeared in the training data of either model."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench tasks derive from public GitHub issues that predate both models' training. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports Cost (USD) per run for each configuration. Token usage and energy consumption (kJ) also reported. E.g., AutoCodeRover+Qwen mean cost $0.1245, OpenHands+Gemma $0.0043."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "1,200 total experimental runs on specified hardware (Section 4.2). Energy consumption per run and total duration are reported. Hardware fully specified (Intel Xeon w3-2435, 32GB RAM, RTX A2000 16GB)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Three full repetitions per configuration with standard deviation reported across runs in Table 1 (Section 4.3). This captures run-to-run variability including stochastic model behavior."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3: 'three full repetitions for each framework and model configuration. This yielded 150 data points' per condition, '1,200 experimental runs' total."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Frameworks are used with fixed settings (32K context, FP16, 30-min timeout) but no discussion of whether these were tuned or how they were selected. No search budget reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations are reported in Table 1 with no selective presentation. The paper evaluates and reports all framework-model combinations rather than selecting a best one."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate third-party frameworks using original implementations, which mitigates this bias, but they do not explicitly discuss author-evaluation bias or the Lucic et al. concern."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper analyzes performance as a function of compute/energy budget. Table 1 jointly reports energy, time, and resolution. The energy-vs-effectiveness trade-off is the central research question."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench Verified Mini is used without questioning whether it measures real-world issue resolution capability. Section 3.1 notes representativeness of the subset but does not discuss construct validity of SWE-bench itself."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The scaffold confound IS the central research question. The paper explicitly compares frameworks (scaffolds) as the primary variable while controlling for model and hardware, directly measuring the scaffold's impact on energy and performance."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SWE-bench tasks predate the models' training data or whether solutions were available during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides hints not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and SWE-bench tasks share structural similarities or overlap from same repositories."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Framework architecture is the primary driver of energy consumption, with AutoCodeRover consuming 9.4x more energy than OpenHands when using the same SLM (Gemma).",
       "evidence": "Table 1: AutoCodeRover+Gemma mean 216.21 kJ vs OpenHands+Gemma mean 23.05 kJ. Correlation analysis (Section 5.2): energy correlates with duration (R=0.89) and output tokens (R=0.88).",
       "supported": "strong"
     },
     {
       "claim": "Task resolution rates are near-zero for all SLM-framework configurations, with only AutoCodeRover+Qwen achieving 4% (2/50 tasks).",
       "evidence": "Table 1 shows 0% resolution for all configurations except AutoCodeRover+Qwen (mean 2 resolved). Verified by SWE-bench evaluation harness (Section 4.3).",
       "supported": "strong"
     },
     {
       "claim": "Low energy consumption does not indicate efficiency — it can mask premature termination or false positive completions.",
       "evidence": "Section 6.1, Lesson 2: Mini SWE Agent showed low energy but many runs terminated prematurely due to context overflow, and some 'successful' runs produced destructive patches (e.g., replacing 2,000-line file with a single invalid line).",
       "supported": "moderate"
     },
     {
       "claim": "Current agentic frameworks are fundamentally mismatched with SLM capabilities, acting as passive orchestrators that assume competent reasoning engines.",
       "evidence": "Section 6.1, Lesson 3, and qualitative log analysis (Section 4.4): frameworks lack mechanisms to guide, scaffold, or correct struggling SLMs. Supported by failure mode analysis (Figure 5) showing Step Repetition and Context Loss dominate.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Near-zero resolution limits conclusions",
       "detail": "With only AutoCodeRover+Qwen achieving any success (4%), conclusions about energy cost of successful resolution are statistically weak. The authors acknowledge this in Section 7 (Conclusion Validity)."
     },
     {
       "flag": "AI-assisted qualitative analysis",
       "detail": "Gemini 2.5 Flash was used to process execution logs for failure mode identification (Section 4.4). While authors manually verified a sample, this introduces potential bias in pattern interpretation."
     },
     {
       "flag": "No contamination analysis",
       "detail": "SWE-bench tasks come from public GitHub issues. Models may have seen these during training, potentially affecting even the near-zero results. No training cutoffs or contamination checks are discussed."
     },
     {
       "flag": "Very narrow model selection",
       "detail": "Only 2 SLMs tested (1.7B and 4B parameters). Results may not generalize to the broader SLM space (e.g., 7B-8B models commonly considered 'small')."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "The benchmark used in this study; foundational evaluation framework for autonomous issue resolution agents."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez"],
       "year": 2024,
       "relevance": "One of the four evaluated agentic frameworks; introduces the ACI concept for agent-driven code editing."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "relevance": "One of the four evaluated frameworks; general-purpose agentic platform for software development."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yiheng Zhang"],
       "year": 2024,
       "doi": "10.1145/3650212.3680384",
       "relevance": "Only framework achieving non-zero resolution in this study; structured three-phase pipeline for issue resolution."
     },
     {
       "title": "Small Language Models are the Future of Agentic AI",
       "authors": ["Peter Belcak", "Greg Heinrich", "Shizhe Diao"],
       "year": 2025,
       "arxiv_id": "2506.02153",
       "relevance": "Argues for SLMs as sustainable alternatives to LLMs for agentic tasks; motivates this study's focus."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Provides the MAST failure taxonomy used to classify failure modes in this study."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "relevance": "The prompting paradigm used by SWE-Agent and OpenHands; relevant to understanding agentic reasoning loops."
     },
     {
       "title": "Estimating the Carbon Footprint of BLOOM, a 176B Parameter Language Model",
       "authors": ["Alexandra Sasha Luccioni", "Sylvain Viguier", "Anne-Laure Ligozat"],
       "year": 2023,
       "relevance": "Foundational work on environmental costs of large language models; motivates the energy efficiency focus."
     },
     {
       "title": "Greening AI-enabled Systems with Software Engineering: A Research Agenda for Environmentally Sustainable AI Practices",
       "authors": ["Luís Cruz", "João Paulo Fernandes", "Maja H. Kirkeby"],
       "year": 2025,
       "doi": "10.1145/3743095.3743099",
       "relevance": "Research agenda for sustainable AI practices that motivates and contextualizes this study's green SE focus."
     },
     {
       "title": "LLM-Based Multi-Agent Systems for Software Engineering: Literature Review, Vision, and the Road Ahead",
       "authors": ["Junda He", "Christoph Treude", "David Lo"],
       "year": 2025,
       "doi": "10.1145/3712003",
       "relevance": "Survey of LLM-based multi-agent systems for SE; provides context on the agentic SE landscape."
     },
     {
       "title": "Small Models, Big Tasks: An Exploratory Empirical Study on Small Language Models for Function Calling",
       "authors": ["Ishan Kavathekar", "Raghav Donakanti", "Ponnurangam Kumaraguru", "Karthik Vaidhyanathan"],
       "year": 2025,
       "arxiv_id": "2504.19277",
       "relevance": "Related work by same group on SLM capabilities for function calling; directly relevant to SLM viability in agentic tasks."
     }
   ]
 }