ai-research-survey

scan.json (28281B)

---

 {
   "paper": {
     "title": "AGENTLESS: Demystifying LLM-based Software Engineering Agents",
     "authors": [
       "Chunqiu Steven Xia",
       "Yinlin Deng",
       "Soren Dunn",
       "Lingming Zhang"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2407.01489"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We have open-sourced AGENTLESS at: https://github.com/OpenAutoCoder/Agentless' in the abstract. A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the publicly available SWE-bench Lite benchmark (300 problems), which is a standard public dataset. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The implementation section mentions LlamaIndex, OpenAI's text-embedding-3-small, and Python's ast library, but no requirements.txt, Dockerfile, or conda environment file with pinned library versions is described in the paper. Mentioning specific libraries without version pinning is insufficient."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the approach's steps in detail but does not include step-by-step reproduction instructions (e.g., commands to run the experiments). These may exist in the GitHub repository but are not provided in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates only (e.g., '96 (32.00%)' resolved). No confidence intervals or error bars are provided for any of the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AGENTLESS outperforms all open-source agent-based approaches but makes no use of statistical significance tests. Comparisons are based solely on raw percentage differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (e.g., Cohen's d, odds ratios) are reported. Improvements are given as raw differences in solve rates without statistical context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses the SWE-bench Lite benchmark with 300 problems without justifying why this sample size is sufficient for the statistical power needed to support its comparative claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results represent single-run outcomes. The paper does not report variance, standard deviation, or results across multiple runs. Figure 6 shows performance vs. number of samples but does not report variance across repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares AGENTLESS against 26 agent-based approaches including both open-source tools (SWE-agent, AutoCodeRover, Moatless, Aider, etc.) and closed-source commercial tools. A RAG agentless baseline is also included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary tools from 2024 such as CodeStory Aide, Bytedance MarsCode, SWE-agent with Claude 3.5 Sonnet, and AutoCodeRover-v2, representing the state-of-the-art at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 contains an extensive ablation study covering localization (Table 2), repair (Table 3), and patch validation (Table 4) components. Individual design choices like skeleton format, embedding retrieval, and majority voting are each evaluated separately."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: % Resolved, Avg. $ Cost, Avg. # Tokens, and % Correct Location at line/function/file granularities. Additionally, Section 5.1.3 evaluates reproduction test quality."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1 describes manual classification of SWE-bench Lite problems, but the schema explicitly states 'manual classification of the benchmark or dataset itself does not count.' There is no human evaluation of AGENTLESS's outputs (generated patches, localizations, or reproduction tests). The evaluation of system outputs is entirely automated via test suites."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWE-bench Lite is used as a fixed test set. The paper does not use any of the benchmark examples for tuning AGENTLESS. The approach is evaluated directly on the benchmark without a separate validation set being used for parameter selection."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 shows solve rates broken down by problem category (description quality, solution type, and location information) for multiple tools. Table 2 provides per-step localization breakdowns, and Table 3 shows per-repair-configuration results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses that closed-source agent tools outperform AGENTLESS on problems where no location clue is provided, explicitly identifying where AGENTLESS fails. Section 5.1.3 discusses the reproduction test drop-off from 213 selected to 94 plausible tests."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports configurations that hurt performance. For example, Table 2 shows that 'direct from file-level' localization achieves lower performance (47.00%) than the default (56.33%). Table 3 shows that merged multi-samples (28.33%) underperform multi-samples (32.00%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AGENTLESS achieves 'the highest performance (32.00%, 96 correct fixes)' among open-source approaches with low cost ($0.70), which is directly supported by Table 1. The claim about OpenAI adoption is also discussed in Section 5.1.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies (Section 5.2) test causal claims about individual components. Each ablation varies one component at a time while holding others constant, making controlled causal comparisons for claims like 'embedding-based retrieval improves file localization from 78.67% to 81.67% when combined with prompting-based.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates only on Python repositories in SWE-bench Lite but the title 'Demystifying LLM-based Software Engineering Agents' implies broad applicability. Section 7 notes the external threat that 'performance might not generalize to other datasets' but the framing throughout overstates generality beyond the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 mentions data leakage (contamination) as a threat but does not discuss alternative explanations for why AGENTLESS outperforms agent-based tools. For instance, it does not consider whether the advantage comes from the specific benchmark characteristics (Python-only, well-formed issues) rather than the agentless approach itself."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The implementation section explicitly states 'GPT-4o (gpt-4o-2024-05-13)' and 'text-embedding-3-small' for the embedding model. Specific model version identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes what prompts do in natural language (e.g., 'we prompt the LLM to localize and rank the top N most suspicious files') and shows example outputs, but does not provide the actual prompt text. No appendix with full prompts is included in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 (Implementation) specifies: greedy decoding by default, sampling temperature of 0.8 for sampling, chunk size of 512 and overlap of 0 for embeddings, top 3 suspicious files, 4 samples of edit locations, 10 patches per location set (1 greedy + 9 samples), 40 total patches, 40 reproduction test samples."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-phase pipeline (localization, repair, patch validation) is described in detail in Section 3 with a workflow diagram (Figure 1). Each step's inputs and outputs are explicitly described, including the hierarchical localization, search/replace diff format, reproduction test generation, and majority voting re-ranking."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes the manual classification procedure for SWE-bench Lite problems with explicit categories and criteria. Section 6.2 describes the filtering criteria used to produce SWE-bench Lite-S (removing problems with exact patches, misleading solutions, or insufficient information)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Threats to Validity') provides dedicated discussion of both internal and external threats to validity, covering data leakage and generalization to other datasets."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: the internal threat identifies GPT-4o training data leakage as the specific mechanism, and Section 6.2 identifies that closed-source tools outperform AGENTLESS specifically on problems with no location clues. These are not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 acknowledges that results may not generalize to other datasets, but does not explicitly state what the results do NOT show (e.g., does not claim results are restricted to Python only, or to self-contained issues, or to a specific issue type distribution)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The SWE-bench Lite benchmark is publicly available, meaning the raw evaluation data (issues and ground truth patches) can be independently verified. The paper also open-sources AGENTLESS, allowing reproduction of the submission patches."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data originates from SWE-bench (cited [44]), whose collection methodology is in the original paper. For the manual classification in Section 6.1, the classification dimensions and categories are described in detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The evaluation uses a standard public benchmark (SWE-bench Lite) and the manual classification was performed by the authors themselves. This criterion does not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from SWE-bench Lite (300 problems) to SWE-bench Lite-S (249 problems) is documented with explicit filtering criteria (removing exact patches, misleading solutions, insufficient information). The count of excluded problems (51) can be inferred from 300 - 249."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgments section thanks two individuals and mentions a bike but does not disclose any funding source (grants, corporate sponsors, or funding agencies). No funding information is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed with their University of Illinois Urbana-Champaign affiliation. The paper evaluates GPT-4o (OpenAI) and Claude 3.5 Sonnet (Anthropic) as baselines, and none of the authors are affiliated with those companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The authors are at a major research university using substantial API credits, so funding likely exists but is not disclosed. Cannot confirm funder independence without knowing the funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement in the paper. The authors may or may not have financial interests (e.g., via commercialization of AGENTLESS or the GitHub repository), but no disclosure is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 acknowledges the threat from GPT-4o training data leakage but does not state GPT-4o's specific training cutoff date. The paper says 'GPT-4o is a closed-source model, we do not have access to the training data.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly addresses this as an internal threat: 'One threat to validity comes from the data leakage of ground truth developer patches in SWE-bench Lite being part of the training data for GPT-4o.' The paper also cites the SWE-bench authors' comparison of resolve rates before and after GPT-4's knowledge cutoff date as mitigating evidence."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly discusses this risk and cites the SWE-bench authors' finding that there was 'no significant difference' in resolve rates between issues collected before and after GPT-4's knowledge cutoff. This is at least an attempt to address the concern, though incomplete for GPT-4o."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants. The manual classification in Section 6.1 is author analysis of a public benchmark, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The evaluation is on a public software engineering benchmark."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports the average cost per issue as $0.70 for AGENTLESS. Additional costs are reported per component in the ablation tables (e.g., $0.02 for prompting-based file localization, $0.25 for reproduction tests)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-query API costs are reported, the total compute budget (total API spend for running all 300 benchmark problems, GPU/compute hardware, wall-clock time) is not explicitly stated. The paper reports average per-problem costs but not the total experimental budget."
       }
     }
   },
   "claims": [
     {
       "claim": "AGENTLESS achieves the highest performance (32.00%, 96 correct fixes) among all open-source approaches on SWE-bench Lite while costing only $0.70 on average.",
       "evidence": "Table 1 in Section 5.1 compares AGENTLESS against 26 agent-based approaches. Among open-source tools, AGENTLESS scores 32.00% while the next best open-source tool (AutoCodeRover-v2) scores 30.67%. Cost is $0.70 vs. $1.62 for SWE-agent (Claude 3.5 S).",
       "supported": "strong"
     },
     {
       "claim": "The simplistic agentless approach can match or exceed the performance of complex autonomous agents for software development issue resolution.",
       "evidence": "Table 1 shows AGENTLESS (32.00%) outperforms all open-source agents. However, 10 closed-source/commercial tools outperform it, with the best (CodeStory Aide) reaching 43.00%. The claim is partially qualified in the paper text (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "AGENTLESS has an upper bound of 42.0% (126/300) issues solvable if an oracle patch selection method were used.",
       "evidence": "Figure 6 in Section 5.2.2 shows that when considering all 40 candidate patches per issue (not just the selected one), 126 issues (42.0%) have at least one correct patch.",
       "supported": "strong"
     },
     {
       "claim": "SWE-bench Lite contains problematic problems including 4.3% with exact ground truth patches in the description, 10.0% with insufficient information, and 5.0% with misleading solutions.",
       "evidence": "Section 6.1 describes the manual classification of all 300 problems. Figure 8b shows the breakdown: 4.3% exact patch, 9.7% complete steps, 5.0% misleading. The classification criteria are described qualitatively.",
       "supported": "moderate"
     },
     {
       "claim": "Combining prompting-based and embedding-based file localization outperforms either method alone (81.67% vs. 78.67% and 67.67% respectively).",
       "evidence": "Table 2 in Section 5.2.1 shows the combined method achieves 81.67% ground-truth file localization vs. 78.67% for prompting-only and 67.67% for embedding-only. These are point estimates without variance.",
       "supported": "moderate"
     },
     {
       "claim": "Adding reproduction tests for patch selection improves performance from 81 to 96 issues resolved (from 27.00% to 32.00%).",
       "evidence": "Table 4 in Section 5.2.3 shows that regression-test-only selection gives 81 (27.00%) while adding reproduction test filtering gives 96 (32.00%). This is a controlled ablation.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AGENTLESS proposes a simple three-phase (localize, repair, validate) non-agentic approach to autonomous software engineering that achieves 32.00% (96/300) on SWE-bench Lite, outperforming all open-source agent-based approaches at an average cost of $0.70 per issue. The paper demonstrates through ablation studies that hierarchical localization, multi-sample patch generation, and LLM-generated reproduction tests each contribute meaningfully to performance. A manual classification of SWE-bench Lite problems reveals quality issues including problems with exact patches in descriptions (4.3%), misleading solutions (5.0%), and insufficient information (10.0%), leading to the construction of SWE-bench Lite-S as a cleaner evaluation benchmark. Despite its simplicity, AGENTLESS was adopted by OpenAI as the default showcase approach for GPT-4o and o1 models on SWE-bench.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All performance comparisons are point estimates with no confidence intervals, error bars, or statistical significance tests. Given that a single-run difference of ~1-2% can affect rankings (e.g., AGENTLESS at 32.00% vs. AutoCodeRover-v2 at 30.67%), the lack of variance reporting makes it impossible to assess whether observed differences are reliable."
     },
     {
       "flag": "No funding disclosure",
       "detail": "The acknowledgments section thanks two named individuals and a bicycle but discloses no funding sources. For academic research using substantial cloud API compute (running 40 patches x 300 problems on GPT-4o), the funding source is relevant for assessing potential conflicts of interest."
     },
     {
       "flag": "Prompts not provided",
       "detail": "The paper describes what prompts ask the LLM to do in natural language but does not provide the actual prompt text used in experiments. Without the prompts, exact reproduction requires reverse-engineering from the GitHub repository, and the paper cannot be independently evaluated from the text alone."
     },
     {
       "flag": "Single-model evaluation",
       "detail": "All AGENTLESS experiments use GPT-4o (gpt-4o-2024-05-13) only. The claim that agentless approaches can match complex agents is tested with only one LLM backbone, while many competing approaches use a variety of models including Claude 3.5 Sonnet. Results may not generalize across LLMs."
     },
     {
       "flag": "Manual benchmark classification by authors",
       "detail": "The manual classification of all 300 SWE-bench Lite problems (Section 6.1) that motivated the construction of SWE-bench Lite-S was performed by the same authors who built and evaluated AGENTLESS. No inter-rater reliability is reported and no independent verifier is mentioned, creating potential for motivated reasoning in which problems are deemed 'problematic.'"
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": [
         "Carlos E Jimenez",
         "John Yang",
         "Alexander Wettig",
         "Shunyu Yao",
         "Kexin Pei",
         "Ofir Press",
         "Karthik R Narasimhan"
       ],
       "year": 2024,
       "relevance": "Primary benchmark used for evaluation; SWE-bench is the foundational evaluation framework for LLM-based software engineering agents."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": [
         "John Yang",
         "Carlos E Jimenez",
         "Alexander Wettig",
         "Kilian Lieret",
         "Shunyu Yao",
         "Karthik Narasimhan",
         "Ofir Press"
       ],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Key agent-based baseline; SWE-agent represents the paradigm of complex LLM agent interfaces that AGENTLESS is compared against."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": [
         "Yuntong Zhang",
         "Haifeng Ruan",
         "Zhiyu Fan",
         "Abhik Roychoudhury"
       ],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Agent-based baseline providing LLMs with code-search APIs for iterative bug localization and repair."
     },
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "authors": [
         "Junwei Liu",
         "Kaixin Wang",
         "Yixuan Chen",
         "Xin Peng",
         "Zhenpeng Chen",
         "Lingming Zhang",
         "Yiling Lou"
       ],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Survey of LLM-based software engineering agents, directly relevant to the survey scope."
     },
     {
       "title": "CodeR: Issue Resolving with Multi-Agent and Task Graphs",
       "authors": [
         "Dong Chen",
         "Shaoxin Lin",
         "Muhan Zeng",
         "Daoguang Zan"
       ],
       "year": 2024,
       "arxiv_id": "2406.01304",
       "relevance": "Multi-agent approach for software issue resolution used as a baseline in AGENTLESS evaluation."
     },
     {
       "title": "SpecRover: Code Intent Extraction via LLMs",
       "authors": [
         "Haifeng Ruan",
         "Yuntong Zhang",
         "Abhik Roychoudhury"
       ],
       "year": 2024,
       "arxiv_id": "2408.02232",
       "relevance": "Agent-based approach that generates reproduction tests and function summaries; used as a baseline and compared for test generation strategy."
     },
     {
       "title": "MASAI: Modular Architecture for Software-engineering AI Agents",
       "authors": [
         "Daman Arora",
         "Atharv Sonwane",
         "Nalin Wadhwa"
       ],
       "year": 2024,
       "arxiv_id": "2406.11638",
       "relevance": "Modular agent architecture for software engineering used as a competitive baseline on SWE-bench Lite."
     },
     {
       "title": "MarsCode Agent: AI-native Automated Bug Fixing",
       "authors": [
         "Yizhou Liu",
         "Pengfei Gao",
         "Xinchen Wang",
         "Chao Peng",
         "Zhao Zhang"
       ],
       "year": 2024,
       "arxiv_id": "2409.00899",
       "relevance": "Commercial agent system for bug fixing partly inspired by AGENTLESS; competitive baseline on SWE-bench Lite."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": [
         "Chunqiu Steven Xia",
         "Yuxiang Wei",
         "Lingming Zhang"
       ],
       "year": 2023,
       "relevance": "Prior work on LLM-based automated program repair that motivates AGENTLESS's patch sampling approach."
     },
     {
       "title": "How to Understand Whole Software Repository?",
       "authors": [
         "Yingwei Ma",
         "Qingping Yang",
         "Rongyu Cao",
         "Binhua Li",
         "Fei Huang",
         "Yongbin Li"
       ],
       "year": 2024,
       "arxiv_id": "2406.01422",
       "relevance": "Repository-level understanding approach for software agents, used as a baseline (RepoUnderstander) in the evaluation."
     },
     {
       "title": "A Systematic Literature Review on Large Language Models for Automated Program Repair",
       "authors": [
         "Quanjun Zhang",
         "Chunrong Fang",
         "Yang Xie"
       ],
       "year": 2024,
       "arxiv_id": "2405.01466",
       "relevance": "Systematic review of LLM-based program repair, directly relevant to the survey scope on methodology quality in AI-assisted software engineering."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": [
         "Zhiheng Xi",
         "Wenxiang Chen",
         "Xin Guo"
       ],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Survey of LLM-based agent frameworks that contextualizes the agent-based approaches AGENTLESS is designed to challenge."
     }
   ]
 }