ai-research-survey

scan.json (28956B)

---

 {
   "paper": {
     "title": "Agent-as-a-Judge: Evaluate Agents with Agents",
     "authors": [
       "Mingchen Zhuge",
       "Changsheng Zhao",
       "Dylan R. Ashley",
       "Wenyi Wang",
       "Dmitrii Khizbullin",
       "Yunyang Xiong",
       "Zechun Liu",
       "Ernie Chang",
       "Raghuraman Krishnamoorthi",
       "Yuandong Tian",
       "Yangyang Shi",
       "Vikas Chandra",
       "Jurgen Schmidhuber"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.10934"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/metauto-ai/agent-as-a-judge listed in the header. This is a working URL to the project code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The DevAI dataset is released on HuggingFace: https://huggingface.co/devai-benchmark, as stated in the paper header. The dataset is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or conda environment file is described in the paper. The trajectory samples show Python dependencies (openhands runtime) but no formal environment specification for reproducing the Agent-as-a-Judge experiments is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a GitHub link but does not include step-by-step reproduction instructions within the paper itself. No section describing how to run the benchmarks or replicate the results is present."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2 and 3 are reported as point estimates (e.g., '90.44% alignment rate') with no confidence intervals or error bars. No uncertainty quantification is provided for the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (Agent-as-a-Judge outperforms LLM-as-a-Judge) but no statistical significance tests (e.g., t-tests, bootstrap tests) are applied. Differences are compared as raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While absolute differences are noted (e.g., Agent-as-a-Judge at 90.44% vs. LLM-as-a-Judge at 70.76%), no standardized effect sizes (Cohen's d, odds ratios) are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The DevAI benchmark has only 55 tasks and 365 requirements. No power analysis or justification for why 55 tasks is sufficient to support the claims made is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run point estimates. No standard deviation, variance across runs, or confidence measures are reported for alignment rates or other metrics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Agent-as-a-Judge against LLM-as-a-Judge (Table 3) as a direct baseline and against Human-as-a-Judge as the gold standard. Multiple baselines are included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LLM-as-a-Judge (Zheng et al., 2024) is a contemporary baseline from 2024, and the human evaluation is the relevant gold standard. The three developer agents (MetaGPT, GPT-Pilot, OpenHands) are current open-source frameworks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 and Appendix K present extensive component ablation studies showing the impact of adding ask, graph, read, locate, retrieve, search, planning, and memory modules. Table 4 and Tables 5-7 show results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics including Alignment Rate, Judge Shift, PR Curves (precision-recall), Requirements Met (independent and dependency-aware), Task Solve Rate, and Self-Termination rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three expert human evaluators performed extensive manual evaluation (86.5 hours total) of the developer agents' outputs, with two rounds including a consensus-building round (Section 3). This is the ground truth the automated judges are compared against."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper benchmarks on the full DevAI dataset of 55 tasks with no train/test split. All 55 tasks are used for evaluation, and the ablation studies for Agent-as-a-Judge components appear to be on the same data used to select the final configuration."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported per developer agent (MetaGPT, GPT-Pilot, OpenHands) but not broken down by task category (supervised learning, reinforcement learning, computer vision, NLP, etc.) despite these categories being defined in the benchmark."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F discusses qualitative failure modes of the developer agents (e.g., MetaGPT not saving files, OpenDevin skipping code execution). Section 4.1 discusses failure modes of the Agent-as-a-Judge components (planning instability, memory chain errors)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies in Section 4.3 and Appendix K show that adding the search, planning, and memory components actually hurt performance relative to the baseline configuration. These negative ablation results are reported and analyzed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Agent-as-a-Judge 'dramatically outperforms LLM-as-a-Judge and is as reliable as our human evaluation baseline.' Table 3 shows alignment rates of ~90% for Agent-as-a-Judge vs. ~68-70% for LLM-as-a-Judge, with human individual evaluators averaging 85-89%. The claim is supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims through ablation studies (e.g., 'adding the graph component increases performance to 75.95%'). The ablation design is sequential addition rather than factorial, so component interactions are not fully isolated. Additionally, ablations are run on OpenHands only, not validated across all three developer agents."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames Agent-as-a-Judge as a general framework for evaluating agentic systems but validates it only on 55 code-generation tasks in AI development. The abstract and conclusion make broad statements ('concrete step forward for modern agentic systems') without adequately bounding results to this narrow setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why Agent-as-a-Judge outperforms LLM-as-a-Judge. For example: whether the improvement is due to the agent's access to workspace files (not available to vanilla LLM judges), or whether a more capable LLM without agentic features would achieve similar gains. The confound of access level is partially noted (gray-box vs. black-box) but not systematically disentangled."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'gpt-4o-2024-05-13' as the backend LLM for all three developer agents and Agent-as-a-Judge (Section 2.3, Table 1, and sample trajectories). This is a specific model version with a date identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix L provides the actual system prompts for the Agent-as-a-Judge system, including the system prompt (L.1), locate module prompt (L.2), retrieve module prompt (L.3), ask module prompt (L.4), and locate module user prompt (L.5). Full prompt text is provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for the LLM calls made by Agent-as-a-Judge or the developer agents. These settings are not disclosed."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix C describe the Agent-as-a-Judge scaffold in detail, including all eight modular components (graph, locate, read, search, retrieve, ask, memory, planning) with their roles. Figure 6 and 9 show the pipeline diagrams."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E documents the DevAI dataset creation pipeline in detail: manually drafted queries (E.1), setting judging criteria (E.2), building dependency DAGs (E.3), refinement rounds (E.4), dataset analysis (E.5), auxiliary information (E.6). The filtering and construction pipeline is described with criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The discussion touches on certain limitations (e.g., 'a perfect Agent-as-a-Judge is not the focus') but there is no labeled section with substantive limitation discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated threats-to-validity discussion is present. The paper briefly notes limitations like the benchmark being small-scale and current agentic systems being limited, but does not systematically enumerate specific threats to the validity of its conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The scope is not explicitly bounded. The paper evaluates only 55 AI development tasks in Python but makes general claims about Agent-as-a-Judge as a framework for evaluating agentic systems. No explicit statements about what domains, tasks, or agent types the results do NOT generalize to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The DevAI dataset is released on HuggingFace (https://huggingface.co/devai-benchmark). The paper states the dataset includes tasks, requirements, and preferences in JSON format (Appendix E.7 shows sample). Raw benchmark data is publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E documents the data collection process: expert annotators manually crafted queries targeting AI development tasks, set binary requirements, built dependency DAGs, and performed two rounds of review and refinement. The process is described in reasonable detail."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Appendix H states 'We recruited three AI experts from the authors to perform human evaluation.' The evaluators are co-authors, which introduces a conflict of interest. Their selection process and qualifications beyond 'AI experts' are not described. This is a notable gap."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from task creation to final evaluation is documented: draft queries → set requirements → build dependency DAGs → refinement rounds → execute with developer agents (with 1800-second time limit and instrumentation) → collect workspaces and trajectories. The flow is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgements section discloses funding from 'King Abdullah University of Science and Technology (KAUST) - Center of Excellence for Generative AI under award number 5940 and the SDAIA-KAUST Center of Excellence in Data Science and Artificial Intelligence.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated on the first page: Meta AI (Yunyang Xiong, Zechun Liu, Ernie Chang, Raghuraman Krishnamoorthi, Yuandong Tian, Yangyang Shi, Vikas Chandra, Changsheng Zhao) and KAUST (Mingchen Zhuge, Dylan Ashley, Wenyi Wang, Dmitrii Khizbullin, Jurgen Schmidhuber). The paper uses GPT-4o (OpenAI), not Meta or KAUST products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "KAUST funding is independent of the outcome — neither KAUST nor the SDAIA-KAUST center has a financial stake in Agent-as-a-Judge outperforming LLM-as-a-Judge or in the specific developer agents evaluated. The funder is an academic institution."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests (patents, equity, startups) is present in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses gpt-4o-2024-05-13 as the backend but does not state its training data cutoff. The DevAI benchmark is newly created in 2024 and likely post-dates training cutoffs, but this is not explicitly confirmed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether gpt-4o's training data could overlap with tasks similar to those in DevAI. The benchmark is novel and hand-crafted, which reduces risk, but this is not formally addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DevAI is a newly created benchmark, likely created after gpt-4o's training cutoff, but the paper does not explicitly confirm this or discuss contamination risk. No analysis of whether any DevAI tasks appear online is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The three human evaluators are co-authors performing annotation work as part of their research contribution, not external human participants in a human subjects study. Pre-registration requirements are structurally inapplicable to co-author annotation work."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The evaluators are co-author researchers, not recruited human participants. IRB review applies to research conducted on human subjects, not to co-authors performing their research roles."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Demographics reporting applies to recruited human participants whose characteristics could affect generalizability. Co-author annotators performing a methodological role do not constitute human participants requiring demographic characterization."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Inclusion/exclusion criteria apply to studies recruiting participants from a population. The evaluators are self-selected co-authors contributing to their own paper, not recruited from a participant pool."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a repeated-measures observational study where all evaluators assess all outputs. Randomization is not applicable to this evaluation design."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Co-author annotators performing a research methodology role do not constitute human participants in a controlled experiment requiring blinding. While blinding would have strengthened the evaluation, the human_studies blinding criterion is structurally inapplicable to co-author annotation work."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "All three evaluators completed both rounds of evaluation. The paper reports final hours for each evaluator (16.5, 19.5, 22.0 for round 1; 9.5 together for round 2), so attrition is not an issue — all evaluators participated fully."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 explicitly reports that Agent-as-a-Judge cost $30.58 in API calls and took 118.43 minutes, while Human-as-a-Judge cost ~$1,297.50 and took 86.5 hours. Table 1 also reports per-agent average costs ($1.19 for MetaGPT, $3.92 for GPT-Pilot, $6.38 for OpenHands)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 states 'a full evaluation on DevAI with one of these three took around 210.65 USD and 14 hours to perform.' Section 4.4 reports LLM-as-a-Judge took 10.99 minutes and cost $29.63. Compute budgets are stated for all major experimental components."
       }
     }
   },
   "claims": [
     {
       "claim": "Agent-as-a-Judge achieves approximately 90% alignment with human consensus evaluation, outperforming LLM-as-a-Judge which achieves approximately 70% alignment.",
       "evidence": "Table 3 shows alignment rates of 88.52%, 83.88%, 90.44% for Agent-as-a-Judge (black-box) vs. 84.15%, 65.30%, 60.38% for LLM-as-a-Judge (black-box) across the three developer agents, evaluated against human consensus.",
       "supported": "moderate"
     },
     {
       "claim": "Agent-as-a-Judge performs comparably to or better than individual human evaluators.",
       "evidence": "Table 3 shows the average alignment rate of individual human evaluators is 89.34%, 84.88%, 85.70% across agents, while Agent-as-a-Judge achieves 88.52%, 83.88%, 90.44% (black-box). PR curves in Figure 7 show Agent-as-a-Judge outperforms any single human evaluator on OpenHands.",
       "supported": "moderate"
     },
     {
       "claim": "Agent-as-a-Judge saves 97.72% of time and 97.64% of cost compared to human evaluation.",
       "evidence": "Section 4.4: Human-as-a-Judge cost ~$1,297.50 and 86.5 hours; Agent-as-a-Judge cost $30.58 and 118.43 minutes. The savings are computed from these reported figures.",
       "supported": "strong"
     },
     {
       "claim": "GPT-Pilot and OpenHands each satisfy approximately 29% of DevAI requirements, with only one task fully solved by each.",
       "evidence": "Table 2: Requirements Met (D) are 28.96% and 28.68% for GPT-Pilot and OpenHands respectively; Task Solve Rate is 1.81% for each (1 out of 55 tasks).",
       "supported": "strong"
     },
     {
       "claim": "Human evaluators disagree with each other at rates ranging from 10-30%, illustrating the unreliability of single-human evaluation.",
       "evidence": "Section 3.2 and Figure 4 report disagreement rates between pairs of evaluators ranging from ~10% to ~30%, and individual evaluator error rates up to 23.77% (cn9o on GPT-Pilot).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "The paper introduces Agent-as-a-Judge, a framework where agentic systems evaluate other agentic systems, and validates it on DevAI, a new benchmark of 55 AI development tasks. Agent-as-a-Judge achieves approximately 90% alignment with human expert consensus evaluation, substantially outperforming LLM-as-a-Judge (approximately 70% alignment) while reducing evaluation cost by 97.64% and time by 97.72%. Ablation studies show that the graph, read, and locate modules are the most important components, while search, planning, and memory modules did not improve performance in this setting. DevAI reveals that current leading agentic frameworks (MetaGPT, GPT-Pilot, OpenHands) satisfy only 22-45% of task requirements and complete only 0-2% of tasks end-to-end.",
   "red_flags": [
     {
       "flag": "Evaluator conflict of interest",
       "detail": "The three human evaluators are recruited 'from the authors,' making them co-authors evaluating work they have a stake in. This introduces a potential conflict of interest in the ground truth used to validate Agent-as-a-Judge. Their evaluations form the gold standard against which the automated judges are compared."
     },
     {
       "flag": "No train/test split for ablations",
       "detail": "The ablation studies in Section 4.3 use the same 55 DevAI tasks to select Agent-as-a-Judge components and then report performance. The configuration is optimized and evaluated on the same data without a held-out test set, risking overfitting of the component selection to this specific benchmark."
     },
     {
       "flag": "Small benchmark with narrow scope",
       "detail": "The benchmark has only 55 tasks focused specifically on AI development in Python. The paper makes broad claims about Agent-as-a-Judge as a general framework for agentic evaluation, but the proof-of-concept is validated on a single narrow domain with a small task count. No statistical power justification is provided."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "The paper's core claim that Agent-as-a-Judge outperforms LLM-as-a-Judge is based on comparing alignment rates (e.g., 90.44% vs. 70.76%) without any statistical tests. With 365 binary requirements, some of these differences may be statistically significant, but this is never verified."
     },
     {
       "flag": "Confounded comparison between judge types",
       "detail": "Agent-as-a-Judge can access the workspace files and execute code to verify requirements, while the baseline LLM-as-a-Judge only has trajectory text or limited context. The performance difference may be explained primarily by information access rather than the agentic architecture. The gray-box vs. black-box comparison partially addresses this but doesn't fully disentangle the confound."
     },
     {
       "flag": "Missing limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. A method claiming to replace human evaluation for agentic systems should clearly bound the conditions under which this claim holds."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": [
         "Lianmin Zheng",
         "Wei-Lin Chiang",
         "Ying Sheng"
       ],
       "year": 2024,
       "relevance": "Foundational LLM-as-a-Judge framework that Agent-as-a-Judge extends; directly relevant to evaluation methodology for LLM-based systems."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": [
         "Carlos E Jimenez",
         "John Yang",
         "Alexander Wettig"
       ],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major agentic code generation benchmark that Agent-as-a-Judge is positioned as an alternative or complement to; widely used evaluation target in the field."
     },
     {
       "title": "OpenDevin: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": [
         "Xingyao Wang",
         "Boxuan Li",
         "Yufan Song"
       ],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "One of three developer agents evaluated on DevAI; represents state-of-the-art agentic code generation."
     },
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "authors": [
         "Sirui Hong",
         "Mingchen Zhuge",
         "Jonathan Chen"
       ],
       "year": 2024,
       "relevance": "One of three developer agents evaluated on DevAI; leading multi-agent code generation framework."
     },
     {
       "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",
       "authors": [
         "Qian Huang",
         "Jian Vora",
         "Percy Liang",
         "Jure Leskovec"
       ],
       "year": 2024,
       "relevance": "Related benchmark for evaluating agentic systems on ML tasks; compared against DevAI in the paper."
     },
     {
       "title": "MLE-Bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": [
         "Jun Shern Chan",
         "Neil Chowdhury",
         "Oliver Jaffe"
       ],
       "year": 2024,
       "relevance": "Related benchmark for ML engineering agent evaluation; discussed as having similar limitations to other existing benchmarks."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": [
         "John Yang",
         "Carlos E Jimenez",
         "Alexander Wettig"
       ],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "State-of-the-art software engineering agent; represents the class of systems that Agent-as-a-Judge is designed to evaluate."
     },
     {
       "title": "HumanEval: Evaluating Large Language Models Trained on Code",
       "authors": [
         "Mark Chen",
         "Jerry Tworek",
         "Heewoo Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Classic code generation benchmark discussed as inadequate for evaluating modern agentic systems; motivation for DevAI."
     },
     {
       "title": "Data Interpreter: An LLM Agent for Data Science",
       "authors": [
         "Sirui Hong",
         "Yizhang Lin",
         "Bangbang Liu"
       ],
       "year": 2024,
       "arxiv_id": "2402.18679",
       "relevance": "The MetaGPT variant used in the DevAI evaluation; represents agentic data science capability."
     },
     {
       "title": "ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate",
       "authors": [
         "Chi-Min Chan",
         "Weize Chen",
         "Yusheng Su"
       ],
       "year": 2023,
       "arxiv_id": "2308.07201",
       "relevance": "Related multi-LLM evaluation framework; compared against as an alternative approach to automated agent evaluation."
     },
     {
       "title": "Let's Verify Step by Step",
       "authors": [
         "Hunter Lightman",
         "Vineet Kosaraju",
         "Yura Burda"
       ],
       "year": 2023,
       "arxiv_id": "2305.20050",
       "relevance": "Process-supervised reward models (PRMs) discussed as a key application of Agent-as-a-Judge intermediate feedback."
     },
     {
       "title": "Language Agents as Optimizable Graphs",
       "authors": [
         "Mingchen Zhuge",
         "Wenyi Wang",
         "Louis Kirsch"
       ],
       "year": 2024,
       "arxiv_id": "2402.16823",
       "relevance": "Graph-based agent optimization framework (GPTSwarm); closely related to the graph module in Agent-as-a-Judge design."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": [
         "Chunqiu Steven Xia",
         "Yinlin Deng",
         "Soren Dunn",
         "Lingming Zhang"
       ],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Demonstrates LLMs solving SWE-Bench tasks without advanced agents; cited as evidence that SWE-Bench has been overfitted."
     }
   ]
 }