ai-research-survey

scan.json (25165B)

---

 {
   "paper": {
     "title": "Process-Centric Analysis of Agentic Software Systems",
     "authors": ["Shuyang Liu", "Yang Chen", "Rahul Krishna", "Saurabh Sinha", "Jatin Ganhotra", "Reyhaneh Jabbarvand"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.02393",
     "doi": "10.48550/arXiv.2512.02393"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'artifacts' and providing a 'rich dataset' for the community but does not provide a repository URL or download link for source code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states it provides 'the community with a rich dataset to explore future process-centric analyses algorithms on top of Graphectory and Langutory' (§1) and uses SWE-bench Verified which is publicly available. However, no explicit download link is given for the 4000 trajectories. The benchmark itself (SWE-bench Verified) is public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned. The paper describes experimental setup but not the software environment needed to reproduce."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README are provided. The paper describes the experimental setup (§3.1) but not how to reproduce the pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports median and Interquartile Range (IQR) for all process-centric metrics in Figure 3: 'each heatmap cell shows the median of the metric and Interquartile Range (IQR), representing the spread around the median.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Mann-Whitney U test (§3.2.3, Figure 5-a) to test correlation between metrics and repair status, and Kendall's τ_b test (§3.2.4, Figure 5-b) for difficulty alignment. P-values are reported with significance threshold p ≤ 0.05."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper reports p-values from statistical tests and median/IQR values, it does not report effect sizes (e.g., Cohen's d, rank-biserial correlation) for the Mann-Whitney U or Kendall's τ_b tests."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of 500 issues × 8 configurations = 4000 trajectories is stated but not justified. No power analysis or justification for why 500 issues is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "IQR is reported for all metrics in Figure 3. The paper states 'reporting the median and interquartile range values' (§3.1)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares two agents (SWE-agent, OpenHands) across four LLMs, providing cross-agent and cross-model comparisons throughout §3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All agents and LLMs are contemporary: SWE-agent, OpenHands, DeepSeek-V3, DeepSeek-R1, Devstral-small-2505, Claude Sonnet 4. All are 2024-2025 systems."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Graphectory is an analysis framework, not a system with components to ablate. The paper's contribution is the representation and metrics, not a system being optimized."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six process-centric metrics are defined and used: Node Count, Temporal Edge Count, Loop Count, Average Loop Length, Structural Edge Count, Structural Breadth (Table 1, §2.2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two human annotators independently labeled actions with phases for the phase map (§2.1). Manual review of 15% of Graphectory (600 samples) was performed for anti-pattern identification (§3.4). Manual investigation of trajectories is referenced throughout."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an analysis study, not a prediction task. There is no train/test split to evaluate."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by agent, by LLM, by repair status (resolved/unresolved), and by problem difficulty (easy/medium/hard/very hard) throughout §3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis is provided: §3.4 identifies nine anti-patterns with qualitative examples (Figures 10-13), and unresolved trajectories are analyzed throughout."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that stronger models (Claude Sonnet 4) exhibit more inefficiencies despite higher success rates (§3.4), and that even successful runs contain significant inefficiencies. DeepSeek-R1's format compliance issues are noted (§3.2.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The three main abstract claims (richer prompts/stronger LLMs → more complex Graphectory; strategies vary with difficulty; successful runs still inefficient) are all supported by results in §3.2-3.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims (e.g., 'agents using richer prompts or stronger LLMs exhibit more complex Graphectory, reflecting deeper exploration'). This conflates correlation with causation — richer prompts and stronger LLMs are confounded with agent design differences. The study is observational, not controlled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Agentic Software Systems' broadly, and the paper claims Graphectory 'applies broadly to any agentic system' (§1), but results are only on two programming agents on SWE-bench Verified. No non-SE agents are evaluated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the correlation between metric complexity and repair status could be confounded by timeout limits rather than agent behavior quality. No threats to validity section exists."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses graph metrics (NC, TEC, LC, etc.) as proxies for 'effort', 'complexity', and 'efficiency' without discussing whether these metrics actually capture those constructs. Higher node count is equated with more effort without validation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Models are specified: 'DeepSeek-V3 671B MoE', 'DeepSeek-R1', 'Devstral-small-2505 24B', 'Claude Sonnet 4' (§3.1). Devstral includes the version suffix '2505'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper references SWE-agent and OpenHands system prompts via URLs to GitHub (refs [38], [45]) but does not include the actual prompt text. The GPT-5 annotation prompt is also not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other LLM hyperparameters are reported. Only agent-level configs are mentioned: '$2 cost cap for SWE-agent, 100 iterations for OpenHands' (§3.1)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Both SWE-agent and OpenHands scaffolding are described: their tool usage, system prompts, and workflow differences are discussed in §2.1 and §3.1. The agents' default configurations and tools are specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The Graphectory construction pipeline is documented: trajectory log parsing, node/edge construction, phase labeling via Algorithm 1, and Langutory construction via Algorithm 2 (§2.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations or threats-to-validity section in the paper. The conclusion (§5) briefly mentions future directions but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Specific concerns such as the impact of cost/iteration limits, the representativeness of SWE-bench, or the reliability of automated phase labeling are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims broad applicability ('Graphectory applies broadly to any agentic system') without stating specific scope boundaries. It does not explicitly state what the results do NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions providing a 'rich dataset' but no download link or data repository is provided for the 4000 trajectories or constructed Graphectory."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: 500 SWE-bench Verified issues, 8 agent-model pairs, default configurations, Pass@1 results (§3.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is SWE-bench Verified, a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from trajectory logs to Graphectory to metrics/analyses is fully documented with formal definitions (§2) and algorithms (Algorithms 1-3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: UIUC and IBM Research. IBM Research authors are evaluating agents that are not IBM products, so no direct product conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. IBM Research affiliations could imply corporate interest in agent analysis tooling."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLMs on SWE-bench Verified but does not state training cutoff dates for any of the four models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the SWE-bench Verified issues appeared in the training data of DeepSeek-V3, DeepSeek-R1, Devstral, or Claude Sonnet 4."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench was published in 2023 and SWE-bench Verified in 2024. All models were potentially trained after these dates. No contamination discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The human annotators for phase labeling are part of the methodology, not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "SWE-agent has a $2 cost cap per instance but actual costs incurred are not reported. No aggregate cost figures are given for the 4000 trajectories."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or API spend is reported for running 4000 trajectories across 4 LLMs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges LLMs are 'inherently non-deterministic' (§3.1) but reports only Pass@1 single-run results. No multi-seed analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states Pass@1 results (first submission per issue) and 500 issues × 8 configurations = 4000 trajectories (§3.1)."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter search was performed; default configurations are used for both agents."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection was performed; the paper uses default settings for each framework."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple Mann-Whitney U and Kendall's τ_b tests are performed across 8 agent-model pairs × 6 metrics (Figures 5a-5b) without any multiple comparison correction mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not propose a new agent system; it analyzes existing agents. No self-comparison bias applies."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different LLMs likely have very different compute costs, but performance is not reported as a function of compute budget. The $2 cost cap for SWE-agent is mentioned but not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses SWE-bench Verified without discussing its construct validity — whether it represents real-world software engineering tasks, or whether Pass@1 on human-validated issues measures agent quality."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly compares agents across the same LLMs and LLMs across the same agents, treating both agent scaffold and LLM as variables. Results are broken down by agent-model pair throughout §3."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench issues are from real GitHub repositories. No discussion of whether these issues/solutions appeared in model training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information (e.g., issue descriptions containing solution hints)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench issues come from 12 repositories but no discussion of whether code from these repositories appeared in training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Agents using richer prompts or stronger LLMs exhibit more complex Graphectory, reflecting deeper exploration and more thorough validation",
       "evidence": "Figure 3 shows OpenHands (richer prompts) has consistently higher metric values than SWE-agent. Claude Sonnet 4 produces more complex Graphectory than other LLMs (§3.2.1-3.2.2).",
       "supported": "moderate"
     },
     {
       "claim": "Process-centric metrics effectively distinguish successful and unsuccessful repairs",
       "evidence": "Mann-Whitney U tests show significant differences (p ≤ 0.05) for most agent-model pairs (Figure 5-a), except Claude Sonnet 4 which shows non-significant differences.",
       "supported": "moderate"
     },
     {
       "claim": "Trajectory complexity grows with task difficulty, aligned with human difficulty ratings",
       "evidence": "Kendall's τ_b test shows significant positive trends between metrics and difficulty (Figure 5-b), with qualitative examples in Figure 6.",
       "supported": "strong"
     },
     {
       "claim": "Even successful agents exhibit significant inefficiency during problem solving",
       "evidence": "Anti-pattern analysis (§3.4, Figures 14-15) shows localization and patching inefficiencies are prevalent even in resolved instances, though at lower rates than unresolved ones.",
       "supported": "strong"
     },
     {
       "claim": "Stronger models (Claude Sonnet 4) exhibit more inefficiency patterns despite higher success rates",
       "evidence": "Figures 14-15 show Claude Sonnet 4 has higher rates of localization inefficiency patterns. Manual analysis suggests extended internal reasoning (§3.4).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Graphectory, a graph-based representation of agentic trajectories, enables systematic process-centric analysis beyond outcome-centric evaluation. Analysis of 4000 trajectories from SWE-agent and OpenHands across four LLMs reveals that unsuccessful runs are longer with more repetitions, trajectory complexity correlates with task difficulty, and even successful agents exhibit significant inefficiencies. Nine anti-patterns are identified in localization and patching phases, with stronger models paradoxically showing more inefficiency despite higher success rates.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations or threats-to-validity section despite making broad claims about agentic systems generally from only two programming agents on one benchmark."
     },
     {
       "flag": "Overclaimed generalizability",
       "detail": "The paper claims Graphectory 'applies broadly to any agentic system' but only evaluates two programming agents on SWE-bench Verified. No evidence for web navigation, robotics, or scientific discovery domains mentioned in the introduction."
     },
     {
       "flag": "No contamination discussion",
       "detail": "Uses SWE-bench Verified with four LLMs that could have been trained on SWE-bench data. Resolution rates may be inflated by data contamination, which would affect the process-centric analysis of resolved vs. unresolved trajectories."
     },
     {
       "flag": "Single-run results with acknowledged non-determinism",
       "detail": "The paper acknowledges LLMs are 'inherently non-deterministic' but reports only Pass@1 single-run results. Trajectory patterns could vary significantly across runs of the same problem."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-agent: agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2025,
       "relevance": "One of the two primary agentic programming systems analyzed in this paper."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang"],
       "year": 2025,
       "arxiv_id": "2407.16741",
       "relevance": "The other primary agentic programming system analyzed in this paper."
     },
     {
       "title": "Swe-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "relevance": "The benchmark used for evaluation (SWE-bench Verified subset)."
     },
     {
       "title": "Agentless: Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Pipeline-based alternative to agentic approaches for automated issue resolution."
     },
     {
       "title": "TRAIL: Trace Reasoning and Agentic Issue Localization",
       "authors": ["Darshan Deshpande", "Varun Gangal", "Hersh Mehta"],
       "year": 2025,
       "relevance": "Prior work on taxonomy of errors in agent trajectories from GAIA and SWE-Bench Lite."
     },
     {
       "title": "Why do multi-agent llm systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "relevance": "Proposes MAST failure taxonomy for multi-agent systems, requiring human annotations."
     },
     {
       "title": "Understanding Software Engineering Agents Through the Lens of Traceability",
       "authors": ["Ira Ceka", "Saurabh Pujar"],
       "year": 2025,
       "relevance": "Manual analysis of agent action flow graphs for software engineering agents."
     },
     {
       "title": "An Empirical Study on Failures in Automated Issue Solving",
       "authors": ["Simiao Liu", "Fang Liu"],
       "year": 2025,
       "relevance": "Develops taxonomy of LLM failure modes for SWE tasks with manual analysis of 150 issues."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao"],
       "year": 2022,
       "relevance": "Foundational ReAct paradigm used by the agentic systems analyzed."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu"],
       "year": 2023,
       "relevance": "Benchmark for evaluating LLMs as agents across diverse tasks."
     },
     {
       "title": "DeepSeek-V3 Technical Report",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2412.19437",
       "relevance": "One of the four backbone LLMs evaluated in the study."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "relevance": "Reasoning model used as one of the four backbone LLMs."
     }
   ]
 }