ai-research-survey

scan.json (28964B)

---

 {
   "paper": {
     "title": "Adversarial Bug Reports as a Security Risk in Language Model-Based Automated Program Repair",
     "authors": ["Piotr Przymus", "Andreas Happe", "Jürgen Cito"],
     "year": 2026,
     "venue": "23rd International Conference on Mining Software Repositories (MSR '26)",
     "arxiv_id": "2509.05372",
     "doi": "10.1145/3793302.3793352"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1.2 states 'We provide a complete replication package, including automation scripts for generating malicious bug reports, code to orchestrate the attack workflows, all defense mechanisms with prompts, as well as the full set of issues, generated patches, and execution trajectories' with DOI 10.6084/m9.figshare.31140619 (reference [12])."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package (figshare DOI provided) includes 'the full set of issues, generated patches, and execution trajectories' used in the experiments, as stated in Section 1.1.2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper names specific tool versions (SWE-agent v1.1.0, SW-ReX v1.2.2, Claude Sonnet claude-sonnet-4-20250514) and names the models used (DevStral:24B, GPT-4.1-mini, o4-mini, etc.) but does not provide a requirements file, Dockerfile, or comprehensive environment specification sufficient to recreate the full experimental environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "A complete replication package is provided via figshare (Section 1.1.2), which the authors describe as containing automation scripts and code to orchestrate attack workflows. The paper references the package for full reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Cost figures include standard deviations (e.g., mean APR cost '$0.87 ± $0.39'), but the main detection/attack success rates are reported as raw counts and percentages without confidence intervals or error bars. The core results in Table 2 are point estimates only."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. The paper compares detection rates across defense mechanisms (e.g., o4-mini detecting 47% vs. GPT-4.1-mini detecting 23%) without any statistical tests to establish whether differences are significant given the small sample of 51 attacks."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw counts and percentages but does not report effect sizes in a statistical sense (e.g., Cohen's d, odds ratios). Percentage comparisons are given but without baseline context sufficient to constitute formal effect size reporting."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper generated 51 adversarial bug reports but does not justify why 51 was chosen or discuss statistical power. The sample size appears to be determined by practical feasibility rather than a principled analysis of what N is needed to detect differences in defense effectiveness."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "LLMs are noted to be non-deterministic (Section 8) and the paper explicitly states they could not fix temperature to 0. However, results are reported from single runs without variance measures across repeated trials. Cost standard deviations are reported but not variance for detection rates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple defense mechanisms are compared against each other: PromptGuard, PromptGuardV2, LlamaGuard v3 and v4, Granite Guardian, GPT-4.1-mini (custom prompt), o4-mini (custom prompt), CodeQL, and GitHub Copilot. The study compares these systematically in Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated defense mechanisms are contemporary (2023-2025 models), including LlamaGuard4, PromptGuardV2, GPT-4.1-mini, o4-mini, and Granite Guardian. The APR system SWE-agent v1.1.0 was selected as the top-ranked system on SWE-bench verified leaderboard at evaluation start."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates individual defense components and their combinations (ensemble), effectively performing an ablation of the defense pipeline. Section 6.5 explicitly evaluates how combining pre- and post-APR countermeasures changes detection rates."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses multiple metrics including: attack success rate, detection rate per defense mechanism, false-positive rate (on 100 real issues), cost per issue, and test suite failure rates. Table 2 captures several of these dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states 'Manual validation was performed by two independent experts—a security researcher and a penetration tester—who jointly reviewed each issue and patch' to determine whether patches fulfilled attacker objectives."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The false-positive rate evaluation used 100 real issues from the psf/requests project (issues #6000-6100) as a separate validation set from the 51 adversarial bug reports used in the main attack evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides a detailed per-attack-type breakdown (Naive APR, Inject, CI/CD, Revert CVE, Noise APR) with counts and percentages for each defense mechanism, allowing comparison across attack categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses failure cases of defense mechanisms (e.g., LlamaGuard failing to detect any malicious issues, PromptGuard detecting only 3/51). Section 7.2 explicitly discusses limitations and failure modes of the defense strategies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 reports that 'Using CodeQL with the python rule-set was not able to detect any of the created patches as malicious' — a clear negative result. LlamaGuard also returned zero detections (0%), and PromptGuard detected only 3/51 (5.88%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 90% of crafted bug reports triggered attacker-aligned patches (supported by Table 2: 46/51=90.2%), best pre-repair filter blocked only 47% (o4-mini unstructured: 24/51=47.05%), and post-repair was effective in 58% (Copilot all-levels: 30/51=58.82%). All abstract figures are consistent with results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about adversarial bug reports causing insecure patches, but this is demonstrated directly through controlled experiments where each adversarial report was manually crafted and submitted, and outcomes were verified by expert annotators. The controlled experiment design (attack → observed outcome) provides adequate causal identification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly bounds external validity: 'Our experiments focus on a subset of open-source projects and a single APR system (SWE-agent). While the selected repositories are diverse and representative...our findings may not generalize to proprietary software or APR tools with substantially different architectures.' The limitations section is specific about scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Threats to Validity) discusses internal validity threats (subjectivity in manual annotation, LLM non-determinism), external validity (single APR system, open-source only), and construct validity (whether the attack success definition captures downstream harm). These are specific threats to this study."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies: SWE-agent v1.1.0 with SW-ReX v1.2.2, Claude Sonnet (claude-sonnet-4-20250514), DevStral:24B, GPT-4.1-mini, o4-mini, LlamaGuard v3 and v4, PromptGuard and PromptGuardV2, Granite Guardian. These are sufficiently specific model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows the full template used for the Revert CVE attack, and Figure 4 shows the full template for pre-APR issue classification. The actual prompt texts are provided in the paper, not just descriptions of what they do."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 states 'when using LLMs, we used their default temperature' but does not specify what that default temperature is for each model. No other hyperparameters (top-p, max tokens, etc.) are reported. Using defaults without stating what those defaults are is insufficient for reproducibility."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The attack scaffolding is described in detail: context-building using LSH and rapidfuzz (Section 3.2.1), the DevStral model for generation, the GitHub API workflow for submitting issues and PRs, and the SWE-agent configuration. Figure 1 illustrates the full pipeline architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.1 describes how diffs are sanitized ('removing tests, documentation, and unrelated changes') for context-building. Section 5 describes how the 51 adversarial reports were allocated across attack types. The false-positive study used issues #6000-6100 excluding one deleted entry."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is titled 'Threats to Validity' and provides substantive discussion of internal, external, construct, and tool limitations as separate subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: the dual-annotator protocol addresses annotation subjectivity; LLM non-determinism is acknowledged with explicit note that temperature could not be fixed without violating construct validity; single APR system limitation is acknowledged with justification for why similar vulnerabilities would likely manifest in others."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states the work is limited to 'open-source projects and a single APR system (SWE-agent)' and that findings 'may not generalize to proprietary software or APR tools with substantially different architectures or training regimes.' Construct validity section notes attack success definitions may not capture downstream harm."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package (figshare DOI: 10.6084/m9.figshare.31140619) includes 'the full set of issues, generated patches, and execution trajectories,' providing raw data for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes the data collection in detail: which projects were used (django, flask, pip, requests, scikit-learn from SWE-bench), how attack reports were generated (DevStral with context-building), and how outcomes were recorded (Copilot comments, CodeQL results, test suite outcomes, manual validation)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a controlled experiment on software repositories with no participant recruitment; the 'data' is adversarial bug reports generated by the authors themselves and existing OSS repositories. NA for this criterion."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: adversarial issue generation → submission to GitHub forks → SWE-agent processing → PR creation → test suite execution → CodeQL scan → Copilot review → manual expert validation. Each step is described in Sections 3-5 and outcomes are captured in Table 2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1.3 states 'This work was supported by the Polish National Agency for Academic Exchange (NAWA) - BPN/BAT/2025/1/00019 and Agentur für Bildung und Internationalisierung (ÖeAD).' Funding is clearly disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are stated on the first page: Piotr Przymus at Nicolaus Copernicus University, Andreas Happe and Jürgen Cito at TU Wien. None of the authors are affiliated with OpenAI, Anthropic, Meta, or GitHub, whose tools are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (NAWA and ÖeAD) are academic exchange agencies with no financial stake in the outcomes of evaluating APR security or any of the evaluated tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration of financial interests (patents, equity, etc.) in the paper. Absence of such a declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Claude Sonnet (claude-sonnet-4-20250514) and other LLMs for patch generation and filtering, but does not state the training data cutoffs for any of the models. The SWE-bench benchmark is used as the source of target repositories, which raises contamination concerns."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether SWE-agent (using Claude Sonnet) may have been trained on data from the five target repositories (django, flask, pip, requests, scikit-learn), which are all major Python projects with extensive online presence. This is a relevant concern but is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The experiments use repositories from SWE-bench, and the APR system uses Claude Sonnet trained before May 2025. The SWE-bench benchmark was published in 2024, meaning these repositories and potentially their issue histories were available during model training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper involves expert manual validation of patches by two annotators, but this is not a human subjects study — the humans are researchers performing analysis, not participants. Human studies criteria do not apply."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; the study operates on software repositories and generated code. IRB approval is not applicable. The paper does note responsible disclosure practices in Section 7.5."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study; the two expert annotators are described by role only (security researcher, penetration tester), which is appropriate for the validation task rather than a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participant recruitment; not a human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.6 provides detailed cost analysis: attack generation cost $0.000295 per bug report, GPT-4.1-mini pre-filter cost $0.00101 per issue, o4-mini cost $0.00268 per issue, APR execution cost $0.87 ± $0.39 per issue. Costs are reported with standard deviations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "API costs are reported per-issue but total computational budget (total API spend for the study, hardware used for local model runs) is not stated. The paper mentions models ran locally on consumer-grade GPUs but gives no specifics on GPU hours or total expenditure."
       }
     }
   },
   "claims": [
     {
       "claim": "90% of crafted adversarial bug reports triggered attacker-aligned patches when submitted to SWE-agent.",
       "evidence": "Table 2 shows 46 out of 51 adversarial issues (90.2%) led to patches consistent with attacker objectives. Section 6.1 provides breakdown by attack type: all 8 Noise APR, 5/5 Naive APR (noise), all 10 CI/CD, 10/14 Injection, 13/14 Revert CVE.",
       "supported": "strong"
     },
     {
       "claim": "The best pre-repair filter (o4-mini with unstructured output) blocked only 47% of adversarial bug reports.",
       "evidence": "Table 2 and Section 6.3 report o4-mini unstructured output detected 24/51 issues (47.05%), making it the best-performing pre-APR detection mechanism among those tested.",
       "supported": "strong"
     },
     {
       "claim": "CodeQL static analysis failed to detect any of the adversarial patches as malicious.",
       "evidence": "Section 6.4 states 'Using CodeQL with the python rule-set was not able to detect any of the created patches as malicious.' Table 2 shows 0 detections for CodeQL across all 51 cases.",
       "supported": "strong"
     },
     {
       "claim": "Generating adversarial bug reports is orders of magnitude cheaper than defending against them, creating a structural asymmetry that favors attackers.",
       "evidence": "Section 6.6 reports attack cost ~$0.000295/report, pre-filter cost ~$0.00268/issue (o4-mini), APR execution ~$0.87/issue — approximately 3 orders of magnitude difference between attack and APR cost.",
       "supported": "strong"
     },
     {
       "claim": "Project test suites are insufficient to detect adversarial patches: 68.6% of all adversarial patches produced no new test failures.",
       "evidence": "Section 6.2 reports 35/51 (68.6%) adversarial patches showed no additional test failures; 12/51 (23.5%) failed the test suite; 4/51 (7.8%) increased failures within expected thresholds.",
       "supported": "strong"
     },
     {
       "claim": "Combining best pre-APR filter and Copilot post-APR review blocks 68.6% of attacks, leaving 31.4% undetected.",
       "evidence": "Section 6.5 reports the targeted configuration (o4-mini + Copilot) blocked 35/51 attacks (68.6%), while the full pre-APR ensemble + Copilot blocked 37/51 (72.5%).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "LLM-based APR systems are highly vulnerable to adversarial bug reports: 46/51 crafted reports (90.2%) induced attacker-aligned patches including CVE reversions, vulnerability injections, and CI/CD exploits. Current defenses are severely insufficient — the best pre-filter (o4-mini) blocked only 47% of attacks, CodeQL detected zero malicious patches, and GitHub Copilot flagged only 58% requiring manual review to be useful. A fundamental economic asymmetry exists: adversarial bug report generation costs ~$0.0003 per report while APR execution costs ~$0.87 per issue, structurally favoring attackers. The paper contributes an open-source framework for generating and evaluating adversarial bug reports to support future security testing of APR systems.",
   "red_flags": [
     {
       "flag": "Small sample size (n=51) for comparing many defense mechanisms",
       "detail": "The study evaluates 10+ defense mechanisms on only 51 adversarial bug reports, which is too small to reliably detect statistically significant differences between mechanisms. Differences like 23% vs 35% detection rate between GPT-4.1-mini structured vs o4-mini structured could easily be noise at this sample size."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes comparative claims about defense effectiveness (e.g., o4-mini outperforms GPT-4.1-mini) without any statistical tests. With n=51, many of these differences are not distinguishable from chance."
     },
     {
       "flag": "LLM non-determinism not controlled",
       "detail": "Section 8 acknowledges that LLMs are non-deterministic and that temperature could not be set to 0 (to maintain construct validity of default configurations). All results are from single runs, so result variability across repeated experiments is unknown."
     },
     {
       "flag": "Single APR system evaluated",
       "detail": "Only SWE-agent is evaluated as the APR system. While the paper argues structural properties are shared across systems, the attack success rates may differ substantially for other APR systems with different prompt engineering or safety measures."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The study uses repositories from SWE-bench with Claude Sonnet (trained before May 2025) as the APR backend. The SWE-bench repositories and issues may have been in Claude's training data, potentially inflating APR success rates on these specific projects."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "doi": "10.48550/arXiv.2310.06770",
       "relevance": "Primary benchmark used in the study; core evaluation platform for LLM-based APR systems."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik R Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "The APR system evaluated in this paper; a leading LLM agent for automated code repair."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "doi": "10.48550/arXiv.2403.17134",
       "relevance": "Related LLM-based APR system demonstrating autonomous agent-based program repair."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "arxiv_id": "2407.16741",
       "relevance": "Related agentic AI system for software engineering; cited as an example of APR system with known prompt injection vulnerabilities."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "doi": "10.1145/3650212.3680384",
       "relevance": "Another leading LLM-based APR system; cited as sharing structural properties that make it potentially vulnerable to similar attacks."
     },
     {
       "title": "Not what you've signed up for: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Foundational work on indirect prompt injection attacks against LLM applications, directly relevant to this paper's threat model."
     },
     {
       "title": "Llama guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["Hakan Inan", "Kartikeya Upasani", "Jianfeng Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "LlamaGuard defense mechanism evaluated in this paper as a pre-APR filter."
     },
     {
       "title": "Granite Guardian",
       "authors": ["Inkit Padhi", "Manish Nagireddy", "Giandomenico Cornacchia"],
       "year": 2024,
       "arxiv_id": "2412.07724",
       "relevance": "IBM's safety classifier evaluated as a pre-APR defense mechanism in this paper."
     },
     {
       "title": "Jailbreak Attacks and Defenses Against Large Language Models: A Survey",
       "authors": ["Sibo Yi", "Yule Liu", "Zhen Sun", "Tianshuo Cong", "Xinlei He", "Jiaxing Song", "Ke Xu", "Qi Li"],
       "year": 2024,
       "arxiv_id": "2407.04295",
       "relevance": "Survey of jailbreak attacks and defenses relevant to pre-APR filtering design."
     },
     {
       "title": "Automatic Programming: Large Language Models and Beyond",
       "authors": ["Michael R. Lyu", "Baishakhi Ray", "Abhik Roychoudhury", "Shin Hwei Tan", "Patanamon Thongtanunam"],
       "year": 2024,
       "arxiv_id": "2405.02213",
       "doi": "10.48550/arXiv.2405.02213",
       "relevance": "Survey on LLM-based automated programming including APR; relevant background for the survey's scope."
     },
     {
       "title": "Evaluating agent-based program repair at Google",
       "authors": ["Pat Rondon", "Renyao Wei", "José Cambronero", "Jürgen Cito", "Aaron Sun", "Siddhant Sanyam", "Michele Tufano", "Satish Chandra"],
       "year": 2025,
       "arxiv_id": "2501.07531",
       "relevance": "Industrial-scale evaluation of LLM-based APR at Google; provides context for real-world APR deployment."
     },
     {
       "title": "Defeating Prompt Injections by Design",
       "authors": ["Edoardo Debenedetti", "Ilia Shumailov", "Tianqi Fan", "Jamie Hayes", "Nicholas Carlini"],
       "year": 2025,
       "arxiv_id": "2503.18813",
       "relevance": "Defense mechanism (CaMeL) against prompt injection attacks relevant to APR security."
     },
     {
       "title": "SWE-bench Multimodal: Do AI Systems Generalize to Visual Software Domains?",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alex L. Zhang", "Kilian Lieret"],
       "year": 2024,
       "arxiv_id": "2410.03859",
       "doi": "10.48550/arXiv.2410.03859",
       "relevance": "Extension of SWE-bench benchmark; relevant as broader evaluation framework for LLM software engineering."
     }
   ]
 }