ai-research-survey

calibration.json (16819B)

---

 {
   "paper_slug": "adversarial-bug-reports-2025",
   "calibration_date": "2026-02-28",
   "model": "opus",
   "total_questions": 50,
   "agreement_count": 50,
   "disagreement_count": 0,
   "agreement_rate": 1.0,
   "disagreements": [],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1.2 states a complete replication package is provided with figshare DOI 10.6084/m9.figshare.31140619, including 'automation scripts for generating malicious bug reports, code to orchestrate the attack workflows, all defense mechanisms with prompts.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package (figshare DOI in Section 1.1.2) includes 'the full set of issues, generated patches, and execution trajectories' used in the experiments."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Tool versions are specified (SWE-agent v1.1.0, SW-ReX v1.2.2, Claude Sonnet claude-sonnet-4-20250514, DevStral:24B) but no requirements.txt, Dockerfile, conda environment, or comprehensive dependency specification is provided. Mentioning tool versions alone is insufficient per schema criteria."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1.2 describes a complete replication package via figshare containing automation scripts and orchestration code. The package includes prompts, seeds, and generated content for reproducing attacks and defenses."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Cost figures include standard deviations (e.g., '$0.87 ± $0.39'), but the primary results — attack success rates and detection rates — are reported as raw counts and percentages without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparative claims about defense mechanisms (e.g., o4-mini 47% vs GPT-4.1-mini 23%) are made without any statistical test, despite the small sample of 51 adversarial reports."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw counts and percentages but no formal effect sizes (Cohen's d, odds ratios, relative risk). Detection rate percentages are standalone metrics, not effect sizes with baseline context as defined by the schema."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "51 adversarial bug reports were generated with no justification for why this number was chosen and no power analysis. The allocation across five attack categories (5, 8, 10, 14, 14) appears driven by practical feasibility, not statistical considerations."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 acknowledges LLM non-determinism and that temperature could not be fixed to 0. All attack and detection results are from single runs with no variance measures. Cost standard deviations are reported but not variance for the main detection/success rate results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple defense mechanisms are systematically compared in Table 2: PromptGuard, PromptGuardV2, LlamaGuard v3 and v4, Granite Guardian, GPT-4.1-mini, o4-mini, CodeQL, and GitHub Copilot."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated defenses are contemporary (2023-2025): LlamaGuard4, PromptGuardV2, GPT-4.1-mini, o4-mini, Granite Guardian. SWE-agent was selected as top-ranked on SWE-bench verified leaderboard at evaluation start (May 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 evaluates individual defense components and their combinations (ensemble), effectively ablating the defense pipeline. Individual pre-APR filters are evaluated separately, then in combination with post-APR detection."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: attack success rate, per-defense detection rate, false-positive rate (on 100 real issues), cost per issue, and test suite failure categorization."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states: 'Manual validation was performed by two independent experts—a security researcher and a penetration tester—who jointly reviewed each issue and patch' to determine whether patches fulfilled attacker objectives."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "False-positive rate evaluation used 100 real issues from psf/requests (issues #6000-6100, excluding one deleted entry) as a separate validation set from the 51 adversarial bug reports."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-attack-type breakdown (Naive APR, Inject, CI/CD, Revert CVE, Noise APR) with detection counts and percentages for each defense mechanism."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sections 6.3-6.4 discuss defense failures: LlamaGuard detected 0/51 issues, PromptGuard detected only 3/51, CodeQL detected 0 malicious patches. Section 7.2 discusses broader failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4: 'Using CodeQL with the python rule-set was not able to detect any of the created patches as malicious.' LlamaGuard returned 0% detection. These are clear negative results reported without hedging."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are verified: 90% attack success (Table 2: 46/51 = 90.2%), best pre-filter blocked 47% (o4-mini unstructured: 24/51 = 47.05%), post-repair effective in 58% (Copilot all levels: 30/51 = 58.82%). All figures match."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim — adversarial bug reports cause insecure patches — is demonstrated through controlled experiments. Each adversarial report is crafted with a specific intent and submitted, then outcomes are verified by expert annotators. This controlled manipulation design supports causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly bounds findings: 'Our experiments focus on a subset of open-source projects and a single APR system (SWE-agent)' and 'our findings may not generalize to proprietary software or APR tools with substantially different architectures or training regimes.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific alternative explanations: annotation subjectivity (dual-annotator mitigation), LLM non-determinism (temperature not fixable without violating construct validity), and whether attack success metrics capture real downstream harm (construct validity)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers provided: SWE-agent v1.1.0, SW-ReX v1.2.2, Claude Sonnet (claude-sonnet-4-20250514), DevStral:24B, GPT-4.1-mini, o4-mini, LlamaGuard v3 and v4, PromptGuard, PromptGuardV2, Granite Guardian."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides the full Revert CVE attack prompt template and Figure 4 provides the full pre-APR issue classification prompt. The replication package contains all prompts with actual fill values."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 states 'when using LLMs, we used their default temperature' without specifying what those defaults are for each model. No temperature values, top-p, or max tokens are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Attack scaffolding is described in detail: context-building using LSH and rapidfuzz (Section 3.2.1), DevStral for generation, GitHub API workflow. Figure 1 illustrates the full APR pipeline architecture. SWE-agent configuration is described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.1 describes diff sanitization ('removing tests, documentation, and unrelated changes'). Section 5 describes attack allocation across types. False-positive study documents issue selection (#6000-6100 excluding one deleted entry)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Threats to Validity' is a dedicated section with substantive discussion covering internal validity, external validity, construct validity, and tool limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific: dual-annotator protocol for annotation subjectivity, inability to fix temperature to 0 without violating construct validity, single APR system (SWE-agent), and attack success metric not capturing downstream exploitation feasibility."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'Our experiments focus on a subset of open-source projects and a single APR system (SWE-agent)' and 'our findings may not generalize to proprietary software or APR tools with substantially different architectures or training regimes.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Figshare replication package (DOI: 10.6084/m9.figshare.31140619) includes 'the full set of issues, generated patches, and execution trajectories' for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes project selection (5 SWE-bench repositories), attack generation process (DevStral with context-building), and outcome recording (Copilot comments, CodeQL results, test suite outcomes, manual validation)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study generates adversarial bug reports and tests them on software repositories. The two expert annotators are researchers performing analysis, not study participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: adversarial issue generation → GitHub fork submission → SWE-agent processing → PR creation → test suite execution → CodeQL scan → Copilot review → manual expert validation. Each step is described in Sections 3-5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1.3: 'This work was supported by the Polish National Agency for Academic Exchange (NAWA) - BPN/BAT/2025/1/00019 and Agentur für Bildung und Internationalisierung (ÖeAD).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed on first page: Piotr Przymus at Nicolaus Copernicus University (Toruń, Poland), Andreas Happe and Jürgen Cito at TU Wien (Vienna, Austria). None affiliated with companies whose tools are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NAWA and ÖeAD are academic exchange agencies with no financial interest in the security of APR systems or any of the evaluated tools (OpenAI, Meta, GitHub, Anthropic products)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests (patents, equity, etc.) appears in the paper. Per schema, absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Claude Sonnet (claude-sonnet-4-20250514) is used for patch synthesis, and other LLMs for filtering, but no training data cutoff dates are stated for any model. The SWE-bench repositories are major Python projects likely present in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Claude Sonnet or other models may have been trained on data from the five target repositories (django, flask, pip, requests, scikit-learn). These are among the most prominent Python projects with extensive online presence."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The experiments use SWE-bench repositories with Claude Sonnet trained before May 2025. SWE-bench was published in 2024 and the repositories predate model training. This contamination risk is not discussed, though it could inflate the APR's patch generation success rate."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Expert annotators performing validation are researchers, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study operates on software repositories and generated code. Responsible disclosure practices are noted in Section 7.5."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The two expert annotators are described by role (security researcher, penetration tester) which is appropriate for a validation task."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not a human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.6 provides detailed cost analysis: attack generation ~$0.000295/report, GPT-4.1-mini pre-filter ~$0.00101/issue, o4-mini ~$0.00268/issue, APR execution $0.87 ± $0.39/issue. Costs reported with standard deviations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-issue API costs are reported but total computational budget is not stated. The paper mentions models ran locally on consumer-grade GPUs (Section 3.2.1) but gives no specifics on GPU hours or total expenditure for the full study."
       }
     }
   }
 }