ai-research-survey

calibration.json (20553B)

---

 {
   "calibration": {
     "paper_slug": "agentless-2024",
     "calibration_date": "2026-02-28",
     "opus_model": "claude-opus-4-6",
     "sonnet_scan_date": null,
     "agreement_rate": 0.96,
     "total_questions": 50,
     "agreements": 48,
     "disagreements": 2,
     "disagreement_details": [
       {
         "category": "evaluation_design",
         "question": "human_evaluation",
         "sonnet": {"applies": true, "answer": true},
         "opus": {"applies": true, "answer": false},
         "direction": "sonnet_generous",
         "explanation": "Sonnet credits the manual classification of SWE-bench Lite problems (Section 6.1) as human evaluation. However, the schema is explicit: 'The humans must be evaluating what the system produced — manual classification of the benchmark or dataset itself does not count.' The Section 6.1 classification is about classifying the benchmark problems by description quality, solution presence, and location information — not evaluating AGENTLESS's outputs. There is no human evaluation of the system's generated patches, localizations, or reproduction tests. The evaluation of system outputs is entirely automated via test suites. Sonnet even acknowledges this in its justification ('though not of system outputs directly') but still answers YES."
       },
       {
         "category": "conflicts_of_interest",
         "question": "funder_independent_of_outcome",
         "sonnet": {"applies": false, "answer": false},
         "opus": {"applies": true, "answer": false},
         "direction": "applies_boundary",
         "explanation": "Sonnet sets applies=false reasoning that no funder is disclosed, so the criterion cannot be assessed. However, the schema says 'NA if unfunded' — meaning applies=false only if the work is confirmed to be unfunded. The paper does not disclose any funding, but absence of disclosure is not confirmation of no funding. The authors are at a major research university (UIUC) and used substantial OpenAI API credits for experiments. It is reasonable to expect funding disclosure exists but was omitted. Opus sets applies=true, answer=false because we cannot confirm unfunded status and absence of disclosure itself is a methodological concern."
       }
     ],
     "opus_checklist": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The abstract states 'We have open-sourced AGENTLESS at: https://github.com/OpenAutoCoder/Agentless'. A working GitHub URL is provided directly in the paper."
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The evaluation uses SWE-bench Lite, a publicly available standard benchmark. The paper did not collect proprietary data. SWE-bench is referenced as a public benchmark created by others."
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Section 4 mentions GPT-4o, LlamaIndex, OpenAI's text-embedding-3-small, and Python's ast library, but no requirements.txt, Dockerfile, or pinned dependency versions are provided in the paper."
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes the approach's pipeline steps in detail (Section 3) and implementation parameters (Section 4), but does not include step-by-step reproduction instructions with commands to run. These may exist in the GitHub repo but are not in the paper."
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are point estimates only (e.g., '96 (32.00%)' resolved, '$0.70' cost). No confidence intervals, error bars, or uncertainty ranges are reported anywhere in the paper."
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims AGENTLESS outperforms all open-source agents (32.00% vs 30.67%) but uses no statistical significance tests. All comparisons are based on raw percentage differences."
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": false,
           "justification": "No formal effect sizes (Cohen's d, odds ratios, relative risk) are reported. Results are presented as raw solve counts and percentages. While the paper provides baseline context (e.g., 32.00% vs 30.67%), these are raw numbers, not standardized effect size measures."
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses SWE-bench Lite (300 problems) without any justification for why this sample size is adequate, nor any power analysis. The benchmark is taken as given."
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results are single-run outcomes. No standard deviations, variance across seeds, or multi-run results with spread measures are reported. Figure 6 shows performance vs. number of samples but this is not variance across repeated runs."
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 compares against 26 agent-based approaches including both open-source (SWE-agent, AutoCodeRover, Moatless, Aider) and closed-source commercial tools. A RAG agentless baseline is also included."
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include contemporary 2024 tools: CodeStory Aide, Bytedance MarsCode, SWE-agent with Claude 3.5 Sonnet, AutoCodeRover-v2, and many others from the SWE-bench leaderboard."
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.2 presents extensive ablation studies covering localization (Table 2), repair (Table 3), and patch validation (Table 4). Individual design choices are systematically varied."
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports % Resolved, Avg. $ Cost, Avg. # Tokens, and % Correct Location at three granularities (line, function, file). Section 5.1.3 also evaluates reproduction test quality."
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "Section 6.1 describes manual classification of SWE-bench Lite problems, but the schema explicitly states 'manual classification of the benchmark or dataset itself does not count.' There is no human evaluation of AGENTLESS's outputs (generated patches, localizations, or reproduction tests). The evaluation of system outputs is entirely automated via test suites."
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "SWE-bench Lite is used as a fixed test set. No part of the benchmark is used for tuning the approach. The paper explicitly notes (footnote 2) that ground truth patches are only used for evaluation of reproduction test quality, not for the AGENTLESS process itself."
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Figure 9 shows solve rates broken down by problem category (description quality, solution type, location information) for multiple tools. Tables 2-4 provide per-component breakdowns."
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.2 identifies that closed-source agent tools outperform AGENTLESS on problems with no location clues. Section 5.1.3 discusses the reproduction test drop-off (213 selected to 94 plausible). These are specific failure modes."
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The ablation studies report configurations that hurt performance: Table 2 shows 'direct from file-level' localization is worse (47.00% vs 56.33%), Table 3 shows merged multi-samples (28.33%) underperform multi-samples (32.00%)."
         }
       },
       "claims_and_evidence": {
         "abstract_claims_supported": {
           "applies": true,
           "answer": true,
           "justification": "The abstract claims 32.00% solve rate (96 correct fixes) at $0.70 cost among open-source approaches, supported by Table 1. The claim about OpenAI adoption is discussed in Section 5.1.4."
         },
         "causal_claims_justified": {
           "applies": true,
           "answer": true,
           "justification": "The ablation studies (Section 5.2) use controlled single-variable manipulation: one component is varied while others remain at default settings. This is adequate causal design for claims like 'reproduction tests improve performance from 81 to 96 issues.'"
         },
         "generalization_bounded": {
           "applies": true,
           "answer": false,
           "justification": "The paper tests only on Python repositories in SWE-bench Lite, but the title 'Demystifying LLM-based Software Engineering Agents' and the broad framing ('autonomous software development') imply generality beyond Python bug-fixing. Section 7 notes results may not generalize but the framing throughout overstates the scope."
         },
         "alternative_explanations_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Section 7 mentions data leakage as an internal threat but does not discuss alternative explanations for why AGENTLESS outperforms agents. The paper does not consider whether the advantage stems from benchmark characteristics (Python-only, well-formed issues with location clues) rather than the agentless approach itself. No discussion of whether GPT-4o specifically is well-suited to the fixed-pipeline approach vs other models."
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 explicitly states 'GPT-4o (gpt-4o-2024-05-13)' with the specific API version, and 'text-embedding-3-small' for embeddings."
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes what prompts do in natural language (e.g., 'we prompt the LLM to localize and rank the top N most suspicious files') but does not provide actual prompt text. No appendix with full prompts."
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 specifies: greedy decoding by default, sampling temperature of 0.8, chunk size 512, chunk overlap 0, top 3 suspicious files, 4 samples of edit locations, 10 patches per location set, 40 total patches, 40 reproduction test samples, and context window of +/- 10 lines."
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The three-phase pipeline (localization, repair, patch validation) is described in detail in Section 3 with workflow diagram (Figure 1). Each step's inputs and outputs are explicitly described. While AGENTLESS is explicitly NOT agentic, it does use LLM-based scaffolding (hierarchical prompting, reproduction test generation, majority voting) which is well-described."
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.1 describes the manual classification procedure with categories and criteria. Section 6.2 documents the filtering from SWE-bench Lite (300) to SWE-bench Lite-S (249) with explicit exclusion criteria."
         }
       },
       "limitations_and_scope": {
         "limitations_section_present": {
           "applies": true,
           "answer": true,
           "justification": "Section 7 'Threats to Validity' provides dedicated discussion of internal (data leakage) and external (benchmark generalization) threats."
         },
         "threats_to_validity_specific": {
           "applies": true,
           "answer": true,
           "justification": "The threats are specific: internal threat identifies GPT-4o training data leakage on SWE-bench patches specifically, citing the SWE-bench authors' analysis. External threat identifies SWE-bench Lite as the sole evaluation dataset. Section 6.2 also specifically identifies that AGENTLESS underperforms on problems with no location clues."
         },
         "scope_boundaries_stated": {
           "applies": true,
           "answer": false,
           "justification": "Section 7 acknowledges results 'might not generalize to other datasets' but does not explicitly state what the results do NOT show. Does not bound claims to Python only, does not acknowledge the single-model (GPT-4o) limitation, does not state that results are for bug-fixing tasks specifically."
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "SWE-bench Lite is publicly available with all issues and ground truth patches. AGENTLESS is open-sourced, so generated patches can be reproduced and verified."
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "The benchmark data originates from SWE-bench (cited), whose collection methodology is documented in the original paper. For the manual classification in Section 6.1, the dimensions and categories are described in detail."
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants were recruited. The evaluation uses a standard public benchmark (SWE-bench Lite). The manual classification was performed by the authors themselves."
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline from SWE-bench Lite (300 problems) to SWE-bench Lite-S (249 problems) is documented with filtering criteria (exact patches, misleading solutions, insufficient information). The AGENTLESS pipeline from localization to patch selection is fully documented."
         }
       },
       "conflicts_of_interest": {
         "funding_disclosed": {
           "applies": true,
           "answer": false,
           "justification": "The acknowledgments section thanks two individuals and mentions a bike but discloses no funding source (grants, corporate sponsors, or funding agencies)."
         },
         "affiliations_disclosed": {
           "applies": true,
           "answer": true,
           "justification": "All four authors are listed with University of Illinois Urbana-Champaign affiliation. They evaluate third-party models (GPT-4o, Claude 3.5 Sonnet) and are not affiliated with OpenAI or Anthropic."
         },
         "funder_independent_of_outcome": {
           "applies": true,
           "answer": false,
           "justification": "No funding source is disclosed. The authors are at a major research university using substantial API credits, so funding likely exists but is not disclosed. Cannot confirm funder independence without knowing the funder."
         },
         "financial_interests_declared": {
           "applies": true,
           "answer": false,
           "justification": "No competing interests statement is present in the paper. The authors may have interests via the AGENTLESS GitHub project or its commercial adoption, but no disclosure is made."
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Section 7 discusses contamination risk but states 'GPT-4o is a closed-source model, we do not have access to the training data.' The specific training cutoff date for GPT-4o is not provided."
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 7 explicitly addresses this: 'One threat to validity comes from the data leakage of ground truth developer patches in SWE-bench Lite being part of the training data for GPT-4o.' Cites SWE-bench authors' temporal analysis as mitigating evidence."
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Section 7 discusses the contamination risk and cites the SWE-bench authors' finding of 'no significant difference' in resolve rates before/after GPT-4's knowledge cutoff. While incomplete for GPT-4o specifically, the concern is explicitly addressed."
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. This is a benchmark evaluation paper."
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved."
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved."
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved."
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved."
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved."
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved."
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports average cost per issue as $0.70. Per-component costs are broken down in Tables 2-4 (e.g., $0.02 for prompting-based file localization, $0.25 for reproduction tests)."
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Per-issue average cost is reported ($0.70) but the total computational budget across all 300 problems is not explicitly stated. No hardware specs, GPU hours, or wall-clock time are provided."
         }
       }
     },
     "comparison_summary": {
       "sonnet_generous_count": 1,
       "opus_generous_count": 0,
       "applies_boundary_count": 1,
       "interpretive_count": 0,
       "disagreement_categories": {
         "evaluation_design.human_evaluation": "sonnet_generous — Sonnet credits manual benchmark classification as human evaluation; schema explicitly excludes this.",
         "conflicts_of_interest.funder_independent_of_outcome": "applies_boundary — Sonnet sets applies=false (assuming unfunded); Opus sets applies=true (absence of disclosure is not confirmation of absence)."
       },
       "notes": "High agreement (48/50 = 96%). Two disagreements: (1) human_evaluation: Sonnet credits Section 6.1 manual classification of SWE-bench Lite problems as human evaluation despite the schema explicitly stating 'manual classification of the benchmark or dataset itself does not count.' Sonnet even notes in its own justification 'though not of system outputs directly' but still answers YES — a clear generosity error. (2) funder_independent_of_outcome: Sonnet sets applies=false reasoning the work is unfunded. However, the authors are at UIUC using substantial OpenAI API compute; unfunded status is not confirmed. The schema says 'NA if unfunded' but absence of disclosure is not evidence of absence. Both disagreements follow known Sonnet patterns: generosity bias (crediting partial evidence as YES) and NA boundary confusion (using applies=false to avoid a negative answer)."
     }
   }
 }