ai-research-survey

calibration.json (19472B)

---

 {
   "calibration": {
     "paper_slug": "agentic-programming-survey-2025",
     "sonnet_scan_date": "2026-02-28",
     "opus_calibration_date": "2026-02-28",
     "agreement_rate": 0.98,
     "total_questions": 50,
     "agreements": 49,
     "disagreements": 1,
     "disagreement_details": [
       {
         "category": "claims_and_evidence",
         "question": "generalization_bounded",
         "sonnet": {"applies": true, "answer": true},
         "opus": {"applies": true, "answer": false},
         "direction": "sonnet_generous",
         "explanation": "The paper states 'We focus primarily on LLM-driven agentic systems for software development' which is a topic statement, not a scope boundary on what the results do NOT show. The schema requires explicit statements about 'what was not tested, what populations/settings are excluded, what claims the authors are NOT making.' The paper provides no search date cutoff, does not state which adjacent areas were excluded, does not state whether non-English papers were excluded, and does not specify what databases were omitted. However, re-reading the schema description again: 'Are generalizations bounded to the tested setting?' — for a survey, the 'tested setting' is the reviewed corpus. The paper does say it focuses on LLM-driven agentic systems for software development and notes Python bias. This is borderline. After careful reconsideration, the paper's scoping statement combined with specific qualifications like noting Python bias in benchmarks (Section 6.3) and the explicit focus statement in the Introduction may constitute adequate bounding for a survey. However, the title 'AI Agentic Programming: A Survey' is quite broad and does not bound to the 152-paper corpus or any time period. Opus answer: false — the broad title and claims are not bounded to the specific corpus. Sonnet answer: true — crediting the focus statement and Python-bias qualification."
       }
     ],
     "opus_checklist": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No repository URL, code archive, or analysis scripts are provided anywhere in the paper. The survey could have released its search/screening tools, data extraction scripts, or classification code but did not."
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The corpus of 152 reviewed papers is not released as a structured dataset. No supplementary data files, appendix listing the full corpus, or data archive link is provided."
         },
         "environment_specified": {
           "applies": false,
           "answer": false,
           "justification": "This is a systematic literature review with no software experiments or computational pipeline requiring an environment specification."
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step instructions for reproducing the systematic review are provided. Section 3 describes the methodology at a high level but lacks sufficient detail (e.g., exact queries per database, access dates, disagreement resolution criteria) for independent replication."
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": false,
           "answer": false,
           "justification": "This is a qualitative survey paper with no primary statistical analysis. Confidence intervals are not applicable."
         },
         "significance_tests": {
           "applies": false,
           "answer": false,
           "justification": "No hypothesis tests are conducted. The paper is a literature survey with taxonomic classification and descriptive statistics only."
         },
         "effect_sizes_reported": {
           "applies": false,
           "answer": false,
           "justification": "No effect sizes are applicable. The paper does not conduct meta-analytic or comparative statistical analyses."
         },
         "sample_size_justified": {
           "applies": false,
           "answer": false,
           "justification": "No statistical sample requiring power analysis. The paper reviews papers using SLR methodology, not a statistical sampling design."
         },
         "variance_reported": {
           "applies": false,
           "answer": false,
           "justification": "No repeated experiments or measurements. Variance reporting is not applicable to this survey."
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": false,
           "justification": "The survey does not structurally compare itself against prior surveys on the topic (e.g., Liu et al. 2024 on LLM-based agents for SE, Hou et al. 2024 on LLMs for SE). While it mentions related surveys exist, no systematic comparison of coverage, methodology, or conclusions is provided."
         },
         "baselines_contemporary": {
           "applies": false,
           "answer": false,
           "justification": "No experimental baselines are used. This criterion applies to comparing against competitive baselines in experiments, which a survey does not have."
         },
         "ablation_study": {
           "applies": false,
           "answer": false,
           "justification": "No system or model is evaluated. Ablation studies are structurally inapplicable to a survey paper."
         },
         "multiple_metrics": {
           "applies": false,
           "answer": false,
           "justification": "The survey does not use evaluation metrics for its own analysis. It describes benchmarks and metrics used by others but does not apply any to its own work."
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No system outputs are produced that require human evaluation. The survey produces a taxonomy and synthesis, which are not evaluated by human raters."
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "No prediction or classification model is trained. Held-out test sets are structurally inapplicable."
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "The survey provides multiple per-category breakdowns: Table 5 compares systems across behavioral dimensions, Table 8 breaks down benchmarks by source/language/task/difficulty, Figure 6 shows temporal distribution of papers by year, and Figure 9 shows per-benchmark model performance."
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5 (Challenges) discusses failure modes and limitations of existing agentic systems, including benchmark inadequacies (5.1), communication protocol deficiencies (5.2), domain-specific weaknesses (5.3), and safety failures (5.4). Figure 9 shows where models fail on software optimization."
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 5 and 6.3 explicitly report negative findings: all models perform poorly on software optimization (GSO), benchmarks are biased toward Python, existing frameworks lack multi-turn evaluation support, and there is no standard taxonomy or evaluation methodology."
         }
       },
       "claims_and_evidence": {
         "abstract_claims_supported": {
           "applies": true,
           "answer": true,
           "justification": "The abstract claims the paper provides 'a taxonomy of agent behaviors and system architectures' (delivered in Section 4), 'examining relevant techniques' (Section 2), 'challenges' (Section 5), and 'opportunities' (Section 6). All abstract claims are descriptive and appropriately supported."
         },
         "causal_claims_justified": {
           "applies": false,
           "answer": false,
           "justification": "The survey makes no causal empirical claims from its own evidence. It describes, categorizes, and synthesizes existing work without asserting causal relationships."
         },
         "generalization_bounded": {
           "applies": true,
           "answer": false,
           "justification": "The paper states 'We focus primarily on LLM-driven agentic systems for software development' but the title 'AI Agentic Programming: A Survey of Techniques, Challenges, and Opportunities' is quite broad. No search date cutoff is stated, no explicit boundaries on what was excluded (non-English papers, specific databases, adjacent domains), and no statement of what the survey's results do NOT show. The scoping is a topic statement rather than explicit scope boundaries per the schema's requirement."
         },
         "alternative_explanations_discussed": {
           "applies": false,
           "answer": false,
           "justification": "This is a pure survey/taxonomy paper with no empirical results of its own. The schema states 'NA only for papers that present no empirical results (e.g., pure surveys or taxonomies).'"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": false,
           "answer": false,
           "justification": "The survey does not deploy or use any LLM. It reviews papers that use LLMs but does not run any models itself."
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "The survey does not use any prompting. It is a human-conducted literature review."
         },
         "hyperparameters_reported": {
           "applies": false,
           "answer": false,
           "justification": "No models or algorithms are run by the survey authors. Hyperparameter reporting is not applicable."
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used by the survey authors. This is a human-conducted literature review."
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "Section 3 describes the filtering pipeline with counts (7,700 -> 395 -> 141 -> 152) and lists inclusion/exclusion criteria (Section 3.2). However, per the schema: 'describing the filtering pipeline stages with counts is YES only if the actual filtering CRITERIA at each stage are also stated.' The criteria are listed globally but not differentiated per stage. The criteria for citation chaining additions (adding 11 papers from 141 to 152) are not specified. Disagreement resolution is described only as 'through discussion' without operationalized rules."
         }
       },
       "limitations_and_scope": {
         "limitations_section_present": {
           "applies": true,
           "answer": false,
           "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section about the survey itself. Section 5 (Challenges) discusses limitations of the field/technology, not the survey's methodology. Section 7 (Conclusion) does not include substantive limitations discussion."
         },
         "threats_to_validity_specific": {
           "applies": true,
           "answer": false,
           "justification": "No threats-to-validity discussion exists for the survey. There is no mention of potential biases in paper selection, publication bias, search string limitations, database coverage gaps, or limitations of the inclusion/exclusion criteria."
         },
         "scope_boundaries_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper says 'We focus primarily on LLM-driven agentic systems for software development' but does not explicitly state what it does NOT cover: no search date range, no list of excluded adjacent areas, no statement about non-English papers, and no explicit negative scope boundaries. The schema requires 'explicit statements about what was not tested... what claims the authors are NOT making.'"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The list of 152 reviewed papers is not released as a structured dataset. No appendix or supplementary material lists the full corpus. The bibliography covers cited papers, which likely includes the corpus but is not identified as such."
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3 describes the search strategy: databases searched (Google Scholar, ACM DL, IEEE Xplore, SpringerLink, arXiv), the Boolean search string with three term clusters, and the three-stage selection process with counts at each stage."
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. This is a literature review where the 'data' is published papers. The criterion about recruitment methods applies to human subjects research."
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "Pipeline counts (7,700 -> 395 -> 141 -> 152) are given in Section 3.3, but: (1) criteria at each stage are not differentiated, (2) the increase from 141 to 152 via citation chaining lacks explanation of selection criteria for those 11 papers, (3) inter-rater reliability for the two-researcher screening is not reported, and (4) disagreement resolution is described only as 'through discussion.'"
         }
       },
       "conflicts_of_interest": {
         "funding_disclosed": {
           "applies": true,
           "answer": false,
           "justification": "No acknowledgments section or funding disclosure is present. There is no mention of grants, institutional support, or funding agencies anywhere in the paper."
         },
         "affiliations_disclosed": {
           "applies": true,
           "answer": true,
           "justification": "All five authors list their affiliation as University of Leeds (UK) on the title page with individual email addresses. Affiliations are clearly disclosed."
         },
         "funder_independent_of_outcome": {
           "applies": false,
           "answer": false,
           "justification": "No funding is disclosed, so funder independence cannot be assessed. The schema says 'NA if unfunded.' Without a funding disclosure, there is no funder to evaluate for independence."
         },
         "financial_interests_declared": {
           "applies": true,
           "answer": false,
           "justification": "No competing interests statement is present. There is no declaration that authors hold (or do not hold) patents, equity, or financial interests related to the tools and systems reviewed."
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "This is a survey paper. The authors do not train or evaluate any pre-trained models on benchmarks. Contamination is structurally inapplicable."
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Survey paper with no model evaluation. Train/test overlap is not applicable."
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Survey paper with no model evaluation by the authors. Contamination is not applicable to the survey itself."
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. This is a literature review of published papers."
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. IRB approval is not applicable to a systematic literature review."
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this survey."
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The inclusion/exclusion criteria in Section 3.2 refer to papers, not human subjects."
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. This is not an experimental study."
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Blinding is not applicable."
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Attrition is not applicable."
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": false,
           "answer": false,
           "justification": "This is a survey paper. The authors do not run any LLM inference. The schema states 'NA if cost is clearly irrelevant (e.g., theoretical paper, survey paper).' The paper reports costs of systems it reviews (Table 6) but not the cost of the survey's own method."
         },
         "compute_budget_stated": {
           "applies": false,
           "answer": false,
           "justification": "This is a survey paper with no computational experiments. No compute budget is required or applicable."
         }
       }
     },
     "summary": "Very high agreement (98%, 49/50 questions) between Sonnet and Opus on this survey paper. Both raters agreed on the applies field for all 50 questions. The single disagreement is on generalization_bounded: Sonnet credited the paper's focus statement and Python-bias qualification as adequate scope bounding, while Opus found the broad title and absence of explicit negative scope boundaries (no search date range, no list of excluded areas) insufficient. This is a borderline interpretive disagreement. The paper is a well-structured systematic literature review that follows standard SLR methodology but lacks a limitations section about the survey itself, does not release its corpus data, and has no funding or competing interests disclosures.",
     "category_breakdown": {
       "artifacts": {"agreements": 4, "disagreements": 0},
       "statistical_methodology": {"agreements": 5, "disagreements": 0},
       "evaluation_design": {"agreements": 9, "disagreements": 0},
       "claims_and_evidence": {"agreements": 3, "disagreements": 1},
       "setup_transparency": {"agreements": 5, "disagreements": 0},
       "limitations_and_scope": {"agreements": 3, "disagreements": 0},
       "data_integrity": {"agreements": 4, "disagreements": 0},
       "conflicts_of_interest": {"agreements": 4, "disagreements": 0},
       "contamination": {"agreements": 3, "disagreements": 0},
       "human_studies": {"agreements": 7, "disagreements": 0},
       "cost_and_practicality": {"agreements": 2, "disagreements": 0}
     }
   }
 }