ai-research-survey

calibration.json (15495B)

---

 {
   "paper_slug": "agent-developer-practices-2025",
   "calibration_date": "2026-02-28",
   "model": "opus",
   "total_questions": 50,
   "agreement_count": 50,
   "disagreement_count": 0,
   "agreement_rate": 1.0,
   "disagreements": [],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, Zenodo archive, or code release link is provided anywhere in the paper. The paper performs automated analysis using GPT-4o and GPT-4 but does not share scripts or code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The study collects 1,575 GitHub repositories and 20,620 developer discussions but provides no download link or public dataset. The raw discussion corpus and cleaned datasets are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or dependency list is provided. The paper only mentions using GPT-4o and GPT-4 without specifying any library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, reproduction script, or step-by-step guide is provided. The methodology is described in natural language in Section 3 but not in a form that would allow direct replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '25.6%', '14%', '25%', '23%'). No confidence intervals, error bars, or uncertainty estimates are provided for any finding."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims across frameworks (e.g., AutoGen vs. LangChain on learning cost, development efficiency) but performs no statistical significance tests. All comparisons are frequency counts."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only proportions and frequency counts are reported. No Cohen's d, odds ratios, relative risk, or formal effect size measures are provided for any comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes (1,575 repos, 8,710 discussions, 11,910 discussions) are described by filtering criteria but never justified relative to statistical power or adequacy for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported for any result. All findings are single aggregate proportions with no measure of uncertainty."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No comparison against prior empirical studies of agent frameworks. The paper claims to be 'the first' such study but does not compare findings with related work (e.g., Cemri et al., Yehudai et al.) quantitatively."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines are included, so the contemporariness of baselines cannot be assessed. The criterion is moot without baselines to evaluate."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining/qualitative study, not a system with components to ablate. No experimental interventions are performed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates frameworks across five dimensions (learning cost, development efficiency, functional abstraction, performance optimization, maintainability) and uses multiple popularity indicators (stars, forks, used-by, contributors, repo counts)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two domain experts independently reviewed all functional role annotations (Section 4.1.2, Cohen's kappa = 0.82) and a two-researcher open coding procedure was used in Section 6.1 (kappa = 0.81). These constitute human evaluation of the LLM-generated classifications."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study mining GitHub discussions. There is no predictive model and no train/test split. The concept of a held-out test set does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per framework for all five evaluation dimensions (Sections 6.2.1-6.2.5) and by challenge category (Logic, Tool, Performance, Version) across SDLC stages (Section 5.2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 5) is entirely dedicated to analyzing failure cases: infinite loops (Figure 7), memory management failures (Figure 8), version incompatibilities (Figure 9), with detailed examples and percentages."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that performance optimization is 'a common shortcoming across all frameworks' (Finding 12) and that AutoGen and LangChain face 'the highest maintenance complexity' (Finding 13)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (ten frameworks identified, taxonomy of four challenge categories, five-dimensional evaluation framework) are all substantiated in the results sections. Specific percentages cited in the abstract match the body."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language such as 'LangChain and CrewAI lower the technical threshold' and 'AutoGen and LangChain excel at rapid prototyping' derived from observational discussion data with no controlled comparison or causal identification strategy."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and title present conclusions about 'agent developers' generally (e.g., 'more than 80% of developers report difficulties') without consistently scoping to GitHub open-source context. Section 7.3 acknowledges this limitation but the main claims are unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 discusses specific alternative explanations: LLM-generated bias in classification, topic tags not reflecting actual usage, and GitHub sample bias excluding closed-source environments."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4o' and 'GPT-4' without specifying API version strings or snapshot dates. These are marketing names; model behavior varies across versions and snapshots."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides the actual system and user prompts used for discussion summarization verbatim, including the full prompt text with the placeholder for batch_text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 states 'we standardize the generation process by setting the temperature to 0.1 and using top-1 sampling.' Temperature and sampling strategy are explicitly reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses LLMs as classification tools in a pipeline, not as autonomous agents with agentic scaffolding. No scaffolding applies."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data collection and cleaning pipeline with explicit filtering criteria (compound keywords, non-empty README, stars>10, forks>5, recency) and counts at each stage (10,265 raw -> 8,710 cleaned)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 'Threats to Validity' provides a dedicated limitations discussion covering internal validity (LLM bias, topic tag accuracy) and external validity (GitHub-only sample)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: LLM-generated bias from using GPT-4o for summarization, topic tags possibly not reflecting actual framework usage, exclusion of closed-source enterprise projects."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 explicitly states 'Our sample covers only GitHub open-source projects, excluding closed-source projects, which may limit the generalizability of our conclusions.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw GitHub discussion corpus (8,710 and 11,910 discussions) is not made available for independent verification. No supplementary data files, dataset DOIs, or download links are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: GitHub API search with compound keyword strategy, filtering criteria, time period (2022-July 2025), and metadata collected (stars, forks, used-by, contributors, topics, discussions)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a repository mining study with no human participants. GitHub projects and discussions are the data source. The sampling criteria are described in Section 3 but this is not human recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the full pipeline from raw collection to analysis for both phases (RQ1/RQ2 and RQ3), with explicit counts at each step (10,265 raw -> duplicates removed -> invalid filtered -> 8,710 cleaned)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section discloses funding from the State Key Laboratory of Blockchain and Data Security at Zhejiang University, NSFC grants (62332004, 62276279), and Guangdong Basic and Applied Basic Research Foundation (2024B1515020032)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are disclosed: Sun Yat-sen University (4 authors), Zhejiang University (1), University of Melbourne (1), Technical University of Munich (1). No authors are affiliated with the evaluated frameworks."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (NSFC, Guangdong research foundation, State Key Lab) are government and academic bodies with no financial stake in any of the ten frameworks evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement, patent disclosure, or declaration of financial interests anywhere in the paper. Absence of a disclosure statement means the criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining/observational study. GPT-4o/GPT-4 are used as classifier/summarizer tools, not evaluated for their capability on a benchmark. Contamination criteria do not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained model capabilities is performed. The LLMs are tools for text classification, not the subject of evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No capability benchmark is used. The study evaluates developer discussions about agent frameworks, not pre-trained model performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "Repository mining study with no human participants. Mining GitHub discussions does not constitute a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes publicly available GitHub data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study examines GitHub repositories and discussion threads."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Repository selection criteria are described in Section 3 but these are data filtering criteria, not participant eligibility."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental assignment. This is an observational study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no longitudinal tracking. Attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o and GPT-4 to summarize and classify 20,620 discussion threads but reports no API costs, token counts, or cost per example."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, API spend, wall-clock time, or hardware specifications are provided for the LLM-based analysis pipeline."
       }
     }
   }
 }