ai-research-survey

calibration.json (17407B)

---

 {
   "paper_slug": "adoption-generative-artificial-2026",
   "calibration_date": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "artifacts",
       "question": "environment_specified",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet marked applies=false, reasoning that a survey/interview study has no software environment to specify. Opus marks applies=true because the study performed computational analyses (k-means clustering, chi-squared tests, Spearman correlations) that require statistical software, and per the scan-agent instructions for survey papers, all artifacts questions should have applies=true. A survey study CAN provide environment specifications for its analysis code. Since they did not, the correct answer is applies=true, answer=false."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or repository URL is provided anywhere in the paper. No analysis scripts or survey instruments are released as downloadable artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The survey data (109 responses from LimeSurvey) and 18 interview transcripts are not publicly released. No data repository link is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The study performed computational analyses (k-means clustering, chi-squared tests, Spearman correlations) but does not specify the statistical software, package versions, or environment used. A survey study with quantitative analysis can and should provide environment specifications for reproducibility."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a high level (LimeSurvey, semi-structured interviews, open coding), but no replication package, protocol document, or detailed procedures are linked."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports means on 5-point scales (e.g., Mean=4.0, Mean=3.9, Mean=3.4) and percentages without any confidence intervals, error bars, or uncertainty estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Chi-squared tests are reported with full statistics: chi2(2, N=109)=10.84, p_adj=0.022 for experience-prompting; chi2(5, N=109)=13.91, p_adj=0.016 and chi2(4, N=109)=11.30, p_adj=0.023 for cluster demographics. Spearman correlations with p-values are reported throughout Section 5."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Spearman correlation coefficients (rho values such as rho=0.33, rho=0.39, rho=-0.33, rho=0.48) are reported throughout Section 5, providing effect sizes. Chi-squared statistics with N are provided. Percentage breakdowns with counts give practical magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of n=109 is not justified by power analysis or formal reasoning. No explanation is given for why 109 is sufficient for the subgroup analyses performed, particularly with small strata (e.g., n=22 junior developers)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports means on 5-point scales without standard deviations or any spread measures. Percentage distributions are shown in figures but no variance, standard deviation, or IQR is provided for the reported mean values."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 ('Global Context') explicitly compares results to the 2025 Stack Overflow Developer Survey, providing an external reference point for the German sample's adoption and challenge patterns."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The Stack Overflow Developer Survey 2025 used for comparison is contemporary with the study's 2025 data collection period."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study, not a system with components to ablate. Ablation is structurally inapplicable."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study uses multiple measurement approaches across six survey blocks: adoption rates, usage frequency across 7 tasks, perceived effectiveness of 10 prompting strategies, severity of 14 challenges, and perceived impact on 5 workflow dimensions."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is itself a human-subjects survey study. There is no system producing outputs that require human evaluation. Human evaluation of system outputs is structurally inapplicable."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not an ML or benchmark evaluation. The concept of held-out test sets does not apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: usage frequency by task (Figure 3), tool adoption by tool type (Figure 2), prompting effectiveness by experience level (Figure 8), challenge severity by challenge (Figure 5), code generation frequency by company size (Figure 10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple failure modes: hallucinations (Section 3.4), context wall limitations (Section 5.4), cases where prompting 'ends up taking longer than writing code' (Section 3.3), and the verification tax imposed by unreliable outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative findings reported: bug fixing improved for only 40% (39% no change), role prompting showed poor effectiveness (Mean=2.9), pre-made prompts rated least effective (Mean=2.8), senior developers skeptical of AI tools (only 39% finding specific instructions effective), and distrust correlating with reduced workflow speed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'experience level moderates perceived benefits' (chi-squared tests, Section 5.1), 'organizational size affects tool selection and intensity' (Figure 10, Section 5.3), 'limited awareness of project context is most significant barrier' (Figure 6, Section 5.4)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language: 'experience seems to have an impact on developers' interaction' (Section 5.1), 'moderates the perceived effectiveness' (abstract). These causal interpretations are drawn from cross-sectional survey data without randomization or causal identification strategy. Section 5.4 disclaims causality for one correlation, but causal language is used elsewhere."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Findings are consistently scoped to 'German software engineers' in title, abstract, and conclusions. Section 7 acknowledges convenience sample limitations and that 'findings were sampled in 2025 and may not generalize to future states.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 explicitly lists four alternative explanations for the Proficiency Cycle: lower baseline performance, task type differences, self-reinforcing proficiency cycle, and response bias. Section 5.4 states the distrust-speed correlation 'is consistent with two interpretations.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study that does not use LLMs as a methodological tool. The LLMs discussed are tools used by surveyed practitioners, not experimental variables controlled by the researchers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This study does not use LLM prompting in its methodology. It asks human participants about their prompting practices but does not itself prompt any LLMs."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLMs are used in the study's methodology. The schema description focuses on LLM hyperparameters (temperature, top-p, etc.). The k-means k=2 is stated but LLM hyperparameters are not applicable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in the study's methodology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 documents filtering: 210 initial responses, 101 excluded for being incomplete (most with no questions answered), leaving n=109 complete submissions. Section 2.1 describes the interview open coding process with two independent coders resolving discrepancies through discussion."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' is a dedicated section discussing internal validity, external validity, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific threats: fixed thematically grouped question order may cause carryover effects, convenience sampling via personal contacts and LinkedIn may not represent the broader population, self-reported frequency/effectiveness introduces measurement error. These are specific to this study's design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states findings may not generalize beyond Germany and 2025 ('findings were sampled in 2025 and may not generalize to future states'). The conclusion explicitly notes the cross-sectional design prevents causal inference, directing future work to longitudinal designs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw survey response data and interview transcripts are not publicly available. No data repository link is provided. Only summary statistics, figures, and correlation values are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the data collection procedure: LimeSurvey platform, distribution via personal contacts and LinkedIn, collection period April 15 to August 20, 2025, voluntary participation with no incentives, anonymous data collection, and survey structure with six blocks."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 states participants were recruited through 'personal contacts and on social media platforms like LinkedIn,' explicitly noting convenience sampling. Section 7 acknowledges this as a threat to external validity."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: 210 raw responses -> 101 excluded (incomplete, most with no questions answered) -> 109 complete submissions. Interview open coding process described: two researchers independently coded, then met to resolve discrepancies through discussion (Section 2.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, acknowledgments section, or mention of grants or sponsors appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the title page: four authors at Technical University of Munich (Heilbronn campus) and one at Heilbronn University of Applied Science. No commercial affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed. Appears to be unfunded academic research from two universities. The schema says NA if unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study that does not evaluate a pre-trained model's capability on any benchmark. Contamination concerns are structurally inapplicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey/interview study with no benchmark evaluation. Train/test overlap is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey/interview study with no benchmark evaluation. Benchmark contamination is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration link (OSF, AsPredicted, or similar) is mentioned. The study was not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite collecting data from 18 interview participants and 109 survey respondents."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 reports detailed demographics: role distribution (62% developers, 13% team leads/managers, 8% architects), experience (mean 12.1 years, median 10 years, 26% senior, 54% mid-level, 20% junior), company size breakdown, education levels (51% Master's, 22% Bachelor's, 14% Doctorate), and geographic distribution (88% Germany)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No formal inclusion/exclusion criteria are stated for survey or interview participation. Participants are described as 'software engineers working mainly in Germany' but no screening process or explicit eligibility criteria are described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey study, not an experimental study with condition assignment. Randomization is structurally inapplicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experiment with treatment/control conditions. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 reports: 210 initial responses recorded, 101 excluded (most with no questions answered at all), leaving n=109 complete submissions. The attrition and primary reason are stated."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study with no LLM inference performed as part of the methodology. Cost is structurally inapplicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study requiring no significant computational budget. The schema says NA for survey papers."
       }
     }
   }
 }