ai-research-survey

calibration.json (15700B)

---

 {
   "paper_slug": "advancing-software-quality-2025",
   "calibration_date": "2026-02-28",
   "model": "opus",
   "total_questions": 50,
   "agreement_count": 48,
   "disagreement_count": 2,
   "agreement_rate": 0.96,
   "disagreements": [
     {
       "question": "evaluation_design.baselines_contemporary",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false reasoning that since no baseline comparisons were made, contemporariness is not applicable. Opus considers that since baselines_included has applies=true (prior surveys exist and should be compared), baselines_contemporary also applies — the question is whether any baselines used are contemporary, and the paper could have included contemporary survey comparisons but did not. The paper simply fails to compare against any prior surveys at all, making applies=true, answer=false the correct assessment."
     },
     {
       "question": "conflicts_of_interest.funder_independent_of_outcome",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false because no funding source is disclosed, interpreting this as 'unfunded.' Opus considers that the author is affiliated with Juniper Networks Inc., which could be the implicit funder (work done on company time). The schema says NA only 'if unfunded,' but absence of a funding disclosure is not evidence of being unfunded — it is a failure to disclose. Since the author has a corporate affiliation, the criterion applies, and the lack of any independence statement means answer=false."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL, GitHub link, or Zenodo archive is provided anywhere in the paper. The survey could have released analysis scripts, search query logs, or paper classification data, but nothing is made available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The corpus of 223+ reviewed papers is not released as a structured dataset. No supplementary data file, spreadsheet, or downloadable list of included papers is provided."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure literature survey with no computational experiments, so there is no software environment to specify."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. A reader cannot reproduce the literature search or paper selection process because search databases, queries, and filtering pipeline are not described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive literature survey reporting raw counts and percentages of papers (Figures 1-6). No statistical aggregation requiring confidence intervals is performed."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative statistical claims are made. The paper reports descriptive counts and proportions from the surveyed corpus without testing differences."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive survey with no meta-analytic aggregation of effect sizes from reviewed papers."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or statistical analyses requiring sample size justification are conducted. This is a literature survey."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted. The survey reports descriptive statistics from the literature corpus."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare itself against any prior surveys in the LLM-SQA space. No prior reviews are referenced as baselines to demonstrate the additive value of this survey."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Since baselines_included applies (prior surveys in LLM-SQA exist and could be compared against), the contemporariness of those baselines also applies. However, since the paper includes no baseline comparisons at all, it cannot be assessed whether baselines are contemporary. Answer is false because no baselines are used."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The survey characterizes the literature across multiple dimensions: publication trends (Figure 1), dataset utilization (Figure 2), evaluation approaches (Figure 3), fine-tuning adoption (Figure 4), LLM usage distribution (Figure 5), and prompting strategies (Figure 6)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of the survey's own outputs is not clearly relevant for this type of literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; no train/test split is applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides a detailed per-standard, per-application breakdown mapping LLM applications to ISO/IEC 12207, 25010, 5055, ISO 9001, CMMI, and TMM. Figures 1-6 also break down by multiple categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (Challenges, Limitations, and Risks) discusses failure modes of LLM-based SQA including data privacy exposure, model bias, black-box explainability, and resource constraints."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V.B reports that 'nearly 30% of the reviewed papers did not specify any dataset,' Section V.E reports 'nearly 19% of papers did not specify which LLM was used,' and Section VI enumerates challenges and failure patterns across the surveyed literature."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract promises a survey of LLM-SQA intersection with standards mapping, case studies, challenges, and future directions. The paper delivers on these with Section V (literature characterization), Table I (standards mapping), Section VI (challenges), and Section VII (future directions)."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that summarizes existing work. It uses hedged possibility language ('can enhance,' 'can augment') when describing LLM capabilities. It makes no original causal claims requiring causal inference study design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's abstract claims to 'provide a comprehensive blueprint for integrating LLMs into SQA in a trustworthy, efficient, and standards-aligned manner' without adequately bounding generalizations to the specific 2023-2025 papers reviewed. The conclusions generalize broadly despite the surveyed literature itself having significant gaps (30% no dataset, 19% no LLM specified)."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive survey that does not present original empirical results requiring alternative explanations. The schema specifies NA for 'pure surveys or taxonomies.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments with LLMs."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No LLM prompting is used in this survey paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM experiments are conducted in this survey."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; this is a literature survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section IV describes 7 high-level inclusion criteria but does not document the actual search process: no databases queried, no search strings, no initial pool size, no filtering stages with counts. 'Over 223 papers' are mentioned with no explanation of how the starting pool was identified or reduced. This fails the schema requirement for filtering CRITERIA at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section VI is titled 'Challenges, Limitations, and Risks' but discusses limitations of LLM technology in SQA generally, not limitations of THIS survey's own methodology. There is no dedicated discussion of the survey's methodological shortcomings (e.g., single reviewer, informal search, no quality assessment of included papers)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity specific to this survey are discussed. Missing: selection bias from informal search, lack of systematic review protocol, single-author classification without inter-rater reliability, possible incomplete literature coverage."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not explicitly state what it does NOT cover. No mention of excluded languages, excluded databases, excluded gray literature, or which LLM application areas are out of scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The full list of 223+ reviewed papers is not provided as a structured, verifiable dataset. Readers cannot independently verify which papers were classified into which categories."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section IV lists inclusion criteria but omits the data collection procedure: no search databases named (e.g., IEEE Xplore, ACM DL, Scopus), no search queries provided, no time window for the search itself stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. This is a literature survey using published papers as data sources, not a study with human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA-style flowchart or equivalent. The pipeline from initial paper discovery to the final 223+ papers is undocumented. No counts of papers retrieved, screened, or excluded at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section, funding disclosure, or grant information is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation with Juniper Networks Inc. is clearly disclosed on the title page alongside an ORCID identifier."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The author is affiliated with Juniper Networks Inc. but no funding source is disclosed. Absence of a funding disclosure is not evidence of being unfunded — the work could have been done on company time. Since we cannot verify funder independence, answer is false."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature survey that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained models is conducted."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained models is conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved; this is a literature survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no original LLM experiments."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no original computational experiments."
       }
     }
   }
 }