ai-research-survey

calibration.json (17310B)

---

 {
   "paper_slug": "adaptive-data-augmentation-2026",
   "total_questions": 50,
   "agreement_count": 48,
   "disagreement_count": 2,
   "agreement_rate": 0.96,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "Tables 2 and 3 report percentage improvements with full baseline context (e.g., OpenAI-text-embedding-3-small achieves 39.21% → 58.15% with ADAMAB, stated as '+18.94%'). The schema description explicitly says: 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper meets this standard by providing both the absolute baseline and final accuracy alongside the improvement delta.",
       "sonnet_justification": "While absolute accuracy improvements are shown (e.g., '+18.94%' over base embedding), no standardized effect sizes (Cohen's d, odds ratios) are reported. The improvement deltas are shown but without denominator context needed for a proper effect size measure.",
       "direction": "sonnet_generous"
     },
     {
       "category": "claims_and_evidence",
       "question": "causal_claims_justified",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The paper's causal claims are primarily via ablation: comparing calibration with ADAMAB vs. random augmentation vs. no augmentation, and varying the exploration hyperparameter alpha. These are controlled single-variable manipulations. The schema states: 'ablation studies... ARE causal claims — check whether the ablation design is adequate (controlled single-variable manipulation counts as YES).' The ablation design holds all other variables constant (same embedding model, same dataset, same calibrator architecture, same total augmentation budget) while varying only the augmentation strategy, which is adequate.",
       "sonnet_justification": "The paper makes causal claims ('ADAMAB diminishes the gradient shifting,' 'adaptive augmentation leads to faster convergence') and ablation comparisons ('calibration with ADAMAB outperforms random augmentation'). Ablation studies manipulate only one variable at a time, but there is no discussion of confounds such as the effect of the specific generative models used or the sensitivity to hyperparameter choices made by grid search.",
       "direction": "sonnet_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, code repository URL, or archive is provided anywhere in the paper or appendices. The ADAMAB framework is described algorithmically but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All six datasets (MultiWD, FQS, TREC, OxfordPets, Flowers102, CUB200) are publicly available standard benchmarks with cited original sources. The paper uses these without modification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B states experiments ran on 'a single MacBook Pro with a single M4 Max chip and 36GB memory' and mentions Adam optimizer, but no requirements.txt, Dockerfile, or library version list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. Appendices describe hyperparameters and prompt templates but lack executable reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 report only point estimates (accuracy percentages) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ADAMAB outperforms baselines and random augmentation based solely on comparing numeric accuracy values with no statistical significance tests (no p-values, t-tests, bootstrap tests, etc.)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 3 report percentage improvements with baseline context (e.g., OpenAI-text-embedding-3-small at 39.21% → ADAMAB at 58.15%, '+18.94%'). The schema states 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' This standard is met."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 2-5 initial samples per class without any power analysis or justification for why these specific numbers were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, IQR, or any spread measures are reported. All results appear to be single-run point estimates with no indication of multiple runs or seed variation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive baselines include decoder LLMs (GPT-4o-mini, Gemini2.0-Flash-Lite, Mistral-Small, Llama-3.2), rerankers (Cohere-v3.5, BGE-reranker-v2-m3, Jina-reranker-m0), embedding models, and calibration with/without random augmentation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o-mini, Gemini2.0-Flash-Lite, QWen-3-emb-06b, and Voyage-multimodal-3, which are contemporary models as of early 2026."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides ablation studies varying the number of augmentation rounds (Fig. 3) and the exploration hyperparameter alpha (Fig. 4), systematically measuring each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only classification accuracy is reported as the evaluation metric. No F1, precision, recall, AUC, or other metrics are used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates classification accuracy on automated benchmarks. Human evaluation of system outputs is not relevant to the claims about classification accuracy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows separate init (training) and test data counts for each dataset. Results in Tables 2 and 3 are on these held-out test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are aggregate accuracy per dataset with no per-class or per-category breakdown, despite datasets having 6 to 200 classes."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure case analysis or error breakdown is provided. Observations about few-shot underperforming zero-shot and synthetic data homogeneity are ablation findings, not systematic failure analysis of the classification outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows accuracy declining when too many synthetic samples are generated due to data homogeneity. The paper also notes few-shot in-context learning can be worse than zero-shot in some cases."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 40% accuracy improvement when training with less than 5 initial data samples of each class,' which is supported by Tables 2 and 3 (e.g., Flowers102 with Voyage: +43.20%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are made through controlled ablation studies: ADAMAB vs. random augmentation vs. no augmentation with all other variables held constant (same embedding model, dataset, calibrator architecture, augmentation budget). The schema states 'controlled single-variable manipulation counts as YES' for ablation-based causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims ADAMAB is 'highly suitable for resource-constrained environments' and establishes 'a new paradigm' without bounding results to the specific 6 datasets, 4 embedding models, and 2 generative models tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results, confounding factors, or threats to validity. The paper does not consider whether improvements stem from the specific datasets, generative models, or other factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o-mini,' 'Gemini2.0-Flash-Lite,' 'GPT-Image-1-mini,' and 'OpenAI-text-embedding-3-small' without snapshot dates or API version identifiers. Per the schema, marketing names without a snapshot date or API version do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B.2 provides prompt templates with placeholders (e.g., '{label1}: {description1}', '{existing_queries}', '{num_generate}'). The schema explicitly states prompt templates with placeholders do NOT count unless the actual fill values are also provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 and Table 4 report Adam optimizer, learning rate (0.005), weight decay (0.0001), cosine annealing scheduler, batch sizes, training rounds, augmentation rounds, delta_n, and alpha per dataset."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method trains lightweight calibrators via gradient descent without agent loops, tool use, or multi-step reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 states initial training and test counts, but does not describe how the initial subset was selected from full datasets (random? stratified? fixed seed?). For TREC, they mention filtering to 30 classes with >5 samples but the broader preprocessing is undocumented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. Section 4.3 notes synthetic data homogeneity as an ablation finding but this is not a systematic limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity discussion exists. Specific threats such as benchmark contamination, hyperparameter tuning on test data, or dependence on specific generative model quality are not discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion makes broad claims about ADAMAB being 'highly suitable for resource-constrained environments' and establishing 'a new paradigm' without explicitly stating what was NOT tested or claimed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All six datasets are publicly available standard benchmarks (MultiWD on Authorea, FQS from ACM CCS 2024, TREC from ACL, OxfordPets/Flowers102/CUB200 from standard CV repositories). Raw data can be independently obtained."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes each dataset with its source, number of classes, and domain. These are well-documented external benchmarks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The paper uses existing public benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states how many initial samples per class were used but does not describe how these specific samples were selected from the full datasets, nor the random seed or selection procedure."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present. Five of six authors are from Accenture but no funding information is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Minxue Tang (Duke University) and five co-authors from 'Center for Advanced AI, Accenture.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Five of six authors are from Accenture, a commercial consulting firm that would benefit from demonstrating effective AI calibration methods. The employer/funder is not independent of the research outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The Accenture affiliation of five authors represents a potential commercial interest that is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o-mini, Gemini2.0-Flash-Lite, and other models as baselines evaluated on benchmarks, but does not state training data cutoff dates for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark datasets (some dating to 2001-2012) appear in the training data of the evaluated LLMs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Datasets like TREC (2001), OxfordPets (2012), Flowers102 (2008), CUB200 (2011) were published well before training cutoffs of GPT-4o-mini and Gemini. The contamination risk is not acknowledged."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; computational benchmark evaluations only."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; no IRB approval needed."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; computational study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; no blinding applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o-mini for text generation and GPT-Image-1-mini for image generation across multiple training epochs without reporting API costs, tokens consumed, or cost per example."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B mentions the hardware ('single MacBook Pro with M4 Max chip and 36GB memory') but no wall-clock training times, total API costs, or total compute budget are reported."
       }
     }
   }
 }