ai-research-survey

scan.json (26842B)

---

 {
   "paper": {
     "title": "Adaptive Data Augmentation with Multi-armed Bandit: Sample-Efficient Embedding Calibration for Implicit Pattern Recognition",
     "authors": [
       "Minxue Tang",
       "Yangyang Yu",
       "Aolin Ding",
       "Maziyar Baran Pouyan",
       "Taha Belkhouja",
       "Yujia Bao"
     ],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2602.19385"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link or code repository is provided anywhere in the paper or appendix. The paper describes a framework (ADAMAB) with non-trivial implementation details but releases no code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All six datasets used (MultiWD, FQS, TREC, OxfordPets, Flowers102, CUB200) are publicly available benchmarks with cited sources and dataset links (e.g., CUB200 at data.caltech.edu). The paper uses these standard public benchmarks without modification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B states experiments ran on 'a single MacBook Pro with a single M4 Max chip and 36GB memory' and mentions Adam optimizer, but no requirements.txt, Dockerfile, or library version list is provided. The specific Python or library versions are not stated."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README are provided. The appendices describe training hyperparameters and prompt templates but do not give commands or scripts to reproduce the results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 report only point estimates (accuracy percentages) with no confidence intervals, error bars, or uncertainty measures. All results are single point values."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (ADAMAB outperforms random augmentation and baselines) with no statistical significance tests (no p-values, t-tests, or bootstrap tests). Conclusions are drawn from direct numeric comparisons only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 3 report percentage improvements with full baseline context (e.g., OpenAI-text-embedding-3-small achieves 39.21% → 58.15% with ADAMAB, stated as '+18.94%'). The schema description explicitly says: 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper meets this standard by providing both the absolute baseline and final accuracy alongside the improvement delta."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 5 initial samples per class for text tasks and 2-3 per class for image tasks, but there is no power analysis or justification for why these specific numbers were chosen. The very small sample sizes for the claims being made are not discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, IQRs, or any spread measures are reported. All results are single-run point estimates. It is unclear whether experiments were repeated with multiple seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares ADAMAB against multiple baselines including GPT-4o-mini, Gemini2.0-Flash-Lite, Mistral-Small, Llama-3.2 (decoder LLMs), Cohere-v3.5, BGE-reranker-v2-m3, Jina-reranker-m0 (rerankers), and embedding models, plus calibration without augmentation and with random augmentation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o-mini, Gemini2.0-Flash-Lite, and QWen-3-emb-06b, which are contemporary models as of early 2026. The comparison set appears to represent current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 contains ablation studies examining how the number of training samples and the exploration hyperparameter α affect performance. These systematically vary individual components (augmentation rounds, α value) to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only classification accuracy as the evaluation metric. No additional metrics (F1, precision, recall, AUC, etc.) are reported. The number of parameters is shown as a proxy for efficiency but is not a performance metric."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates classification accuracy on automated benchmarks; human evaluation of model outputs is not relevant to the claims being made about classification accuracy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows separate counts for initial (training) data and test data for each dataset. Results in Tables 2 and 3 are reported on these held-out test sets (e.g., 227 test examples for MultiWD, 5394 for CUB200)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as aggregate accuracy across each dataset with no per-class or per-category breakdown. Given datasets have 6 to 200 classes, per-category performance could vary significantly but is not shown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes some cases where few-shot in-context learning underperforms zero-shot due to long context, and observes that homogeneous synthetic data hurts performance at high sample counts. However, no qualitative failure case analysis or error breakdown is provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows that generating too many synthetic samples causes accuracy to decrease due to synthetic data homogeneity—a genuine negative result where more augmentation is worse. The paper also notes cases where few-shot in-context learning is worse than zero-shot."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 40% accuracy improvement when training with less than 5 initial data samples of each class,' which is supported by results in Tables 2 and 3 (e.g., Flowers102 with Voyage: +43.20%, CUB200 with CLIP: +35.42%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily via ablation: comparing calibration with ADAMAB vs. random augmentation vs. no augmentation, and varying the exploration hyperparameter alpha. These are controlled single-variable manipulations. The schema states: 'ablation studies... ARE causal claims — check whether the ablation design is adequate (controlled single-variable manipulation counts as YES).' The ablation design holds all other variables constant (same embedding model, same dataset, same calibrator architecture, same total augmentation budget) while varying only the augmentation strategy, which is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims ADAMAB achieves 'extraordinary calibration accuracy in pattern recognition tasks of different domains' and the conclusion calls it 'highly suitable for resource-constrained environments,' but experiments are limited to 6 datasets with specific embedding models. Claims of broad generalization are not adequately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for results, such as whether ADAMAB's advantage stems from the specific datasets chosen, the particular generative models used for augmentation, or the architecture choices of the calibrator. No threats-to-validity section is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o-mini,' 'Gemini2.0-Flash-Lite,' 'GPT-Image-1-mini,' and 'OpenAI-text-embedding-3-small' without snapshot dates or API version identifiers. Marketing names without version pinning do not satisfy this criterion."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B.2 provides prompt templates with placeholders (e.g., '{label1}: {description1}', '{existing_queries}') but does not provide the actual filled values used in experiments. Templates without the actual fill values do not allow reconstruction of the prompts sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 and Table 4 report Adam optimizer with learning rate η₀=0.005, weight decay λ=0.0001, cosine annealing scheduler, batch sizes, number of training rounds, augmentation rounds (∆n), and exploration hyperparameter α per dataset."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. It trains lightweight calibrators on top of fixed embedding models via gradient descent, with no agent loops, tool use, or multi-step reasoning scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 states the number of initial training examples and test examples per dataset, but the paper does not describe how the initial subset was selected from the full datasets or what preprocessing was applied to the raw data."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. Section 4.3 briefly notes issues with synthetic data homogeneity but this is discussed as an ablation finding, not a systematic limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity section exists. The paper does not discuss specific threats such as the small number of initial samples, potential data leakage from public benchmarks into generative models, or the choice of evaluation datasets."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion makes broad claims about ADAMAB being 'highly suitable for resource-constrained environments' without bounding results to the specific models, datasets, and task types tested. The paper does not explicitly state what it did NOT test."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All six datasets are publicly available standard benchmarks (MultiWD on Authorea, FQS published in ACM CCS 2024, TREC from ACL proceedings, OxfordPets/Flowers102/CUB200 from standard CV repositories). Raw data can be independently obtained."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes each dataset with its source, number of classes, and domain. The datasets are well-documented external benchmarks with cited original papers describing their collection procedures."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses existing public benchmark datasets and no human participants were recruited. NA applies as there is no human participant data collection."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states how many initial samples per class were used (e.g., 5/class for text datasets) but does not describe how these specific samples were selected from the full datasets or whether selection was random or stratified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper. The five co-authors from Accenture (a commercial consulting firm) have a potential financial interest in the evaluated approach but no funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Minxue Tang (Duke University) and the remaining five authors (Yangyang Yu, Aolin Ding, Maziyar Baran Pouyan, Taha Belkhouja, Yujia Bao) from 'Center for Advanced AI, Accenture.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Five of six authors are from Accenture, a commercial entity that stands to benefit from the positive evaluation of this AI calibration framework. The funder/employer is not independent of the research outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The Accenture affiliation of five authors represents a potential commercial interest in the method being shown to work, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o-mini and Gemini2.0-Flash-Lite as baselines evaluated on classification benchmarks, but does not state the training data cutoff dates for any of the models used. Some benchmark datasets (e.g., TREC from 2001-2002, Flowers102 from 2008) predate any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether any of the benchmark datasets (TREC, OxfordPets, Flowers102, CUB200) may appear in the training data of the evaluated models (GPT-4o-mini, Gemini). These are long-established public datasets that are likely in LLM training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Datasets like TREC (2001), OxfordPets (2012), Flowers102 (2008), and CUB200 (2011) were published long before the training cutoffs of models like GPT-4o-mini. The paper does not acknowledge or discuss the risk that baseline LLMs may have been trained on these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; experiments are computational benchmark evaluations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; no IRB approval is needed."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; this is a computational study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; no blinding is applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o-mini for text generation and GPT-Image-1-mini for image generation across multiple training epochs without reporting API costs or tokens consumed. Compute costs for the calibrators themselves are not quantified beyond noting the number of trainable parameters."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B states experiments ran on 'a single MacBook Pro with a single M4 Max chip and 36GB memory' but no wall-clock training times or total compute budget (GPU/CPU hours, API call counts) are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "ADAMAB achieves up to 40% accuracy improvement over the base embedding model when training with fewer than 5 initial data samples per class.",
       "evidence": "Tables 2 and 3 show improvements such as +43.20% for Flowers102 with Voyage-multimodal-3 and +35.42% for CUB200 with CLIP-VIT-Large. Abstract and Section 4.2 make this claim.",
       "supported": "moderate"
     },
     {
       "claim": "ADAMAB consistently outperforms random data augmentation across all tested datasets and embedding models.",
       "evidence": "Tables 2 and 3 show ADAMAB accuracy exceeds random augmentation in all 18 reported conditions (6 datasets × ~3 embedding models each). Section 4.2 discusses this.",
       "supported": "moderate"
     },
     {
       "claim": "The MAB-based adaptive augmentation strategy provably converges to a stationary point with the rate O(1/T) + O(sqrt(log(T)/T)).",
       "evidence": "Theorem 2 and its proof in Appendix A.2 establish this convergence under Assumptions 1 (β-smoothness) and 2 (bounded gradients).",
       "supported": "strong"
     },
     {
       "claim": "ADAMAB achieves higher accuracy than GPT-4o-mini and other large decoder LLMs on implicit pattern recognition tasks despite being much smaller.",
       "evidence": "Tables 2 and 3 show ADAMAB's calibration (e.g., 61.63% on TREC vs. GPT-4o-mini's 60.03%, 89.54% on FQS vs. 80.31%) using only ~2.6M additional parameters. Section 4.2 discusses this.",
       "supported": "moderate"
     },
     {
       "claim": "Generating too many synthetic samples decreases accuracy due to homogeneity in the synthetic data.",
       "evidence": "Figure 3 shows accuracy declining when average samples per class exceed the sweet spot for each dataset. Section 4.3 discusses this as a limitation of small generative models.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ADAMAB is a lightweight embedding calibration framework that trains small neural networks on top of fixed embedding models (CLIP, OpenAI text embeddings) using adaptive data augmentation guided by a Multi-Armed Bandit UCB algorithm. On six classification datasets spanning text and image modalities with 2-5 initial samples per class, ADAMAB achieves 10-40% accuracy improvements over base embedding models and outperforms random augmentation. The paper provides a theoretical convergence guarantee showing ADAMAB achieves O(1/T) + O(sqrt(log(T)/T)) convergence. A key negative finding is that generating too many synthetic samples decreases accuracy due to homogeneity in outputs from smaller generative models.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All results in Tables 2 and 3 are point estimates with no confidence intervals, standard deviations, or indication of whether experiments were run multiple times. With test sets as small as 227 examples (MultiWD), the uncertainty in accuracy estimates is non-trivial."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "Baseline LLMs (GPT-4o-mini, Gemini2.0-Flash-Lite) are evaluated on datasets like TREC (2001), OxfordPets (2012), and Flowers102 (2008) that almost certainly appear in their training data. The paper does not acknowledge this confound, which could artificially deflate baseline performance if test examples were memorized, or inflate it if the models have learned the label space."
     },
     {
       "flag": "No model version pinning",
       "detail": "The paper uses 'GPT-4o-mini' and 'Gemini2.0-Flash-Lite' without snapshot dates or API version identifiers. These models are updated frequently, making exact reproduction impossible and results non-reproducible over time."
     },
     {
       "flag": "Commercial affiliation bias",
       "detail": "Five of six authors are employed at Accenture's Center for Advanced AI, a commercial consulting firm that would benefit from a successful AI calibration framework. No competing interests statement is present, and no funding disclosure is made."
     },
     {
       "flag": "Single accuracy metric",
       "detail": "The paper reports only top-1 classification accuracy. For datasets like FQS (safety policy classification), recall or precision per category would be more meaningful for the stated application domain."
     },
     {
       "flag": "Hyperparameter selection on test data",
       "detail": "Appendix B states 'we conduct experiments on a wide range of hyperparameters and select the best hyperparameters with the highest accuracy.' It is unclear whether hyperparameter selection used a validation set separate from the test set, raising risk of overfitting reported results to the test set."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": [
         "Tom Brown",
         "Benjamin Mann",
         "Nick Ryder"
       ],
       "year": 2020,
       "relevance": "Foundational paper on LLM few-shot learning, motivating the few-shot setting evaluated in this paper."
     },
     {
       "title": "Learning transferable visual models from natural language supervision",
       "authors": [
         "Alec Radford",
         "Jong Wook Kim",
         "Chris Hallacy"
       ],
       "year": 2021,
       "relevance": "Introduces CLIP, the primary vision embedding model evaluated and calibrated in this paper."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": [
         "Edward J Hu",
         "Yelong Shen",
         "Phillip Wallis"
       ],
       "year": 2022,
       "relevance": "Baseline efficient fine-tuning method that ADAMAB is positioned against as not requiring model access."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": [
         "Patrick Lewis",
         "Ethan Perez",
         "Aleksandra Piktus"
       ],
       "year": 2020,
       "relevance": "RAG framework that is part of the embedding retrieval context motivating this work."
     },
     {
       "title": "Active learning principles for in-context learning with large language models",
       "authors": [
         "Katerina Margatina",
         "Timo Schick",
         "Nikolaos Aletras",
         "Jane Dwivedi-Yu"
       ],
       "year": 2023,
       "relevance": "Directly related work on adaptive learning for LLM in-context learning, cited as motivation for ADAMAB."
     },
     {
       "title": "Data augmentation using large language models: Data perspectives, learning paradigms and challenges",
       "authors": [
         "Bosheng Ding",
         "Chengwei Qin",
         "Ruochen Zhao"
       ],
       "year": 2024,
       "relevance": "Survey of LLM-based data augmentation, directly in scope for the survey as related work on the same topic."
     },
     {
       "title": "A survey on data synthesis and augmentation for large language models",
       "authors": [
         "Ke Wang",
         "Jiahui Zhu",
         "Minjie Ren"
       ],
       "year": 2024,
       "relevance": "Survey on synthetic data augmentation for LLMs, directly relevant as background for this paper's contribution."
     },
     {
       "title": "\"Do Anything Now\": Characterizing and Evaluating In-The-Wild Jailbreak Prompts on Large Language Models",
       "authors": [
         "Xinyue Shen",
         "Zeyuan Chen",
         "Michael Backes",
         "Yun Shen",
         "Yang Zhang"
       ],
       "year": 2024,
       "relevance": "Source paper for the Forbidden Question Set benchmark used in evaluation; relevant to LLM safety evaluation research."
     },
     {
       "title": "From selection to generation: A survey of LLM-based active learning",
       "authors": [
         "Yu Xia"
       ],
       "year": 2025,
       "relevance": "Recent survey of LLM-based active learning, directly in the topic area of this paper and the broader survey scope."
     },
     {
       "title": "SimCSE: Simple contrastive learning of sentence embeddings",
       "authors": [
         "Tianyu Gao",
         "Xingcheng Yao",
         "Danqi Chen"
       ],
       "year": 2021,
       "relevance": "Foundational embedding model work cited in related work on embedding-based similarity search."
     },
     {
       "title": "LLM-DA: Data augmentation via large language models for few-shot named entity recognition",
       "authors": [
         "Junjie Ye",
         "Nuo Xu",
         "Yikun Wang"
       ],
       "year": 2024,
       "relevance": "Related work on LLM-based data augmentation for few-shot NLP tasks, directly comparable to ADAMAB's text augmentation component."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": [
         "Lingjiao Chen",
         "Matei Zaharia",
         "James Zou"
       ],
       "year": 2023,
       "relevance": "Related work on cost-efficient use of LLMs, relevant to the cost reduction motivation of ADAMAB."
     }
   ]
 }