ai-research-survey

scan.json (25389B)

---

 {
   "paper": {
     "title": "Ask Me Anything: A Simple Strategy for Prompting Language Models",
     "authors": [
       "Simran Arora",
       "Avanika Narayan",
       "Mayee F. Chen",
       "Laurel Orr",
       "Neel Guha",
       "Kush Bhatia",
       "Ines Chami",
       "Frederic Sala",
       "Christopher Ré"
     ],
     "year": 2022,
     "venue": "International Conference on Learning Representations",
     "arxiv_id": "2210.02441",
     "doi": "10.48550/arXiv.2210.02441"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/HazyResearch/ama_prompting. The Reproducibility Statement (Section 7) states: 'We release prompts and code for reproducing all benchmark results for few-shot and AMA prompting, and our diagnostic evaluation splits.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All 20 evaluation benchmarks are publicly available standard datasets (SuperGLUE, ANLI, AGNews, DBPedia, SST2, Amazon, DROP, NQ, WebQs, RealTimeQA, StoryCloze). The paper also releases diagnostic evaluation splits via the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using A100 NVIDIA GPUs (Appendix A) and lists models downloaded from HuggingFace, but does not provide a requirements.txt, Dockerfile, or detailed dependency/library version listing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The Reproducibility Statement (Section 7) states code is released for 'reproducing all benchmark results.' The appendix (Section H) provides all prompts used for each of the 20 datasets, and Appendix A provides experiment details including model names and metrics."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5a reports 95% confidence intervals for the AMA lift across model sizes. Results in Tables 1 and 3 include ± notation (e.g., '83.9±0.0', '37.8±0.2') across 5 random seeds. The overall lift is reported as '10.2% ± 6.1% absolute.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AMA 'exceeds the average few-shot performance of the GPT3-175B model on 15 of 20 benchmarks' but provides no statistical significance tests for any comparative claims. Differences are assessed by comparing point estimates only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout: '10.2% ± 6.1% absolute (21.4% ± 11.2% relative) lift' (Section 5.2), '72% performance improvement' converting prompt formats (Section 3.2), and per-task absolute point differences in Tables 1 and 3."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The diagnostic tasks use only 50 manually-labeled samples per task (Appendix E), with no justification for why 50 is adequate. The number of random seeds (5) and prompt-chains (3-6) are stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 3 report standard deviations across 5 random seeds for AMA results (e.g., '83.9±0.0', '37.8±0.2'). Figure 5a shows 95% confidence intervals across tasks."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against few-shot baselines (k=3 for open-source models and k∈[32..70] for GPT-3), majority vote aggregation, weighted majority vote (Table 5), Self-Consistency (Table 7), and calibration methods (Zhao et al., 2021)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods: GPT-3-175B few-shot (Brown et al., 2020), calibration (Zhao et al., 2021), Self-Consistency (Wang et al., 2022b), and T0 with PromptSource prompts (Sanh et al., 2022). These were all recent at time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies are presented: (1) Table 4 ablates prompt reformatting vs. aggregation, (2) Table 5 ablates aggregation methods (MV vs. WS vs. WMV vs. Pick Best vs. AMA no deps), (3) Figure 6 varies amount of unlabeled data, (4) Section 3.2 ablates prompt formats (restrictive vs. cloze vs. QA)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses matching accuracy for most tasks, text F1 for DROP/RealTimeQA, span overlap accuracy for WebQ/NQ, and F1a accuracy for MultiRC (Appendix A.2). The information-theoretic conditional entropy H(y|ŷ) is also used as an evaluation lens (Section 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of AMA's outputs is conducted. All evaluation is automated via benchmark metrics. For a prompting method that claims to improve LLM quality, human evaluation of output quality would be relevant but is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard test sets for all 20 benchmarks. The weak supervision algorithm uses an unlabeled dataset constructed from the test set plus 1000 training samples with labels ignored (Appendix A.3), maintaining proper separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task category (NLU, NLI, Classification, QA) in Tables 1 and 3, by model family in Section 5.2, by model size in Figure 5a, and per-class F1 in Table 10."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G provides an extended error analysis with three categories of failure modes: knowledge errors (factual and commonsense), instruction-following errors (invalid output class), and long-context errors. Concrete examples are provided for each."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that AMA 'performs worse than MV by at most 1.0 points' on 4 of 20 tasks (Section 5.3), that the method has 'limitations as seen on NQ' (Section 5.1), and that T0 sees lower lift because fine-tuning 'may compromise its in-context learning abilities' (Section 5.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'average performance lift of 10.2% over the few-shot baseline' and 'GPT-J-6B model to match and exceed the performance of few-shot GPT3-175B on 15 of 20 popular benchmarks' — both supported by Tables 1 and 3 and Section 5.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., removing QA reformatting or changing aggregation method) which are controlled single-variable manipulations. Section 3.2 isolates prompt format effects, Table 4 separates reformatting from aggregation, and Table 5 isolates aggregation method effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to tested settings: 'across open-source model families (EleutherAI, BLOOM, OPT, and T0) and sizes (125M-175B parameters)' (abstract). The conclusion notes limitations for 'private data' applications. Section 5.1 distinguishes between context-dependent and knowledge-intensive tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Appendix F investigate why QA prompts are effective by analyzing pretraining corpus frequencies, offering an empirical alternative explanation. The paper discusses that T0's lower lift may be due to fine-tuning on zero-shot prompts. The error analysis (Appendix G) discusses knowledge limitations as confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 specifies exact model variants: GPT-Neo-125M, GPT-Neo-1.3B, GPT-J-6B, GPT-NeoX-20B, BLOOM-560M, BLOOM-1.7B, BLOOM-7.1B, BLOOM-176B, OPT-125M, OPT-1.3B, OPT-6.7B, OPT-13B, OPT-175B, T0-3B, and GPT-3 davinci endpoint. These are specific model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H provides the full prompt text for all 20 datasets, including both few-shot and AMA prompt()-chains with actual in-context demonstrations. The prompts are released in the code repository as well."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature and sampling parameters are not reported for the main experiments. For Self-Consistency comparisons (Table 7), temperatures are listed (0.0, 0.3, 0.5, 0.6, 0.7), but the main AMA experiments lack these details. The WS structure learning uses 'default parameters from Varma et al. [2019]' without specifying them."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-step prompt()-chain pipeline is described in detail: question() prompts transform inputs to questions (Section 3.3), answer() prompts generate answers, and weak supervision aggregates votes (Section 3.4). Figure 1 provides a visual workflow diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H documents dataset sizes (train/test splits) for each benchmark. Appendix A.2 documents the metrics used per task. The mapping from open-ended answers to output classes is described in Section 3.2. Appendix A.3 describes the unlabeled dataset construction."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "While there is no dedicated 'Limitations' section, Sections 5.1 and 6 discuss limitations substantively. The error analysis in Appendix G serves as a de facto limitations discussion covering knowledge gaps, instruction-following failures, and long-context issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific threats: (1) AMA provides less lift on knowledge-intensive tasks where factual recall is needed (Section 5.1), (2) T0 sees lower lift due to fine-tuning compromising in-context learning (Section 5.2), (3) instruction-following failures on multi-class tasks (Appendix G), (4) long-context degradation (Appendix G)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly states: 'we find limitations when tasks cannot rely on the latent knowledge' and gives specific examples (NQ, RealTimeQA, Amazon Instant Video). The conclusion acknowledges applicability constraints for 'private data or require operating over large amounts of data.' The error analysis categorizes what AMA does not handle."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All 20 benchmark datasets are publicly available. The code repository is released for reproducing results, and diagnostic evaluation splits are also released. Per-task results are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are standard benchmarks with published collection procedures (cited in Appendix H). The diagnostic tasks (Appendix E) are described with their construction methodology and 50 manually-labeled samples per task."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. All evaluation uses standard NLP benchmark datasets and automated metrics."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: input → question() prompt → LLM generates question → answer() prompt → LLM generates answer → map to output space → weak supervision aggregation (Section 3.3, 3.4, Algorithm 1). Appendix A.3 documents WS data construction."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section discloses extensive funding: DARPA, NIH, NSF, ONR, Moore Foundation, and numerous industry sponsors. Individual fellowship support is listed for SA and LO."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Stanford University (most authors), Numbers Station (Ines Chami), and University of Wisconsin-Madison (Frederic Sala). The Acknowledgements also mention Together Computer, Numbers Station, and Snorkel."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from government agencies (DARPA, NIH, NSF, ONR) and academic foundations, which do not have a direct stake in AMA outperforming specific prompting methods. While industry sponsors are listed, the paper evaluates open-source models, not products from those sponsors."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Christopher Ré is associated with Snorkel (mentioned in acknowledgements), which is based on weak supervision — the same technique central to AMA. This potential conflict is not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the models used (GPT-J, BLOOM, OPT, T0, GPT-3 davinci), despite evaluating them on public benchmarks that could be in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. Many benchmarks used (SuperGLUE, ANLI, etc.) were publicly available before model training and could have been included in training data, but this is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates models on benchmarks like SuperGLUE (2019), ANLI (2020), and others that were publicly available before model training. No contamination analysis is provided. This is especially relevant since the GPT-3 comparison uses 'davinci' which was likely trained on data including these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All evaluation is automated using benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports total inference cost in seconds for GPT-J-6B on three tasks (RTE: 8310s, WSC: 3141s, AGNews: 53200s). Appendix B.3 reports WS latency (13.0s without dependencies, 84.3s with dependencies)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using A100 GPUs (Appendix A) and that computation was provided by Together Computer, but does not report total GPU hours, total API spend for GPT-3 davinci calls, or overall computational budget."
       }
     }
   },
   "claims": [
     {
       "claim": "AMA enables the open-source GPT-J-6B model to match or exceed the few-shot GPT3-175B on 15 of 20 popular benchmarks.",
       "evidence": "Table 1 shows per-task comparisons where GPT-J-6B with AMA (QA + WS) surpasses GPT-3-175B few-shot (k∈[32..70]) on 15 tasks. Section 5.1 discusses this result.",
       "supported": "strong"
     },
     {
       "claim": "AMA provides an average performance lift of 10.2% ± 6.1% absolute (21.4% ± 11.2% relative) over few-shot baseline across models.",
       "evidence": "Section 5.2 reports this figure across 14 LLMs from 4 model families on 7 tasks. Figure 5a shows the lift by model size with 95% confidence intervals.",
       "supported": "strong"
     },
     {
       "claim": "Open-ended QA prompt formats outperform restrictive prompt formats.",
       "evidence": "Section 3.2 shows converting CB, RTE, WSC from restrictive to open-ended formats improves average performance from 41.7% to 71.5% (+72%) on GPT-J-6B. Pretraining corpus analysis (Appendix F) shows QA structures appear ~1000x more frequently.",
       "supported": "strong"
     },
     {
       "claim": "Weak supervision aggregation outperforms majority vote by up to 8.7 points.",
       "evidence": "Section 5.3 and Table 5 show WS vs. MV comparisons across 16 tasks. WS does not underperform MV by more than 1.0 point on any task while achieving up to 8.7 points improvement.",
       "supported": "strong"
     },
     {
       "claim": "WS recovers dependencies among prompts to boost performance by up to 9.6 points on 9 tasks.",
       "evidence": "Section 5.3 states this, comparing AMA (WS) with AMA (no deps). Table 5 provides per-task numbers showing RTE improves from 65.1 to 75.1 with dependency modeling.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AMA (Ask Me Anything) is a prompting strategy that reformats task inputs into open-ended QA formats using the LLM itself, then aggregates multiple prompt outputs via weak supervision. The method achieves an average 10.2% absolute lift over few-shot baselines across 14 open-source LLMs from 4 families (125M-175B parameters) on 20 benchmarks. Most notably, GPT-J-6B (6B parameters) with AMA matches or exceeds few-shot GPT-3-175B on 15 of 20 tasks. The gains are largest on context-dependent tasks (reading comprehension) and smaller on knowledge-intensive tasks (factual recall).",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "All benchmarks (SuperGLUE, ANLI, etc.) were publicly available before model training. No analysis of potential train/test overlap is provided, which could inflate benchmark numbers for all models tested."
     },
     {
       "flag": "Unfair shot comparison with GPT-3",
       "detail": "The GPT-3-175B baseline uses k∈[32..70] shots from Brown et al. (2020), while AMA uses 3-6 prompt()-chains plus unlabeled data from the test set and 1000 training samples. The comparison is between different prompting paradigms with different information access, which is not always clearly flagged."
     },
     {
       "flag": "Snorkel conflict of interest undisclosed",
       "detail": "Christopher Ré is a co-founder of Snorkel AI, which commercializes weak supervision — the core aggregation technique in AMA. Snorkel is listed in the acknowledgements but this conflict of interest is not formally disclosed."
     },
     {
       "flag": "No significance tests for comparative claims",
       "detail": "Claims that AMA 'exceeds' GPT-3-175B on 15/20 tasks are based on point estimate comparisons without statistical tests. Some margins are very small (e.g., StoryCloze 87.8 vs 87.7) and could be within noise."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper establishing few-shot prompting as a paradigm; provides the primary baselines for AMA comparison."
     },
     {
       "title": "Calibrate before use: Improving few-shot performance of language models",
       "authors": ["Tony Z. Zhao", "Eric Wallace", "Shi Feng", "Dan Klein", "Sameer Singh"],
       "year": 2021,
       "arxiv_id": "2102.09690",
       "relevance": "Addresses prompt sensitivity and calibration methods for LLMs; complementary approach to managing prompt brittleness."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Prompt aggregation method via sampling diverse reasoning paths; directly compared against AMA in Table 7."
     },
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting strategy for LLM reasoning; AMA offers an alternative approach to improving LLM performance."
     },
     {
       "title": "Multitask prompted training enables zero-shot task generalization",
       "authors": ["Victor Sanh", "Albert Webson", "Colin Raffel"],
       "year": 2022,
       "relevance": "Introduces T0 model and PromptSource; used as a baseline model family and prompt source in AMA evaluation."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "InstructGPT/RLHF approach to improving LLM following; complementary training-based strategy vs. AMA's inference-time approach."
     },
     {
       "title": "Snorkel: Rapid training data creation with weak supervision",
       "authors": ["Alexander Ratner", "Stephen H. Bach", "Henry Ehrenberg"],
       "year": 2017,
       "doi": "10.14778/3157794.3157797",
       "relevance": "Core weak supervision framework that AMA adapts for prompt aggregation; foundational to AMA's aggregation strategy."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Studies scaling properties of LLMs; AMA demonstrates prompt aggregation as an alternative axis for scaling performance."
     },
     {
       "title": "OPT: Open Pre-trained Transformer Language Models",
       "authors": ["Susan Zhang", "Stephen Roller", "Naman Goyal"],
       "year": 2022,
       "arxiv_id": "2205.01068",
       "relevance": "Open-source LLM family used in AMA evaluation; relevant to assessing prompting methods across model families."
     },
     {
       "title": "On the opportunities and risks of foundation models",
       "authors": ["Rishi Bommasani", "Drew A Hudson", "Ehsan Adeli"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Comprehensive survey of foundation model capabilities and risks; frames the broader context of LLM prompting research."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McClandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Establishes scaling laws for LLMs; AMA suggests prompt aggregation as another axis beyond parameter scale."
     }
   ]
 }