ai-research-survey

scan.json (23118B)

---

 {
   "paper": {
     "title": "An Investigation on Group Query Hallucination Attacks",
     "authors": ["Kehao Miao", "Xiaolong Jin"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.19321",
     "doi": "10.48550/arXiv.2508.19321"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Group Query Attack (GQA) significantly degrades performance of fine-tuned LLMs when QGS increases from 1 to 2, with models collapsing to outputting a single option. GQA can trigger backdoors injected via 0.5% poisoned training data, causing models to preferentially output option A. For non-fine-tuned models, GQA has limited impact on multiple-choice and translation tasks but pronounced degradation on code generation and mathematical reasoning, with code performance dropping to near-zero for some models at QGS≥2.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code link, or supplementary material archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available standard benchmarks: HumanEval, MedMCQA, PubMedQA, Aqua-RAT, MathQA, WMT20-MLQE-Task1. The paper references these with citations."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, requirements file, or dependency list is provided. Hardware details are not mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The appendix describes prompt templates and hyperparameters but lacks executable reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '53.3 / 19.7') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GQA 'significantly degrades performance' but provides no statistical significance tests — comparisons are made by visually comparing raw accuracy numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results show baseline vs. degraded accuracy with enough context to compute effect size (e.g., llama2-7b MedMCQA drops from 53.3% to 19.7%, mistral-7b from 61.1% to 32.1%). The magnitude of degradation is clear from the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of datasets, number of models, or number of evaluation examples. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 mentions performing random partitioning three times and computing average metrics for Q3, but no standard deviations or variance measures are reported in any table. For Q1/Q2, only a single partition is used."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "QGS=1 (single query) serves as the baseline comparison for all experiments. Performance at QGS=1 is always reported alongside higher QGS values."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models tested include contemporary choices: Llama 3, Mistral 7B, Gemma 7B, Qwen 1.5, Mixtral 8x7B. These were reasonably current at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper does not investigate which aspects of group queries cause degradation (e.g., context length vs. task confusion vs. attention dilution)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy for multiple-choice/code/math tasks and sacreBLEU for translation tasks. It also reports predominant output option proportions."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant — all tasks have automated ground-truth evaluation (accuracy, sacreBLEU, unit tests for HumanEval)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard train/test splits from established benchmarks. Section 3.3 describes randomly partitioning evaluation data to separate first queries from additional queries. Section 4.1 notes use of corresponding test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (MedMCQA, PubMedQA, Aqua-RAT, MathQA, HumanEval, WMT20), per model, and per QGS level across extensive tables in the appendix."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper observes that models collapse to outputting a single option but does not analyze why or show qualitative failure examples beyond the single Figure 1 illustration."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GQA has limited impact on multiple-choice and translation tasks for non-fine-tuned models (Section 4.3, Q3), which is a negative result for the attack's generality."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GQA degrades fine-tuned models (supported by Tables 1-2), triggers backdoors (Tables 2, 8), and affects reasoning tasks (Tables 3, 14-15). All are demonstrated in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GQA 'degrades performance' and 'triggers backdoors' — causal language — but does not control for confounds like increased context length alone. The mechanism is not isolated; longer input could explain the degradation regardless of multi-query structure."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Group Query Hallucination Attacks' broadly, but results are limited to specific 7B-33B parameter models on specific benchmarks. No acknowledgment that results may not apply to larger models or different task types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations. The degradation could be due to increased context length, attention dilution, or format confusion, but none of these are analyzed or discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy and sacreBLEU directly and does not overclaim beyond these metrics. Claims about 'performance degradation' match the granularity of what is measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are listed: llama2-7b, mistral-7b-v0.1, gemma-7b, qwen-7b, gpt-j-6b, mixtral-8x7b-v0.1, llama-33b, plus aligned versions with version identifiers (mistral-7b-it-v0.3, gemma1.1-7b-it, qwen1.5-7b-chat, llama3-8b-instruct)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates with actual fill values are provided in Appendix B (Figures 3-5, Table 4), including system prompts, user/assistant prefixes, and task-specific values for all configurations."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 reports: 10% warmup ratio, final LR decayed to 10% of peak, 3 epochs, LR 2e-5, batch size 64, sequence length 2048, greedy search decoding."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct prompting of LLMs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the evaluation procedure: random partitioning into additional queries and first queries, fixing order of additional queries, and averaging over 3 partitions for Q3. Appendix B describes prompt formatting. Backdoor injection procedure described in Section 4.2 (1% sampling, combining into group queries, 0.5% of total)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations' and discusses three specific limitations: limited scenarios tested, only first-query responses analyzed, and insufficient model fine-tuning due to time constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 mentions specific limitations: 'users tend to ask more open-ended questions rather than restricting themselves to the specific tasks mentioned in this paper' and 'this paper only examines metrics related to responses to the first query.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section acknowledges some gaps but does not explicitly state what the results do NOT show — e.g., does not state that results may not apply to larger models, closed-source APIs, or real-world conversational settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs, model predictions, or intermediate data are released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides detailed descriptions of each dataset with sizes, splits, and sources. Section 4.2 describes the backdoor data generation procedure."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents the evaluation pipeline: random partitioning, fixing additional query order, evaluating first query response. Section 4.2 documents the backdoor injection pipeline: 1% sampling where answer=A, combining into group queries, reintegrating at 0.5% of total."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Science and Technology of China and Purdue University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the pre-trained models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark data appeared in model training sets. HumanEval (2021) could have been seen by later models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and all models tested were trained after that. MedMCQA and other benchmarks are also publicly available. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API cost is reported despite running extensive experiments across 7+ models and multiple QGS levels."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget is mentioned despite fine-tuning 7 models on multiple datasets."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Section 3.3 mentions 3 random partitions for Q3, but no seed variation for model training or inference."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states the random partitioning is performed three times for Q3, and once for Q1/Q2. This establishes the number of runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper uses fixed hyperparameters from prior work (Chen et al.) but does not state whether any tuning was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the specific fine-tuning configuration was selected. Hyperparameters are adopted from prior work without justification for this specific setting."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons, despite comparing many models across many datasets and QGS values."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose GQA and evaluate it themselves without acknowledging any evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "GQA is not a method that trades compute for performance — it is an attack that changes input format. Compute differences between QGS levels are negligible relative to the question."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks used (HumanEval, MedMCQA, etc.) are valid measures of the capabilities claimed to be degraded by GQA."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — models are prompted directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models were trained on the benchmark data. HumanEval (2021), MedMCQA (2022), and other benchmarks predate several models tested."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The few-shot examples and prompt format could provide hints not available in natural usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence for any benchmark used."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "GQA significantly degrades performance of fine-tuned models, with most models collapsing to outputting a single option at QGS=2",
       "evidence": "Table 1 and Table 5 show accuracy drops across 7 models on 4 datasets (e.g., mistral-7b on MedMCQA: 61.1% → 32.1%). Table 6 shows >98% single-option output frequency at QGS=2.",
       "supported": "strong"
     },
     {
       "claim": "GQA triggers potential backdoors in LLMs fine-tuned on poisoned datasets, causing models to preferentially output option A",
       "evidence": "Table 2 and Table 8 show that models fine-tuned on 0.5% backdoor data output A at 83-100% rate at QGS=2, compared to mixed option distributions for non-backdoored models.",
       "supported": "moderate"
     },
     {
       "claim": "GQA has limited impact on multiple-choice and translation tasks for non-fine-tuned models but pronounced effect on code and mathematical reasoning",
       "evidence": "Table 3: MedMCQA accuracy drops only ~3-4pp (e.g., llama3-8b-it: 59.9→57.9) but HumanEval drops from 28.5% to 0.0% for gemma-7b-it at QGS≥2. Math reasoning also shows larger drops (gemma-7b-it: 43.3→22.5).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims 'significant' performance degradation throughout but never performs any statistical test. All comparisons are raw accuracy numbers without uncertainty quantification."
     },
     {
       "flag": "Context length confound not controlled",
       "detail": "GQA increases context length. The paper does not control for whether degradation is caused by the multi-query structure specifically or simply by longer inputs. Table 13 shows input tokens increase substantially with QGS."
     },
     {
       "flag": "Backdoor claim is weak",
       "detail": "The Q2 backdoor experiment injects group queries with answer A into training, then tests with group queries — finding model outputs A. This may simply be the model learning the training distribution pattern rather than demonstrating a meaningful backdoor risk."
     },
     {
       "flag": "No code or data release",
       "detail": "Despite fine-tuning 7+ models and running extensive experiments, no code, fine-tuned model weights, or experimental outputs are released."
     },
     {
       "flag": "Missing variance/uncertainty",
       "detail": "For Q1/Q2, only a single random partition is used. For Q3, three partitions are averaged but no standard deviation is reported. Impossible to assess result stability."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, used as evaluation dataset in this paper."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "authors": ["Evan Hubinger"],
       "year": 2024,
       "relevance": "Studies backdoor persistence in LLMs through safety training, directly relevant to the backdoor triggering investigated in this paper."
     },
     {
       "title": "BadChain: Backdoor Chain-of-Thought Prompting for Large Language Models",
       "authors": ["Zhen Xiang"],
       "year": 2024,
       "relevance": "Demonstrates backdoor attacks via chain-of-thought prompting in LLMs."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F Liu"],
       "year": 2024,
       "relevance": "Studies how LLMs process long contexts, directly relevant to the context accumulation effects studied in GQA."
     },
     {
       "title": "The Reversal Curse: LLMs trained on 'A is B' fail to learn 'B is A'",
       "authors": ["Lukas Berglund"],
       "year": 2024,
       "relevance": "Studies LLM failure modes in generalization, part of the broader failure mode literature this paper contributes to."
     },
     {
       "title": "Large language models can be easily distracted by irrelevant context",
       "authors": ["Freda Shi"],
       "year": 2023,
       "relevance": "Studies LLM distractibility, closely related to the context accumulation degradation observed in GQA."
     },
     {
       "title": "DecodingTrust: A Comprehensive Assessment of Trustworthiness in GPT Models",
       "authors": ["Boxin Wang"],
       "year": 2024,
       "relevance": "Comprehensive trustworthiness evaluation of LLMs including robustness and adversarial attacks."
     },
     {
       "title": "Backdooring Instruction-Tuned Large Language Models with Virtual Prompt Injection",
       "authors": ["Jun Yan"],
       "year": 2023,
       "relevance": "Studies backdoor injection methods for instruction-tuned LLMs via fine-tuning."
     }
   ]
 }