ai-research-survey

scan.json (24935B)

---

 {
   "paper": {
     "title": "Does Prompt Formatting Have Any Impact on LLM Performance?",
     "authors": ["Jia He", "Mukund Rungta", "David Koleczek", "Arshdeep Sekhon", "Franklin X Wang", "Sadid Hasan"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2411.10541"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Prompt formatting (plain text, Markdown, JSON, YAML) significantly affects GPT model performance, with up to 40% variation on code translation tasks for GPT-3.5-turbo. Larger models (GPT-4) are more robust to format changes but still show significant sensitivity. No single format is universally optimal, and format preferences do not transfer across model generations (e.g., GPT-3.5 prefers JSON while GPT-4 prefers Markdown on MMLU).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or mention of code release found anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses publicly available benchmarks: MMLU, HumanEval, NER Finance (OpenAI Evals), FIND, CODEXGLUE, and HumanEval-X. All are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details provided. Only mentions Azure OpenAI as the platform."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3-8 in Appendix E report results with ± notation (e.g., '59.705 ± 16.594' in Table 3)."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "One-sided matched pairs t-tests are used throughout (Section 3.1), with p-values reported in Table 1. Almost all p < 0.01."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance differences with baseline context (e.g., Table 1 shows max and min scores for each model/dataset, and Section 3.2 reports percentage improvements like '200% improvement' and '300% boost'). The raw scores provide sufficient context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. For NER Finance, '500 examples randomly sampled' with no justification. MMLU uses full test set (14,079) and HumanEval uses all 164 samples, but no power analysis or justification is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in Tables 3-8 (e.g., '59.705 ± 16.594' in Table 3 for MMLU). These appear to be across examples rather than across runs, but the ± notation is consistently provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study's design compares four prompt formats against each other, with each serving as a baseline for the others. The comparison is the point of the paper."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The prompt formats compared (plain text, Markdown, JSON, YAML) are all contemporary and widely used formats. Prior work (Sclar et al. 2023, Voronov et al. 2024) is cited as context."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The study varies a single factor (prompt format) across conditions. There is no multi-component system to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: accuracy (MMLU), pass@1 (HumanEval), F1-like entity matching (NER Finance), BLEU (CODEXGLUE, HumanEval-X), string indicator metric (FIND), plus consistency (Section 4) and IoU transferability (Section 5) as meta-metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates prompt format sensitivity via automated benchmarks. Human evaluation of model outputs is not relevant to the claims about format sensitivity."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used: MMLU dev set for few-shot examples and test set (14,079 questions) for evaluation (Appendix B). HumanEval, CODEXGLUE, etc. all have established test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (7 benchmarks), per model (4 models), and per format (4 formats). Figure 5 provides per-domain MMLU breakdown (humanities, social science, STEM, others)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section D.2 discusses the GPT-4-32k-0613 failure on HumanEval with JSON format (21.95% vs 76.22% plain text), analyzing that the model generated chain-of-thought in plain text but failed to generate code."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that no single format is universally optimal, which is itself a negative result about format transferability. The GPT-4-32k JSON failure on HumanEval is a notable negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'up to 40% variation in code translation' and 'GPT-4 more robust' are supported by Table 1 (CODEXGLUE Java2CS: 78.4 vs 66.5 for GPT-35-turbo) and Figure 6 showing lower CMD for GPT-4 models."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims prompt format 'impacts' performance. The controlled experimental design holds content constant while varying only format, which is adequate for this causal claim. Statistical significance is tested via matched pairs t-test."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title asks about 'LLM Performance' generally but the study only tests OpenAI GPT models. The Limitations section (Section 7) acknowledges this but the title and abstract make broad claims about 'LLMs' while testing only one model family."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why formats affect performance. Section D.2 hypothesizes about the GPT-4-32k JSON failure being related to 'laziness' but does not discuss confounds like tokenization differences across formats, training data format distributions, or whether the observed differences are artifacts of specific API versions."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures benchmark scores (accuracy, pass@1, BLEU) and claims these show format sensitivity. The claims match the granularity of measurements — no proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 specifies exact model versions: 'gpt-35-turbo-0613', 'gpt-35-turbo-16k-0613', 'gpt-4-32k-0613', and 'gpt-4-1106-preview'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C (Table 2) provides the four prompt templates with their structure. The NER Finance examples in the appendix show actual prompt text including persona, instructions, output format, and placeholders. The fill values for the examples are described as coming from the datasets."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is mentioned only for MMLU consistency experiment ('set the temperature to zero', Section 4.2). No systematic reporting of temperature, top-p, max tokens, or other sampling parameters for the main experiments across all benchmarks."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study sends prompts directly to GPT models via Azure OpenAI API."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes dataset usage: MMLU dev set for few-shot examples and test set for evaluation, NER Finance 500 random samples, HumanEval all 164 samples, FIND strings category with 500 functions and 5 example pairs, CODEXGLUE 1000 test samples. The data selection and preparation is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' is a dedicated section discussing three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific limitations: (1) only GPT models tested, not LLaMA/Gemini/PaLM/Phi; (2) missing HTML/XML formats; (3) no interaction with other prompt engineering techniques like few-shot count variation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what was NOT tested: other model families, other format types (HTML, XML), and interaction with other prompt engineering techniques. Section 2.3 also explains why GPT models specifically were chosen."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw model outputs, per-example predictions, or response logs are released. Only aggregate statistics are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes each dataset's source, size, and evaluation metric. The benchmarks used are all well-documented public datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from benchmark data to formatted prompts to model outputs to scores is not documented step by step. How prompts were constructed programmatically, how API calls were made, and how responses were parsed is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: five authors from Microsoft, one from MIT. Microsoft is clearly identified."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft (employer of 5/6 authors) is a major investor in OpenAI and resells GPT models via Azure. The paper evaluates GPT models accessed through Azure OpenAI. Microsoft has a financial interest in GPT model adoption, making the funder non-independent."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found. Microsoft's investment in OpenAI and Azure OpenAI revenue from GPT model access are undisclosed financial interests relevant to a paper evaluating GPT models."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of training data cutoff dates for any of the GPT models used. The paper tests models on benchmarks like HumanEval (2021) and MMLU (2020) without stating when model training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark problems appeared in GPT model training data. HumanEval and MMLU were published years before the models' training cutoffs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2020), HumanEval (2021), and CODEXGLUE (2021) were all published well before GPT-3.5 and GPT-4 training. No discussion of contamination risk despite this being a study of benchmark performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of API costs, token consumption, or inference time despite running 4 models × 4 formats × 6 benchmarks through Azure OpenAI."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total API spend, compute time, or resources used for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Section 4.2 mentions setting temperature to zero for MMLU consistency experiments, but no multi-seed analysis is reported for any benchmark."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per condition is not stated. It appears results are from single runs, but this is never made explicit."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning is involved. The study compares fixed prompt formats on existing benchmarks using API models."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection is involved. All four formats are evaluated and reported."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper performs 28 t-tests (7 benchmarks × 4 models) in Table 1 alone, plus additional comparisons. No correction for multiple comparisons (Bonferroni, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not propose a new system — it compares existing prompt formats on existing models. No self-comparison bias is present."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All conditions use the same compute (single API call per example). Compute differences across conditions are negligible."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 6 benchmarks without discussing whether they adequately capture the construct of 'LLM performance' or whether format sensitivity on these specific benchmarks generalizes to real-world usage."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Prompts are sent directly to models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. MMLU (2020), HumanEval (2021), CODEXGLUE (2021) all predate the models' training cutoffs, meaning models may have seen solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information. For MMLU, few-shot examples from the dev set are provided in-context, but no analysis of whether this creates leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence for any benchmark."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied despite using benchmarks that predate model training."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-3.5-turbo's performance varies by up to 40% depending on prompt template in code translation tasks",
       "evidence": "Table 1 shows CODEXGLUE Java2CS performance ranges from 66.5 (plain text) to 78.4 (JSON) for GPT-35-turbo-0613, roughly an 18% relative difference. The 40% figure appears to reference other tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Performance differences due to format changes are statistically significant across nearly all model-benchmark combinations",
       "evidence": "Table 1 shows p-values < 0.01 for 27 of 28 model-benchmark combinations (exception: GPT-4-1106-preview on HumanEval, p=0.055).",
       "supported": "strong"
     },
     {
       "claim": "Larger models (GPT-4) are more consistent across prompt formats than smaller models (GPT-3.5)",
       "evidence": "Figure 2 shows GPT-4 consistency scores above 0.5 vs GPT-3.5 below 0.5 on MMLU. Figure 6 shows lower CMD values for GPT-4 series across all benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "There is no universally optimal prompt format — preferences do not transfer across GPT model generations",
       "evidence": "Figure 3 and Section 5.2 show IoU below 0.2 between GPT-3.5 and GPT-4 series for top templates. GPT-3.5 prefers JSON on MMLU while GPT-4 prefers Markdown (Table 1).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4-turbo demonstrates greater resilience to prompt format changes compared to predecessors",
       "evidence": "Figure 6 shows GPT-4-1106-preview has CMD consistently below 0.036 across all benchmarks, lower than all other models.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Conflict of interest not disclosed",
       "detail": "Five of six authors are Microsoft employees. Microsoft is a major investor in OpenAI and resells GPT models through Azure. The paper evaluates GPT models via Azure OpenAI without disclosing this financial relationship."
     },
     {
       "flag": "No contamination discussion",
       "detail": "All benchmarks (MMLU 2020, HumanEval 2021, CODEXGLUE 2021) predate the GPT models' training data. If models memorized benchmark solutions, format-dependent performance differences could reflect format-dependent recall of memorized answers rather than genuine reasoning sensitivity."
     },
     {
       "flag": "Multiple comparisons without correction",
       "detail": "28+ t-tests reported in Table 1 with no family-wise error rate correction. While most p-values are very small (< 0.001), the one borderline case (GPT-4-1106-preview on HumanEval, p=0.055) would likely not survive correction."
     },
     {
       "flag": "Single model family tested with broad claims",
       "detail": "The title asks about 'LLM Performance' broadly but only OpenAI GPT models are tested. Claims about format sensitivity may not generalize to other architectures or training approaches."
     },
     {
       "flag": "Variance may not represent run-to-run variability",
       "detail": "The ± values in Tables 3-8 appear to be standard deviation across test examples, not across multiple runs. It is unclear whether experiments were run multiple times. With temperature=0 stated only for MMLU consistency, the stochasticity of other experiments is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "Quantifying language models' sensitivity to spurious features in prompt design",
       "authors": ["Melanie Sclar", "Yejin Choi", "Yulia Tsvetkov", "Alane Suhr"],
       "year": 2023,
       "arxiv_id": "2310.11324",
       "relevance": "Directly related study on LLM sensitivity to fine-grained prompt modifications like separators and capitalization."
     },
     {
       "title": "Mind your format: Towards consistent evaluation of in-context learning improvements",
       "authors": ["Anton Voronov", "Lena Wolf", "Max Ryabinin"],
       "year": 2024,
       "arxiv_id": "2401.06766",
       "relevance": "Shows that fixed templates in evaluation may lead to misleading conclusions about LLM capabilities."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "HumanEval benchmark used in this study, foundational for LLM code generation evaluation."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks"],
       "year": 2020,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU benchmark used in this study, widely used for LLM evaluation."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique that interacts with prompt formatting choices."
     },
     {
       "title": "A systematic survey of prompt engineering in large language models: Techniques and applications",
       "authors": ["Pranab Sahoo"],
       "year": 2024,
       "arxiv_id": "2402.07927",
       "relevance": "Comprehensive survey of prompt engineering techniques in LLMs."
     },
     {
       "title": "Table meets LLM: Can large language models understand structured table data?",
       "authors": ["Yuan Sui"],
       "year": 2024,
       "relevance": "Closest related work examining format influence on LLM understanding, focused on tabular data."
     },
     {
       "title": "CodeGeeX: A pre-trained model for code generation with multilingual evaluations on HumanEval-X",
       "authors": ["Qinkai Zheng"],
       "year": 2023,
       "arxiv_id": "2303.17568",
       "relevance": "HumanEval-X benchmark used for multilingual code translation evaluation."
     },
     {
       "title": "Calibrate before use: Improving few-shot performance of language models",
       "authors": ["Tony Z. Zhao"],
       "year": 2021,
       "arxiv_id": "2102.09690",
       "relevance": "Addresses LLM sensitivity to prompt construction, related to calibration of few-shot predictions."
     }
   ]
 }