ai-research-survey

scan-v5.json (26176B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Does Prompt Formatting Have Any Impact on LLM Performance?",
     "authors": [
       "Jia He",
       "Mukund Rungta",
       "David Koleczek",
       "Arshdeep Sekhon",
       "Franklin X Wang",
       "Sadid Hasan"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2411.10541",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are empirically supported: performance variations documented in Table 1 (up to 54pp on HumanEval); GPT-4's robustness confirmed in Figure 6 via Coefficient of Mean Deviation; significant variations shown via p-values < 0.001.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Paper uses ablation design: isolates prompt format as sole variable by keeping semantic content identical across formats (Appendix C). Causal claims about format effects are justified by this controlled experimental design.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Scope appropriately bounded to OpenAI GPT models and 6 specific benchmarks. Section 7 explicitly acknowledges GPT-only focus and plans future work on other LLM families.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Paper documents that format affects performance but does not discuss why—no exploration of tokenization, training data distribution, or transformer architecture mechanisms. Section D.2 speculates about laziness vs. format processing but doesn't systematically explore alternatives.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims directly match measurements: 'format affects performance' is measured via accuracy (MMLU), pass@1 (HumanEval), BLEU (code translation). No proxy conflation.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated Section 7 'Limitations' explicitly lists three specific limitations: GPT-only focus, formats excluded (HTML/XML), other prompt engineering dimensions not varied.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats named: 'GPT-based models' vs. other LLM families, missing format types, other prompt design elements held constant. Not boilerplate disclaimers.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Scope clearly bounded to 4 formats, 6 benchmarks, 4 GPT models, temperature=0 for consistency. Implicitly states findings don't explain mechanism or provide universal format guidance.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source statement, no acknowledgments section identifying funder. Affiliations with Microsoft and MIT stated but no explicit funding disclosure.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Microsoft and MIT for each author.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Microsoft affiliation evaluates OpenAI's competing models (not Microsoft's own Copilot), suggesting independence. However, funding source unconfirmed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement provided. No declaration of patents, equity, or consulting relationships.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key metrics formally defined: 'sensitivity' (Section 3.1, with formula), 'consistency' (Section 4.1, with formula), 'transferability' (IoU, Section 5.1). Prompt templates shown in examples (Figure 1, Appendix C).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Contribution explicitly stated: 'first to compare impact of different prompt formats on GPT models' performance across various tasks.' Three research questions clearly framed (Sensitivity, Consistency, Transferability).",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section A (Related Work) extensively cites prior prompt engineering research. Clearly distinguishes this work: 'Our research diverges...by examining global prompt format modifications' vs. prior work on fine-grained changes. Positions contribution within landscape.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository, GitHub link, or code availability statement provided. Experiments depend on proprietary OpenAI API access.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Uses standard public benchmarks (MMLU, HumanEval, CODEXGLUE, NER Finance, HumanEval-X, FIND). Raw data available via original benchmark sources.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only mentions 'Azure OpenAI' without Python version, dependency versions, or Docker spec. No requirements.txt or environment file referenced.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions. Would require Azure OpenAI access, which model versions to call, and how to replicate exact API setup. Prompts partially shown (Appendix C) but not comprehensively.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Tables 3-8 report standard deviations (±) for each condition. Figures 4 and 9 display error bars.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports one-sided matched pairs t-tests with p-values; all comparisons yield p<0.001 except HumanEval GPT-4-1106 (p=0.055).",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute performance differences shown (Max/Min in Table 1, e.g., 59.7 vs. 50.0). Relative differences noted in text ('40%', '200%', '300%'). Formal effect size metrics (Cohen's d) not reported but magnitude is clear.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "Sample sizes mentioned (MMLU 14,079, HumanEval 164, FIND 500) but not justified. No power analysis or rationale for sufficiency provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Standard deviations reported in Tables 3-8 and error bars displayed in Figures 4 and 9 across all runs.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Compares 4 format templates (plain text, Markdown, JSON, YAML), with one implicitly serving as baseline. Max/Min comparisons frame within-design baselines.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Tests current GPT models (GPT-3.5-turbo and GPT-4 variants from 2023-2024). Baselines are contemporary and appropriate for the research question.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Systematic ablation: varies format while keeping prompt content, persona, instructions, examples identical (Appendix C). Format is isolated as sole experimental variable.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Uses task-specific metrics (accuracy, pass@1, BLEU, NER F1) plus meta-metrics (consistency, IoU). Evaluates from three angles (sensitivity, consistency, transferability).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable—benchmark evaluation only, no subjective quality assessment needed.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All benchmarks use standard test/dev splits. MMLU uses official dev set for few-shot, test set for evaluation. HumanEval, CODEXGLUE, etc. use established splits.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Figure 5 breaks MMLU by domain (STEM, humanities, social science, other) and shows per-domain format sensitivity. Less detailed for other benchmarks.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section D.2 explicitly discusses GPT-4-32k's extreme HumanEval failure with JSON (21.95% vs. 76.2% plain text), hypothesizing 'laziness' in chain-of-thought generation.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Reports no universal optimal format; IoU scores mostly <0.2 for cross-model comparisons, indicating non-transferability. GPT-3.5 consistency <0.5 is noted as negative.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model IDs provided: 'gpt-35-turbo-0613', 'gpt-35-turbo-16k-0613', 'gpt-4-32k-0613', 'gpt-4-1106-preview'. Date snapshots implicit in version tags.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C shows complete prompt templates for NER Finance task with placeholder examples. Only one task shown in detail, but templates and structure fully transparent.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Temperature=0 reported for MMLU. For FIND, defers to (Schwettmann et al., 2023) settings without repeating. Top-p, max_tokens not reported.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding; straight prompt engineering. Not applicable.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Describes sampling (NER Finance: random 500/500, FIND: random 500/500) and split usage (MMLU dev for few-shot, test for eval). Limited but adequate for standard benchmarks.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All data sourced from published benchmarks (MMLU, HumanEval, CODEXGLUE, etc.). Raw benchmark data publicly available.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": false,
           "answer": false,
           "justification": "Paper does not collect new data; uses existing benchmarks. NA.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects. NA.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Pipeline is simple: ingest public benchmark → vary prompt format → run on API → collect results. Described adequately for standard benchmarks.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No explicit training data cutoff date stated for GPT-3.5 or GPT-4. ArXiv date 2024-11-15 but model training dates not discussed.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether MMLU, HumanEval, CODEXGLUE, or other benchmarks were present in GPT training data. Potential overlap not addressed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No analysis of whether benchmarks entered training data before model cutoff. Standard benchmarks may be over-represented in GPT training.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. NA.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. NA.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. NA.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. NA.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. NA.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. NA.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. NA.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No API costs, token counts, or latency metrics reported. Significant cost incurred but not quantified.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Total computational budget not stated. No count of API calls, tokens consumed, or estimated costs provided.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Prompt formatting significantly affects GPT model performance, with variations ranging up to 54pp (300% relative) depending on task and model",
       "evidence": "Table 1 documents max/min performance across formats for each benchmark; HumanEval GPT-4-32k shows 21.95% (JSON) vs. 76.2% (plain text); text reports '200% improvement' and '300%' for FIND and HumanEval respectively",
       "supported": "strong"
     },
     {
       "claim": "Larger models (GPT-4) are more robust to prompt format changes than smaller models (GPT-3.5)",
       "evidence": "Figure 6 shows Coefficient of Mean Deviation (CMD) consistently lower for GPT-4 (0.035–0.043) than GPT-3.5 (0.035–0.176); Figure 2 shows consistency scores GPT-3.5 <0.5, GPT-4 >0.5",
       "supported": "strong"
     },
     {
       "claim": "No universal optimal format exists even within the same GPT model family",
       "evidence": "Section 5.2 shows IoU <0.2 for cross-model pairs; GPT-3.5 prefers JSON while GPT-4 prefers Markdown (Section 5.2); conclusion: 'no single format excelling universally'",
       "supported": "strong"
     },
     {
       "claim": "Model sensitivity to format is task-agnostic, not contingent on task-specific skill requirements",
       "evidence": "Figure 5 breaks MMLU by domain (STEM, humanities, social science); shows performance spread exists across all domains. Section D.1: 'Model's sensitivity...is a general characteristic, rather than being contingent on specific skills'",
       "supported": "moderate"
     },
     {
       "claim": "GPT-3.5 outputs show low consistency across formats (<0.5 identical responses), while GPT-4 exceeds 0.5",
       "evidence": "Figure 2 and Figure 8 explicitly show consistency scores for MMLU and FIND datasets. MMLU: GPT-3.5 'displayed low consistency, with scores below 0.5, and only 16% identical responses between Markdown and JSON'",
       "supported": "strong"
     },
     {
       "claim": "Model size correlates with increased robustness to prompt format variation",
       "evidence": "CMD analysis (Figure 6) shows inverse relationship: GPT-4 lower CMD than GPT-3.5; Section D.2 concludes 'larger models are more robust to template variation'",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "Prompt formatting significantly impacts GPT model performance, with variations exceeding 300% relative improvement on some tasks (e.g., HumanEval). GPT-4 demonstrates substantially greater robustness to format changes (lower Coefficient of Mean Deviation, higher consistency scores) compared to GPT-3.5. No single prompt format universally optimizes performance across different GPT models or even within the same family. Format sensitivity is a general model characteristic independent of task-specific skill requirements or domain expertise.",
   "red_flags": [
     {
       "flag": "No code release",
       "detail": "Experiments depend entirely on proprietary OpenAI API access via Azure. Cannot be reproduced without credentials and budget."
     },
     {
       "flag": "Training data contamination not addressed",
       "detail": "No analysis of whether MMLU, HumanEval, CODEXGLUE, or other benchmarks were present in GPT training data. Standard benchmarks are likely over-represented in LLM training corpora."
     },
     {
       "flag": "No mechanistic explanation",
       "detail": "Paper documents that format matters but does not explain why. No investigation of tokenization, embedding distribution, transformer attention patterns, or other mechanisms."
     },
     {
       "flag": "GPT-only scope limits generalizability",
       "detail": "Results restricted to OpenAI models; does not test open-source LLMs (LLaMA, Phi), proprietary competitors (Gemini, Claude), or smaller models. Acknowledged limitation but weakens contribution."
     },
     {
       "flag": "Sample size not justified",
       "detail": "Benchmark samples mentioned (MMLU 14,079, HumanEval 164) but no power analysis or statistical justification for sufficiency provided."
     },
     {
       "flag": "Hyperparameter transparency incomplete",
       "detail": "Temperature reported only for MMLU. Top-p, max_tokens, and other sampling parameters not consistently reported across all experiments."
     },
     {
       "flag": "Limited mechanistic depth",
       "detail": "Section D.2 speculates about 'laziness' in GPT-4-32k's JSON output on HumanEval but does not systematically investigate. Alternative explanations (tokenization, prompt structure parsing) unexplored."
     },
     {
       "flag": "No alternative explanation discussion",
       "detail": "Does not consider why format sensitivity exists (e.g., training data composition, tokenization artifact, transformer architecture preference) or whether findings might be confounded."
     }
   ],
   "cited_papers": [
     {
       "title": "Quantifying language models' sensitivity to spurious features in prompt design",
       "relevance": "Directly related—prior work showing LLM sensitivity to fine-grained prompt changes (Sclar et al., 2023); this paper extends to global format variations."
     },
     {
       "title": "Mind your format: Towards consistent evaluation of in-context learning improvements",
       "relevance": "Closely related—argues that evaluation standards must marginalize across prompt formats to avoid spurious conclusions (Voronov et al., 2024)."
     },
     {
       "title": "You don't need a personality test to know these models are unreliable: Assessing the reliability of large language models on psychometric instruments",
       "relevance": "Related on output consistency and reliability under prompt variation (Shu et al., 2023)."
     },
     {
       "title": "Evaluating large language models trained on code (HumanEval benchmark)",
       "relevance": "Benchmark paper defining HumanEval, one of six benchmarks used in this study (Chen et al., 2021)."
     },
     {
       "title": "Measuring massive multitask language understanding (MMLU)",
       "relevance": "Benchmark paper defining MMLU, the largest benchmark used in this study (Hendrycks et al., 2020)."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "relevance": "Foundational prompting technique cited; this paper tests whether CoT benefits persist across format variations (Wei et al., 2023)."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "relevance": "Foundational in-context learning work; motivates few-shot example ordering as confounding variable this paper controls (Brown et al., 2020)."
     },
     {
       "title": "Table meets LLM: Can large language models understand structured table data?",
       "relevance": "Most closely related concurrent work; provides cursory format exploration but limited to tabular data (Sui et al., 2024)."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners should care about format choice, but paper provides no guidance on which format to select. Findings are task-model specific without generalizable heuristics."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Moderately surprising that semantic content can vary 20–40pp based on formatting alone. Challenges assumption that meaning is content-agnostic, but not radical for practitioners aware of prompt brittleness."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety implications. Paper is purely about engineering optimization for task performance, not safety or alignment."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Minimal drama. Finding that evaluation standards may be invalid if ignoring format has some methodological bite, but not a major controversy or conflict narrative."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Reproducible on Azure OpenAI but requires API credentials, budget, and knowledge of how to call proprietary models. Not accessible to casual users without cost."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Strong brands: Microsoft Research authorship, evaluating OpenAI GPT models. Both highly recognized in AI/ML community."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42266742",
         "title": "The Rise and Fall of Ideas' Popularity [pdf]",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42266742",
         "created_at": "2024-11-28T16:54:44Z"
       },
       {
         "hn_id": "44854721",
         "title": "Does Prompt Formatting Have Any Impact on LLM Performance?",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44854721",
         "created_at": "2025-08-10T12:23:36Z"
       },
       {
         "hn_id": "45930419",
         "title": "A Large-Scale Computational Analysis of Errors in ArXiv Papers",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45930419",
         "created_at": "2025-11-14T18:52:29Z"
       },
       {
         "hn_id": "33707451",
         "title": "Knowledge Graph Generation from Text",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=33707451",
         "created_at": "2022-11-22T16:21:57Z"
       }
     ],
     "top_points": 3,
     "total_points": 7,
     "total_comments": 0
   }
 }