ai-research-survey

scan.json (31453B)

---

 {
   "paper": {
     "title": "The Perplexity Paradox: Why Code Compresses Better Than Math in LLM Prompts",
     "authors": ["Warren Johnson"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.15843"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper validates the 'perplexity paradox' mechanism: code syntax tokens have 79× higher perplexity than content words and are preserved by compression algorithms, while numerical values in math problems have low perplexity and are pruned despite being task-critical. Signature injection recovers +34pp in pass rate (Cohen's h = 0.890) at aggressive compression ratios. TAAC, a task-aware adaptive compression algorithm, achieves 95.6% quality preservation with 21.8% cost savings, outperforming fixed-ratio compression. Length-controlled analysis (ANCOVA) confirms the code vs. CoT compression dichotomy is independent of prompt length.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL: 'We release our extended benchmark suite, per-token analysis tools, and TAAC implementation at https://github.com/micoverde/taac-llm-compression' (Abstract, Section 10)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (HumanEval, MBPP, GSM8K, MATH, ARC-Challenge, MMLU-STEM) and claims to release the extended benchmark suite at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or library version specifications are provided in the paper. The software environment needed to reproduce the experiments is not documented."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. A GitHub URL is provided but the paper itself does not contain commands to run or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported throughout: Wilson score CIs in Table 5 for MBPP pass rates, 95% CIs for Cohen's d in Table 3, and the ANCOVA results include standard error estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Multiple significance tests are used: ANCOVA F-tests (Table 1, p < .001), two-way ANOVA (Table 2, p = .0002), Cochran-Armitage trend test (p < 0.001), Kolmogorov-Smirnov test for bin matching (D = 0.089, p = .312), and TOST equivalence testing is proposed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d with interpretations at each compression level (Table 3), Cohen's h = 0.890 for signature preservation (Section 8.3), η² partial eta-squared values in Tables 1 and 2, and percentage point differences throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or formal justification for sample sizes. The paper states n=1,800 for MBPP, n=488 for signature experiments, n=723 for perplexity analysis, and 220 for TAAC evaluation, but does not justify why these sizes were chosen or whether they provide adequate statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Std dev is reported for perplexity analysis (Table 12) but this is across token categories, not experimental runs. The main experimental results (Tables 5, 14) report only Wilson score CIs derived from point estimates, not variance across repeated experimental runs. No mention of multiple runs or seed-level variation for pass rate experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "TAAC is compared against four baselines: uncompressed baseline (r=1.0), fixed r=0.7, fixed r=0.6, and task-based fixed compression (Table 15). The signature experiment compares baseline vs. signature injection conditions. Multi-algorithm comparison includes Random compression as a control."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Compression baselines include LLMLingua-2 (Pan et al., 2024), LLMLingua-1 (Jiang et al., 2023), and Selective Context (Li et al., 2023b), which are contemporary methods. Prior task-aware methods ATACompressor (Huang et al., 2024) and TACO-RL (Shi et al., 2024) are discussed."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly mentions 'Component ablation shows quality gating contributes most to quality preservation' (Section 8.5) but does not present the ablation results in a table or with detailed data. The actual ablation study is referenced but not shown."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass rate, quality retention, cost savings (Table 15), Cohen's d and Cohen's h effect sizes, η² for variance explained, NameError/AssertionError rates (Table 7), and perplexity measurements."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via test suite pass/fail on benchmarks. Human evaluation could assess compression quality beyond binary pass/fail (e.g., code readability, partial correctness)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For TAAC, the quality predictor is trained on Phase 1 experiments (~50K samples) and evaluated on a separate 'synthetic validation set (220 prompts: 100 code, 100 CoT, 20 hybrid)' (Section 8.5). The main benchmark experiments use standard test sets without tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by compression ratio (Tables 5, 11, 14), by task type (code vs. CoT in Table 11), by error type (Table 7), by token category (Table 12), and by algorithm type (Section 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Error type analysis in Table 7 shows specific failure modes: 86.1% NameError at baseline (function identity collapse), AssertionError after signature injection (logic failures). The perplexity paradox analysis explains why compression fails for math tasks (numbers pruned despite being task-critical)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "MBPP results show severe degradation at aggressive compression (3.7% at r=0.3, Table 14). Table 15 shows TAAC achieves lower cost savings (21.8%) than fixed r=0.6 (41.2%), demonstrating a tradeoff. Fixed r=0.7 and Task-Based Fixed strategies are shown as dominated (Table 15, Figure 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims validation 'across six code benchmarks (HumanEval, MBPP, HumanEval+, MultiPL-E in Python/JavaScript/Java)' but the paper only shows detailed results for MBPP. HumanEval+, MultiPL-E Python/JavaScript/Java results are not presented in any results table. The abstract also claims '7%' improvement for TAAC; actual data shows 6.5pp (95.6% vs 89.1%). The cross-language generalization claim is unsupported by the presented results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The signature preservation experiment (Section 8.3) is a controlled intervention with clear before/after comparison, providing causal evidence for the perplexity paradox mechanism. The ANCOVA and bin-matched sampling (Section 3) use appropriate causal inference designs to rule out the length confound."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims broadly 'Why Code Compresses Better Than Math in LLM Prompts' but results are from specific benchmarks (primarily MBPP for code) with three specific models (Claude 3 Haiku, DeepSeek-Chat, GPT-4o-mini). The abstract claims cross-language generalization across Python/JavaScript/Java but only Python results are shown. The limitations section partially bounds scope but the title and abstract overclaim."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 is entirely devoted to ruling out prompt length as an alternative explanation for the code vs. CoT dichotomy, using ANCOVA and bin-matched sampling. Section 6 addresses whether the pattern is an artifact of a specific compression algorithm. The paper systematically tests alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k rates on benchmarks and frames results in terms of 'quality preservation' under compression. The measurements (pass/fail on test suites) closely match the claims (compression's effect on code generation quality). The proxy-outcome gap is minimal and appropriately scoped."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 lists 'Claude 3 Haiku, DeepSeek-Chat, GPT-4o-mini' without specific version identifiers, snapshot dates, or API versions. 'DeepSeek-Chat' is especially vague. Per schema guidance, marketing names without snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is shown in the paper. The paper describes what the compression algorithms do and references a GitHub repository, but does not include the actual prompts or system instructions used for LLM evaluation in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM inference hyperparameters are reported (temperature, top-p, max tokens). Compression ratios are specified, and TAAC's architecture parameters are described, but the inference settings for the underlying LLMs are absent."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates prompt compression on standard benchmark problems without multi-step agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents stratified sampling of 500 problems from MBPP's 974 (Section 4.3), the bin-matched sampling procedure with KS validation (Section 3.2.2), and the 12-category token classification methodology (Section 5.2, Table 8)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.3 'Limitations' provides four specific limitation bullet points covering code benchmark scope, pilot model dependency, TAAC training distribution, and missing agentic/multi-turn evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'Code benchmarks focus on function completion; longer code files may exhibit different patterns,' 'Perplexity analysis uses a single pilot model; different model families may yield different patterns,' and 'We do not evaluate agentic or multi-turn scenarios.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states what was NOT tested: longer code files, different model families for perplexity analysis, distribution shift for the quality predictor, and agentic/multi-turn scenarios."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to release tools and implementation but does not explicitly state that raw experimental data (individual trial results, per-token perplexity measurements) is available. The GitHub repository contains 'benchmark suite, per-token analysis tools, and TAAC implementation' but not explicitly raw data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The experimental design sections describe data collection: MBPP benchmark characteristics (Table 4), trial counts per condition (Section 4.3), perplexity token analysis methodology (Section 5.2), and signature preservation experiment design (Section 4.5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (MBPP, HumanEval, GSM8K, etc.)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw model outputs to reported metrics is not fully documented. The experimental design specifies 9,000 total trials (Section 4.3) but results show only 1,800 (Table 5) without explaining how trials were aggregated or which models' results are shown. How pass/fail was determined from model outputs is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgments section states: 'Computational resources were provided by Microsoft Azure.' The funding/compute source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is listed as 'Bona Opera Studios, Sammamish, WA, USA' with an email address. The paper does not evaluate products from the author's organization."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Microsoft Azure provided compute resources. While Microsoft has LLM products, the paper does not evaluate Azure-specific products and the funder has no direct financial stake in whether prompt compression works better for code than math."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. There is no disclosure of patents, equity, or other financial interests related to prompt compression technology."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three models used (Claude 3 Haiku, DeepSeek-Chat, GPT-4o-mini). This is critical since benchmarks like MBPP (2021) and HumanEval (2021) predate these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' training data includes MBPP, HumanEval, or other benchmark problems. All benchmarks used (2018-2021) were public well before the 2026 models were trained."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), MBPP (2021), GSM8K (2021), MATH (2021), and ARC (2018) were all published years before the models' training cutoffs. Contamination risk is significant but not discussed anywhere in the paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. All experiments involve automated benchmark evaluation of LLM compression."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Costs are reported for each experimental phase: MBPP validation '$1.52' (Section 4.3), Phase 1 '$0.08', Phase 2 '$0.59', Phase 3 '$5.35' (Section 6.2). TAAC cost savings are quantified at 21.8% (Table 15). General API cost context ($3-$15 per million tokens) is provided in Section 1."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Estimated runtime of 6.2 hours for MBPP validation (Section 4.3), per-phase cost estimates are given, and computational resources from Microsoft Azure are acknowledged. Compression algorithm latencies are reported: LLMLingua-2 ~150ms, LLMLingua-1 ~3s, Selective Context ~500ms."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. Results appear to be from single experimental runs without assessing variability across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Trial counts are explicitly stated: n=1,800 for MBPP validation, n=488 for signature preservation, n=723 for perplexity analysis, 220 prompts for TAAC evaluation, N=600 for length-controlled analysis."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "TAAC includes a trained 2-layer MLP quality predictor and a DistilBERT classifier, but no hyperparameter search budget is reported for training these components. The compression ratios are pre-specified but model training details are missing."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "TAAC thresholds are justified with explicit rationale: r*_code = 0.65 as 'conservative buffer above the r = 0.6 cliff,' r*_cot = 0.80 as 'minimal compression for reasoning tasks' (Table 10). The compression ratios tested are pre-specified from prior work, not selected post-hoc."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs numerous statistical tests (ANCOVA, two-way ANOVA, KS test, Cochran-Armitage, multiple Cohen's d calculations across 6 compression levels) without applying any multiple comparison correction (Bonferroni, Holm, or Benjamini-Hochberg)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TAAC is the authors' own system, compared against baselines the authors implemented. No discussion of author-evaluation bias or independent evaluation. The comparison in Table 15 is entirely self-administered."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 15 and Figure 4 explicitly show performance (quality preservation) as a function of cost savings for each strategy. Compression algorithm latencies are also reported (LLMLingua-2 ~150ms, LLMLingua-1 ~3s, Selective Context ~500ms per prompt)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MBPP, HumanEval, or other benchmarks actually measure the capabilities the paper claims to evaluate. The paper uses these benchmarks without questioning construct validity or comparing with alternative evaluation methods."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The experiments evaluate prompt compression on standard benchmarks without agent scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All benchmarks used (HumanEval 2021, MBPP 2021, GSM8K 2021, MATH 2021, ARC 2018) predate the 2026 models by years. Temporal leakage is not discussed despite being a clear risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the compressed prompts or evaluation setup leaks answer information. The signature injection experiment deliberately provides additional information but this is not framed as a leakage consideration."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between benchmark problems and model training data. MBPP and HumanEval problems are publicly available and likely in the training data of 2026 models."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection methods are applied. No canary strings, membership inference tests, n-gram overlap analysis, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "The compression threshold (r ≥ 0.6) generalizes beyond HumanEval to MBPP, with pass rates degrading systematically from 54.7% at r=1.0 to 3.7% at r=0.3.",
       "evidence": "MBPP validation with n=1,800 trials across 6 compression ratios (Table 5, Section 4.4). Cochran-Armitage trend test confirms linear trend (p < 0.001).",
       "supported": "strong"
     },
     {
       "claim": "The code vs. CoT compression dichotomy is independent of prompt length, reflecting task structure rather than prompt brevity.",
       "evidence": "ANCOVA controlling for length: F(5, 2019) = 57.84, p = .000108, η² = .081 (Table 1). Length-matched samples show larger interaction effect (η² = .102 vs .081, Table 2).",
       "supported": "strong"
     },
     {
       "claim": "Python syntax tokens exhibit 79× higher perplexity than content words, explaining their preservation under compression, while numbers have low perplexity despite being task-critical.",
       "evidence": "Per-token perplexity analysis of n=723 tokens (Table 12, Figure 2). Python syntax mean PPL = 928,636 vs content words 11,697. Numbers PPL = 9,195. Kept vs removed tokens: 71,000× difference.",
       "supported": "moderate"
     },
     {
       "claim": "Signature injection recovers +34 percentage points in pass rate under aggressive compression, with NameError rates dropping from 86.1% to 6.1%.",
       "evidence": "Controlled experiment with n=488 pooled trials across 3 compression ratios (Tables 6, 7, 13). Cohen's h = 0.890 (very large effect). Consistent across r=0.3, 0.4, 0.5.",
       "supported": "strong"
     },
     {
       "claim": "TAAC achieves 95.6% quality preservation with 21.8% cost savings, outperforming fixed-ratio compression by 6.5 percentage points in quality.",
       "evidence": "Comparison against 4 baselines on 220-prompt synthetic validation set (Table 15, Figure 4). Pareto-optimal on cost-quality frontier.",
       "supported": "moderate"
     },
     {
       "claim": "The compression threshold generalizes across six code benchmarks and programming languages (Python, JavaScript, Java).",
       "evidence": "The abstract and Section 1 claim cross-language generalization, but detailed results are only presented for MBPP (Python). HumanEval+, MultiPL-E Python/JavaScript/Java results are not shown in any results table.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Abstract overclaims benchmark coverage",
       "detail": "The abstract lists 'six code benchmarks (HumanEval, MBPP, HumanEval+, MultiPL-E in Python/JavaScript/Java)' but detailed results are only presented for MBPP. The claimed cross-language generalization across Python/JavaScript/Java is not supported by presented data."
     },
     {
       "flag": "Complete absence of contamination analysis",
       "detail": "All benchmarks used (HumanEval 2021, MBPP 2021, GSM8K 2021, MATH 2021, ARC 2018) were publicly available 3-8 years before the 2026 models were trained. No contamination discussion, training cutoffs, or decontamination measures are mentioned. This is a fundamental threat to validity for a benchmark-eval paper."
     },
     {
       "flag": "Trial count discrepancy",
       "detail": "Section 4.3 describes a design with '9,000 trials' (500 problems × 6 ratios × 3 models) but Table 5 only reports 1,800 trials (300 per ratio). The discrepancy between designed and reported trials is unexplained—unclear whether only one model's results are shown or whether the design was modified."
     },
     {
       "flag": "Small TAAC evaluation set",
       "detail": "TAAC's main evaluation (Table 15) uses only 220 prompts (100 code, 100 CoT, 20 hybrid). This is quite small for validating an adaptive system, and the 'synthetic validation set' construction is not described."
     },
     {
       "flag": "No seed sensitivity or repeated runs",
       "detail": "All results appear to be from single experimental runs. No random seed analysis or repeated trials to assess result stability. Wilson CIs capture sampling uncertainty but not experimental variability."
     },
     {
       "flag": "Self-citing unpublished prior work as foundation",
       "detail": "The entire paper builds on 'Compress or Route?' (Johnson, 2026), a Zenodo publication by the same sole author. The foundational findings (r ≥ 0.6 threshold) cannot be independently verified from peer-reviewed sources."
     }
   ],
   "cited_papers": [
     {
       "title": "LLMLingua: Compressing Prompts for Accelerated Inference of Large Language Models",
       "authors": ["H. Jiang", "Q. Wu", "C.-Y. Lin", "Y. Yang", "L. Qiu"],
       "year": 2023,
       "relevance": "Core prompt compression method using perplexity-based token pruning; baseline algorithm in this paper's comparisons."
     },
     {
       "title": "LLMLingua-2: Data Distillation for Efficient and Faithful Task-Agnostic Prompt Compression",
       "authors": ["Z. Pan", "Q. Wu", "H. Jiang"],
       "year": 2024,
       "relevance": "Learned BERT-based compression method used as primary compression algorithm in experiments."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Foundational work on LLM cost reduction through cascading model strategies; related cost-optimization approach."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "LLM routing using preference data for cost-quality optimization; complementary approach to compression."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and pass@k metric for code generation evaluation; key benchmark used in this paper."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["J. Austin", "A. Odena", "M. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for Python code generation; primary validation benchmark in this paper."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Real-world code generation benchmark for GitHub issue resolution; referenced as more complex code evaluation."
     },
     {
       "title": "MultiPL-E: A Scalable and Polyglot Approach to Benchmarking Neural Code Generation",
       "authors": ["F. Cassano", "J. Gouwar", "D. Nguyen"],
       "year": 2023,
       "relevance": "Multi-language code generation benchmark; claimed as part of cross-language validation but results not shown."
     },
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["W. Kwon", "Z. Li", "S. Zhuang"],
       "year": 2023,
       "relevance": "Systems-level LLM serving optimization through memory management; complementary efficiency approach."
     },
     {
       "title": "FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness",
       "authors": ["T. Dao", "D. Fu", "S. Ermon", "A. Rudra", "C. Ré"],
       "year": 2022,
       "relevance": "Foundational work on efficient attention computation for transformer inference."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2022,
       "relevance": "Introduced chain-of-thought prompting; the CoT reasoning task type is a core experimental condition in this paper."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2024,
       "relevance": "Introduced HumanEval+ with additional test cases for rigorous code generation evaluation."
     }
   ]
 }