ai-research-survey

scan.json (24661B)

---

 {
   "paper": {
     "title": "The Price of Progress: Algorithmic Efficiency and the Falling Cost of AI Inference",
     "authors": ["Hans Gundlach", "Jayson Lynch", "Matthias Mertens", "Neil Thompson"],
     "year": 2025,
     "venue": "NeurIPS 2025 Workshop on Evaluating the Evolving LLM Lifecycle",
     "arxiv_id": "2511.23455"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states: 'Our dataset of benchmark prices, along with the code used for analysis, is available here: The Price of Progress Project: https://github.com/hansgundlach/Algorithmic Progress Inference'. A GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states that the dataset of benchmark prices is available at the linked GitHub repository. The dataset is described as 'the largest dataset of current and historical prices to run benchmarks to date.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of environment specifications, requirements.txt, library versions, or any dependency information in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper links a GitHub repository but does not include step-by-step reproduction instructions in the paper itself. There is no README description, no command examples, and no 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report 90% confidence intervals for all regression estimates. For example, GPQA-D Pareto Restricted: 5.315 [2.449, 11.534]. Figures also show confidence bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., open-weight vs. closed-weight trends differ, higher-performance bins decline faster) but does not report formal significance tests such as p-values or hypothesis tests for these comparisons. Only confidence intervals on individual regression coefficients are provided."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as annual reduction factors with baseline context throughout. For example, '5× to 10× per year' for frontier models, '3× per year' for algorithmic progress after hardware adjustment. Tables 1 and 2 provide precise factor estimates with confidence intervals."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are reported (e.g., n=53 for GPQA-D Pareto, n=13 for SWE-V Pareto) but there is no discussion of whether these are sufficient for the claims made or any power analysis. The paper acknowledges limited SWE-V data but does not formally justify sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports R-squared values and confidence intervals from regressions, but does not report variance or standard deviation across individual data points or model-specific variation in a systematic way. No spread measures (std dev, IQR) are reported for the price data itself."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly compares its estimates against prior work: Cottier et al. [2025] (10-1000x), Erol et al. [2025] (24.5x and 3.23x), and Saad-Falcon et al. [2025] (3.1x). Section 2.3 discusses differences from these baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All comparison works are from 2025: Cottier et al. [2025], Erol et al. [2025], and Saad-Falcon et al. [2025]. These represent the most recent estimates available."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies components of its analysis: all models vs. Pareto-restricted, all licenses vs. open-weight only, pooled regression vs. binned analysis, and hardware-adjusted vs. unadjusted estimates. These variations serve an ablation-like function showing how each analytical choice affects results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results across three benchmarks (GPQA-Diamond, OTIS Mock AIME, SWE-bench Verified) and reports both annual reduction factors and R-squared values."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is an econometric analysis of pricing data. Human evaluation of system outputs is not relevant to the claims about price trends."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning model evaluation. The paper fits regression models to observational pricing data; train/test splits are not applicable to this type of econometric analysis."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark (GPQA-D, AIME, SWE-V), by model type (all vs. open-weight), by restriction type (Pareto vs. all), and by performance bin (Figs. 2-3). Section 2.4 provides detailed per-bin analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where its approach yields weak or uncertain results: SWE-V has 'less data, and our estimates feature much larger confidence bands' (Section 2.3). It also notes the open-weight unrestricted GPQA-D estimate of 1.214x includes 1.0 in its CI, suggesting possibly no progress."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 reports that despite falling inference prices, 'benchmarking costs have stayed constant or increased' — a finding that runs counter to the main positive trend. The paper also reports that SWE-V results are too uncertain to draw strong conclusions."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 5-10x annual price reduction for frontier models (supported by Table 1), ~3x for algorithmic progress (supported by Table 2 hardware-adjusted estimates), and increasing benchmarking costs (supported by Section 3 and Figs. 4-5). All abstract claims are consistent with the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about decomposing price changes into 'economic forces, hardware efficiency improvements, and algorithmic efficiency improvements' (Abstract). The strategy of isolating open-weight models and dividing by hardware trends is described as yielding 'algorithmic progress' — a causal interpretation from observational data. However, this decomposition relies on assumptions (e.g., open-weight pricing reflects only technical factors) that are stated but not formally tested. Confounds such as changes in inference optimization software, quantization adoption, or batch scheduling are not controlled for."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('The Falling Cost of AI Inference') and abstract generalize broadly, but results are limited to April 2024 - November 2025, three specific benchmarks, and pricing from Artificial Analysis. The paper notes limitations for SWE-V but the title and abstract frame findings as general AI inference trends without explicitly bounding to the tested benchmarks and time period."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper considers several alternative explanations: market competition vs. technical progress (Section 2.2-2.3, motivating the open-weight analysis), differences from prior estimates attributed to token-level vs. benchmark-level measurement and time period differences (Section 2.3), and the 40-60% bin price drop attributed to 'increased market competition' rather than technical progress (Section 2.4)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses models generically (e.g., 'Claude 3.7 with different reasoning levels') but does not provide a full list of exact model versions used in the analysis. Individual model names and versions are not enumerated in the paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting. It is an econometric analysis of pricing data, not a study that prompts language models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Regression specifications are provided (Eq. 1) but key details like standard error computation method, exact software/packages used, and optimization settings are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is an econometric analysis paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B documents preprocessing decisions: excluding $0 cost models, handling cached tokens differently for GPQA-D vs. SWE-V, excluding models with ambiguous version matching, normalizing multi-run benchmarks by dividing tokens by number of runs, and excluding price increases from platforms dropping legacy models. Section 2.1 also describes the Pareto frontier filtering."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. Some limitations are scattered in the text (e.g., limited SWE-V data, time period constraints) but they are not consolidated into a substantive section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity section exists. While the paper notes some specific issues (limited SWE-V data, time period differences from prior work), these are not framed as threats to validity and many potential threats are not discussed (e.g., selection bias in Artificial Analysis coverage, Internet Archive data gaps, potential survivorship bias in pricing data)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It notes SWE-V data is limited and the time period is April 2024 - November 2025, but does not systematically bound the claims. The broad framing ('The Falling Cost of AI Inference') extends beyond what was tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper links to a GitHub repository (Appendix A) containing the dataset and analysis code, enabling independent verification of the underlying data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the data collection: 'We collect data on input and output token prices over time by gathering Internet Archive data from Artificial Analysis' covering April 2024 to October 2025, supplemented with Epoch AI benchmark data. The matching process between pricing and benchmark data is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public sources (Artificial Analysis, Epoch AI benchmark hub, Internet Archive). This is not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B documents the pipeline: collecting prices from Internet Archive snapshots of Artificial Analysis, matching with Epoch AI benchmark data, handling cached tokens, removing $0 prices, normalizing multi-run benchmarks, and filtering to Pareto frontier. Sample sizes are reported at each stage (e.g., 138 price data points with 93 unique models for GPQA-D)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists author affiliations (MIT CSAIL, MIT Sloan, MIT FutureTech) and acknowledges Zachary Brown for comments, but does not disclose any funding sources or grants."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: all four authors are affiliated with MIT CSAIL or MIT Sloan and MIT FutureTech. The paper does not evaluate any MIT product, so there is no obvious vendor-product conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on any benchmark. It analyzes pricing data for models that others have benchmarked. Contamination is not relevant."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. It is an econometric study of pricing trends."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. Benchmark contamination is not relevant to this pricing analysis."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is an econometric analysis of publicly available pricing and benchmark data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study with treatment assignment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not propose a method that performs inference. It is an econometric study about inference costs. The costs it reports are the subject of study, not the cost of its own method."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "The computational requirements of the regression analysis itself are negligible and not the subject of the paper. This is an econometric study, not a compute-intensive method."
       }
     }
   },
   "claims": [
     {
       "claim": "The price for a given level of benchmark performance has decreased by approximately 5x to 10x per year for frontier models on knowledge, reasoning, math, and software engineering benchmarks.",
       "evidence": "Table 1 shows annual reduction factors: GPQA-D Pareto 5.3x [2.4, 11.5], AIME Pareto 11.7x [5.3, 25.9], SWE-V Pareto 4.7x [0.7, 32.2]. Figure 1 visualizes these results.",
       "supported": "strong"
     },
     {
       "claim": "Algorithmic efficiency progress accounts for approximately 3x annual improvement after controlling for hardware price declines.",
       "evidence": "Table 2 shows hardware-adjusted reduction factors for open-weight Pareto models: GPQA-D 3.2x [1.5, 6.8], AIME 3.3x [1.4, 7.9]. This uses Rahman [2024]'s 30% annual hardware price improvement as the divisor.",
       "supported": "moderate"
     },
     {
       "claim": "Despite falling per-unit inference prices, benchmarking costs have remained flat or increased.",
       "evidence": "Figures 4 and 5 show benchmark evaluation prices over time for GPQA-D and SWE-V. Section 3 attributes this to 'demand for much higher quality models, which are much larger and use much longer reasoning traces.'",
       "supported": "moderate"
     },
     {
       "claim": "Higher-performance models show faster price declines than lower-performance models.",
       "evidence": "Section 2.4 and Figure 2 show that the highest GPQA-D bin declined by 31x/year vs. 1.7x/year for the lowest bin.",
       "supported": "moderate"
     },
     {
       "claim": "Overall price-performance progress is closer to 10x than 1,000x per year, lower than Cottier et al. [2025] estimates.",
       "evidence": "Section 2.3 explains three factors for lower estimates: benchmark-level rather than token-level measurement, different time period (April 2024 vs. 2022/2023 start), and larger model sample. Table 1 confirms most estimates are in the 5-12x range.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["observational"],
   "key_findings": "AI inference prices for a given performance level are declining by approximately 5-10x per year for frontier models across knowledge, reasoning, math, and coding benchmarks. After isolating open-weight models and adjusting for hardware price improvements, algorithmic progress contributes roughly 3x annual improvement. Higher-capability models show faster price declines (up to 31x/year) than lower-capability models (1.7x/year). Despite these per-unit price declines, total benchmarking costs have stayed flat or increased because frontier performance demands larger models with longer reasoning traces.",
   "red_flags": [
     {
       "flag": "Very short time window",
       "detail": "The analysis covers only April 2024 to November 2025 (~19 months), which is a narrow window for estimating annual trends. Extrapolating exponential trends from such a short period carries substantial uncertainty, particularly for SWE-V which has only 13 Pareto-restricted data points."
     },
     {
       "flag": "No formal limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Issues like Internet Archive data gaps, selection bias in Artificial Analysis coverage, survivorship bias (dropped models/providers), and the strong assumptions in the hardware decomposition are not systematically addressed."
     },
     {
       "flag": "Causal decomposition from observational data",
       "detail": "The claim that algorithmic progress accounts for ~3x improvement relies on dividing open-weight model price trends by hardware price trends. This assumes these two factors are the only drivers and are multiplicatively separable. Confounds such as inference software optimization (vLLM, TensorRT), quantization adoption, batch scheduling improvements, and provider-level efficiency gains are not controlled for."
     },
     {
       "flag": "Broad title relative to evidence",
       "detail": "The title 'The Price of Progress: Algorithmic Efficiency and the Falling Cost of AI Inference' suggests general findings, but results are limited to three benchmarks from a single pricing source over 19 months. SWE-bench Verified results have confidence intervals consistent with no progress at all."
     }
   ],
   "cited_papers": [
     {
       "title": "LLM inference prices have fallen rapidly but unequally across tasks",
       "authors": ["Ben Cottier", "Ben Snodin", "David Owen", "Tom Adamczewski"],
       "year": 2025,
       "relevance": "Directly comparable study of LLM inference price trends, finding 10-1000x annual declines depending on performance level."
     },
     {
       "title": "Cost-of-pass: An economic framework for evaluating language models",
       "authors": ["Mehmet Hamza Erol", "Batu El", "Mirac Suzgun", "Mert Yuksekgonul", "James Zou"],
       "year": 2025,
       "arxiv_id": "2504.13359",
       "relevance": "Proposes cost-of-pass metric for benchmarks and finds 3-25x annual declines on math benchmarks, directly compared in this paper."
     },
     {
       "title": "Algorithmic progress in language models",
       "authors": ["Anson Ho", "Tamay Besiroglu", "Ege Erdil", "David Owen", "Robi Rahman", "Zifan C Guo", "David Atkinson", "Neil Thompson", "Jaime Sevilla"],
       "year": 2024,
       "relevance": "Foundational work on measuring algorithmic efficiency gains in language models, defining the concept this paper extends to inference pricing."
     },
     {
       "title": "Intelligence per watt: Measuring intelligence efficiency of local AI",
       "authors": ["Jon Saad-Falcon", "Avanika Narayan", "Hakki Orhun Akengin"],
       "year": 2025,
       "arxiv_id": "2511.07885",
       "relevance": "Measures energy efficiency gains in AI models (3.1x from 2023-2025), used as a comparison point for algorithmic progress estimates."
     },
     {
       "title": "Inference economics of language models",
       "authors": ["Ege Erdil"],
       "year": 2025,
       "relevance": "Analyzes the price/latency tradeoff in LLM inference, providing economic framework relevant to understanding inference cost dynamics."
     },
     {
       "title": "How predictable is language model benchmark performance?",
       "authors": ["David Owen"],
       "year": 2024,
       "arxiv_id": "2401.04757",
       "relevance": "Studies predictability of benchmark performance as function of training compute, providing theoretical basis for the logistic performance model used in this paper."
     },
     {
       "title": "Observational scaling laws and the predictability of language model performance",
       "authors": ["Yangjun Ruan", "Chris J Maddison", "Tatsunori B Hashimoto"],
       "year": 2024,
       "relevance": "Establishes that benchmark performance increases logistically with training compute, supporting the logit transformation used in this paper's regression."
     },
     {
       "title": "Demand for LLMs: Descriptive Evidence on Substitution, Market Expansion, and Multihoming",
       "authors": ["Andrey Fradkin"],
       "year": 2025,
       "arxiv_id": "2504.15440",
       "relevance": "Provides economic analysis of LLM market dynamics and pricing behavior, corroborating that model prices rarely change over time."
     },
     {
       "title": "Infinity Bench: Extending long context evaluation beyond 100k tokens",
       "authors": ["Xinrong Zhang", "Yingfa Chen", "Shengding Hu"],
       "year": 2024,
       "arxiv_id": "2402.13718",
       "relevance": "Example of expensive benchmarking ($5,000 for GPT-4 evaluation), illustrating the rising cost of AI evaluation discussed in this paper."
     }
   ]
 }