ai-research-survey

scan.json (25395B)

---

 {
   "paper": {
     "title": "CodePDE: An Inference Framework for LLM-driven PDE Solver Generation",
     "authors": [
       "Shanda Li",
       "Tanya Marwah",
       "Junhong Shen",
       "Weiwei Sun",
       "Andrej Risteski",
       "Yiming Yang",
       "Ameet Talwalkar"
     ],
     "year": 2026,
     "venue": "Transactions on Machine Learning Research",
     "arxiv_id": "2505.08783"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link (https://github.com/LithiumDA/CodePDE) in Section 3 and the abstract. Evaluation data is released on HuggingFace (https://huggingface.co/datasets/LDA1020/codepde-data/tree/main)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Datasets are drawn from PDEBench (Takamoto et al., 2022) and FNO paper (Li et al., 2022b), both with MIT license. The evaluation data is also released on HuggingFace."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the framework is 'implemented in Python' and that evaluation uses an NVIDIA GeForce RTX 2080 Ti GPU (Appendix B.3), but no requirements.txt, Dockerfile, or detailed dependency specifications are provided in the paper itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released on GitHub, the paper itself does not contain a 'Reproducing Results' section or specific commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 1, 3, and 4 report only point estimates of nRMSE. No confidence intervals or error bars are provided for the primary accuracy metrics. The test-time scaling curves in Figure 3 appear smoothed but do not show uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'CodePDE with refinement outperforms the hand-crafted reference solvers on 4 out of 5 evaluated tasks') but provides no statistical significance tests. Differences are assessed by direct numerical comparison only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context, e.g., 'Gemini 2.0 Flash achieves an nRMSE of 1.06×10^-4 via refinement, compared to the reference's 3.55×10^-4, representing a 70% improvement in accuracy' (Section 5.1). Bug-free rates are given with baseline context (41% to 84%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 100 instances per PDE family and 32 i.i.d. samples per model per PDE for code generation (Appendix B.4), but provides no justification for why these numbers are sufficient or any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as best-of-n samples (best of 32 or best of 12) rather than as averages with standard deviations. No variance or standard deviation across runs is reported for the main nRMSE results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against reference numerical solvers, PDE solving softwares (Dedalus, PyPDE), neural network baselines (U-Net, FNO, PINN, ORCA, PDEformer, UPS), and agentic workflows (FunSearch, AIDE). See Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models and methods: UPS (2024b), ORCA (2023), PDEformer (2024), FunSearch (2024), and AIDE (2025). The LLM baselines span 16 models including the latest reasoning models (o3, DeepSeek-R1, Gemini 2.5 Pro)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents an ablation study removing components progressively: CodePDE full → w/o refine → w/o refine+scale → w/o refine+scale+debug. Each component's contribution is quantified."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses four metrics: nRMSE (primary accuracy), debug success rate, convergence rate (with order classification), and execution time. See Section 4 (Evaluation Metrics)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not applicable here. The paper evaluates automatically generated PDE solvers against ground truth numerical solutions. The quality of solvers is objectively measurable via nRMSE against reference solutions."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 explicitly states: 'We randomly sample 100 instances for each family for testing. We also sample another 50 instances as the development set, which is used to provide the execution feedback to LLMs without leaking the true test data.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by PDE family (Advection, Burgers, Reaction-Diffusion, CNS, Darcy) and by model in Tables 1, 3, 4. Bug-free rates are broken down per model and per PDE in Figure 2 and Tables 5-6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.7 provides detailed failure case analysis on the Reaction-Diffusion Equation, explaining why LLM-generated solvers fail (they use finite-difference for the reaction term instead of the analytical solution). Table 9 and Appendix D provide code-level comparisons."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that all LLMs fail to match the reference solver on Reaction-Diffusion (Section 5.1, 5.7). It also notes that reasoning models are 'not necessarily better than standard ones in the refinement stage' (Section 5.3), an unexpected negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims LLMs 'can achieve strong performance across a range of representative PDE problems' with advanced inference strategies. Table 1 supports this, showing CodePDE outperforms reference solvers on 4/5 tasks. The abstract also mentions 'trade-offs between solver reliability and sophistication' which is supported by the convergence analysis in Figure 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (Table 2), which use controlled single-variable manipulation. Removing components individually (refinement, scaling, debugging) demonstrates their causal contribution to performance. The ablation design is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'LLM-driven PDE Solver Generation' and abstract claims about 'PDE solving' are broader than what is tested. The paper evaluates only 5 PDE families (all standard benchmarks), but claims like 'LLMs can generate effective and efficient solver code for PDEs' are stated broadly. The conclusions refer to 'scientific computing' generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, the best-of-n selection strategy could mean the improvements are driven by selection pressure rather than LLM capability. The role of training data contamination (LLMs may have seen PDE solver code during training) is not discussed as an alternative explanation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to models by marketing names: 'GPT-4o', 'Claude-3.7-sonnet', 'Gemini 2.5 Pro', 'o3', etc. No API version, snapshot date, or specific model ID (e.g., 'gpt-4o-2024-05-13') is provided. Appendix Table 3 references models by the same marketing names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix B: the system prompt (B.2), PDE descriptions used as task specifications (B.1), debugging prompt template with {code_output} and {error_message} placeholders, and refinement prompt template. The actual PDE descriptions that fill the templates are fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.4 reports decoding temperature (0.7 for most models, 1.0 for OpenAI o-series), sample count (32 for generation, 12 for refinement), debug iteration limit (4), and FunSearch/AIDE-specific parameters (island count, reset period, search steps)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The CodePDE framework is described in detail in Section 3 with a 5-step pipeline: task specification, code generation, debugging (with iterative error feedback), evaluation, and solver refinement. The agentic workflow is fully described with prompts in Appendix B."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 documents data handling: 100 instances sampled per PDE family for testing, 50 for development. Data sources (PDEBench, FNO paper) are specified with licenses. The separation between dev and test sets is clearly documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The Conclusions section (Section 6) is brief and does not discuss limitations substantively."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address issues like: whether LLMs may have seen PDE solver code in training data, whether the 5 chosen PDEs are representative, or whether best-of-n selection inflates apparent capability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to the specific PDEs tested, the specific hardware used, or the particular model versions evaluated. The conclusions make broad claims about 'LLMs and agents for scientific computing' without scoping."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation data is released on HuggingFace (https://huggingface.co/datasets/LDA1020/codepde-data/tree/main) and code on GitHub. The underlying PDE benchmark datasets (PDEBench, FNO) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Appendix B.3 describe data collection: datasets drawn from PDEBench and FNO paper (both publicly available with MIT license), 100 instances randomly sampled per family for testing, 50 for development."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is standard public benchmarks (PDEBench, FNO)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: PDE task specification → LLM code generation (32 samples) → debugging (up to 4 rounds) → evaluation against reference solutions → optional refinement (12 samples from top 5 seeds). Appendix B.3-B.4 provide full details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was supported in part by the National Science Foundation grants IIS1705121, IIS1838017, IIS2046613, IIS2112471, and funding from Datadog.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Carnegie Mellon University, Flatiron Institute, Polymathic AI, and Datadog (for Ameet Talwalkar). The paper evaluates multiple LLMs from various providers without the authors being affiliated with those providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from NSF grants and Datadog. Neither funder has a direct stake in whether LLMs can generate PDE solvers. The paper evaluates multiple third-party LLMs without favoring any particular provider."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Ameet Talwalkar is affiliated with Datadog, which provides funding, but no competing interests statement is included."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 16 LLMs on code generation benchmarks but does not state the training data cutoff for any of the models. This is important because LLMs may have seen PDE solver code (including from PDEBench implementations) during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs may have seen PDE solver implementations from PDEBench or similar public codebases during training. This is a significant concern since PDEBench code is publicly available on GitHub."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PDEBench was published in 2022 and its code is publicly available. All evaluated models were trained after 2022 and likely had access to PDE solver implementations. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.7 reports token counts across PDE benchmarks for generation, debugging, and refinement stages. It states 'solving a single PDE using the full CodePDE framework generally incurs a total cost of less than $5 USD when using API-based services.' Execution times are also reported in Appendix A.5."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports hardware (NVIDIA GeForce RTX 2080 Ti GPU, 11GB memory) in Appendix B.3, execution time analysis in Appendix A.5 (Figure 11), and token usage analysis in Appendix A.7 (Figures 14-16). Runtime limits (1200s and 2400s) are specified."
       }
     }
   },
   "claims": [
     {
       "claim": "CodePDE with refinement outperforms hand-crafted reference solvers on 4 out of 5 PDE families.",
       "evidence": "Table 1 shows that after refinement, LLM-generated solvers achieve lower nRMSE than reference solvers on Advection, Burgers, CNS, and Darcy Flow. Only Reaction-Diffusion remains worse.",
       "supported": "strong"
     },
     {
       "claim": "Self-debugging raises the average bug-free rate from 41% to 84% across all PDEs.",
       "evidence": "Figure 2 and Tables 5-6 in the appendix provide detailed bug-free rates before and after debugging for each model and PDE family. The average across 11 LLMs goes from approximately 41% to 84%.",
       "supported": "strong"
     },
     {
       "claim": "Test-time scaling (best-of-n) improves solution quality, with most significant gains between n=4 and n=16.",
       "evidence": "Figure 3 shows smoothed test-time scaling curves for representative models across PDE families. The curves show diminishing returns after n=16.",
       "supported": "moderate"
     },
     {
       "claim": "Advanced reasoning models excel at generation but are not necessarily better at refinement than standard models.",
       "evidence": "Table 1 shows o3 and DeepSeek-R1 lead in 'Reasoning + Debugging' but GPT-4o and DeepSeek-V3 are competitive or better in 'Reasoning + Debugging + Refinement'. Section 5.3 discusses this finding.",
       "supported": "moderate"
     },
     {
       "claim": "The combination of debugging, refinement, and test-time scaling is essential; without them, naive prompting yields nRMSE of 1.49×10^-1.",
       "evidence": "Table 2 ablation study with Claude-3.7-sonnet shows progressive degradation: full CodePDE (4.44×10^-3) → w/o refine (5.15×10^-3) → w/o refine+scale (8.68×10^-2) → w/o all (1.49×10^-1).",
       "supported": "strong"
     },
     {
       "claim": "LLMs consistently fail on Reaction-Diffusion because they use finite-difference for the reaction term instead of the analytical solution.",
       "evidence": "Section 5.7 and Table 9 in Appendix D compare LLM-generated solvers against the reference, showing LLMs discretize the reaction term while the reference uses the analytical solution.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodePDE demonstrates that LLMs paired with structured inference frameworks (debugging, refinement, test-time scaling) can generate PDE solvers competitive with hand-crafted numerical implementations on 4 of 5 benchmark PDE families. Self-debugging raises bug-free rates from 41% to 84%, and best-of-n sampling shows clear test-time scaling benefits. The paper identifies a reliability-sophistication trade-off where some models favor simple robust methods while others explore diverse higher-order schemes, and reveals that code generation and code refinement may be distinct LLM skills.",
   "red_flags": [
     {
       "flag": "Best-of-n selection inflates reported performance",
       "detail": "Main results use best-of-32 (generation) and best-of-12 (refinement) selection. This reports the performance ceiling rather than expected performance. No average or median nRMSE is reported, making it difficult to assess typical performance a practitioner would experience."
     },
     {
       "flag": "No uncertainty quantification on main results",
       "detail": "All nRMSE values in Tables 1, 3, 4 are point estimates without confidence intervals, standard deviations, or error bars. With stochastic sampling (temperature > 0), results will vary across runs, but this variance is never reported."
     },
     {
       "flag": "Training data contamination not addressed",
       "detail": "PDEBench code and PDE solver implementations are publicly available on GitHub. All 16 evaluated LLMs were trained after PDEBench's 2022 release. The LLMs may have memorized PDE solver patterns from training data, which would inflate performance relative to truly novel scientific problems."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a limitations or threats-to-validity section. Significant concerns (contamination, generalization to harder PDEs, best-of-n inflation, cost at scale) are not discussed."
     },
     {
       "flag": "Overgeneralized conclusions",
       "detail": "The conclusions claim insights for 'developing more capable LLMs and agents for scientific computing' based on only 5 standard PDE families. These are well-studied equations with known numerical methods; the results may not extend to novel or complex multi-physics problems."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Key inference-time technique for iterative code improvement used in CodePDE's refinement stage."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2024,
       "relevance": "Foundational work on LLM self-debugging capability, a core component of CodePDE's debugging pipeline."
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Parameters for Reasoning",
       "authors": ["Charlie Victor Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2025,
       "relevance": "Motivates the test-time scaling strategy (best-of-n sampling) used in CodePDE's evaluation framework."
     },
     {
       "title": "Mathematical Discoveries from Program Search with Large Language Models",
       "authors": ["Bernardino Romera-Paredes"],
       "year": 2024,
       "relevance": "FunSearch is used as a baseline agentic workflow for comparison with CodePDE."
     },
     {
       "title": "AIDE: AI-Driven Exploration in the Space of Code",
       "authors": ["Zhengyao Jiang", "Dominik Schmidt"],
       "year": 2025,
       "relevance": "AIDE is used as a baseline agentic workflow featuring tree search for code generation comparison."
     },
     {
       "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2025,
       "relevance": "Benchmark for evaluating LLM agents on ML engineering tasks, related to code generation capability evaluation."
     },
     {
       "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich"],
       "year": 2024,
       "relevance": "Prior work on repeated sampling as a test-time compute scaling strategy for LLMs."
     },
     {
       "title": "From Decoding to Meta-Generation: Inference-time Algorithms for Large Language Models",
       "authors": ["Sean Welleck", "Amanda Bertsch", "Matthew Finlayson"],
       "year": 2024,
       "relevance": "Survey of inference-time algorithms for LLMs that motivates CodePDE's framework design."
     },
     {
       "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces",
       "authors": ["Mike A Merrill", "Alexander Glenn Shaw", "Nicholas Carlini"],
       "year": 2026,
       "relevance": "CodePDE has been integrated into Terminal-Bench via Harbor Adapters, demonstrating practical deployment of the framework."
     },
     {
       "title": "PDEBench: An Extensive Benchmark for Scientific Machine Learning",
       "authors": ["Makoto Takamoto", "Timothy Praditia"],
       "year": 2022,
       "relevance": "Primary benchmark dataset source for CodePDE's evaluation; defines nRMSE metric used throughout."
     },
     {
       "title": "Inference Scaling Laws: An Empirical Analysis of Compute-Optimal Inference for LLM Problem-Solving",
       "authors": ["Yangzhen Wu", "Zhiqing Sun", "Shanda Li"],
       "year": 2025,
       "relevance": "Empirical analysis of inference compute scaling that informs CodePDE's test-time scaling strategy."
     }
   ]
 }