ai-research-survey

scan.json (34431B)

---

 {
   "paper": {
     "title": "Planning In Natural Language Improves LLM Search For Code Generation",
     "authors": [
       "Evan Wang",
       "Federico Cassano",
       "Catherine Wu",
       "Yunfeng Bai",
       "Will Song",
       "Vaskar Nath",
       "Ziwen Han",
       "Sean Hendryx",
       "Summer Yue",
       "Hugh Zhang"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2409.03733",
     "doi": "10.48550/arXiv.2409.03733"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code can be found at https://github.com/scaleapi/plansearch' — a specific GitHub repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HumanEval+, MBPP+, and LiveCodeBench, all of which are publicly accessible. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed dependency list is provided in the paper. Only model names and API parameters are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. While code is released, the paper itself does not contain a 'Reproducing Results' section or specific commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 and figures are reported as point estimates (e.g., 'pass@200 of 77.0%') with no confidence intervals or error bars. The pass@k curves also lack uncertainty bounds."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PLANSEARCH outperforms baselines across all models and benchmarks but never uses statistical significance tests. Comparisons are based solely on comparing raw pass@k numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: 'pass@200 of 77.0% on LiveCodeBench, outperforming both the best pass-rate achieved without any search (pass@1 = 41.4%) and using standard repeated sampling (pass@200 = 60.6%).' Appendix C reports relative gains explicitly."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of problems used from each benchmark or the choice of k=200 as the maximum number of completions. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any results. The paper uses a single run per model/method configuration. Appendix O acknowledges PLANSEARCH generations may not be independent but provides no variance data."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Repeated Sampling, IdeaSearch (their intermediate method), Chain-of-Thought (Appendix E), and o1-mini (a search-augmented model). Multiple baseline comparisons are provided."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models: Claude 3.5 Sonnet, GPT-4o, GPT-4o-mini, DeepSeek-Coder-V2, and o1-mini — all state-of-the-art at time of writing (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H provides ablations on maximum subset size S (Figure 28), observation tree depth L (Figure 29), and the observation-to-code translation pipeline (Figure 30), all using GPT-4o-mini on LiveCodeBench."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1, pass@k curves for various k, pass@200, diversity scores (Section 6.1), and compute-normalized comparisons (Appendix D). Results are also reported across three benchmarks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is entirely automated through test-suite execution (pass/fail on test cases). No human evaluation of code quality, readability, or correctness of plans is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench uses a temporal split — only problems from May–September 2024 are used, after Claude 3.5 Sonnet's April 2024 training cutoff. HumanEval+ and MBPP+ are standard held-out benchmarks with hidden test cases."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (4 models + o1-mini), per method (Repeated Sampling, IdeaSearch, PLANSEARCH), and per benchmark (HumanEval+, MBPP+, LiveCodeBench) in Table 1 and Appendix A."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses that 'PLANSEARCH often hurts pass@1 for several models, including most notably Sonnet 3.5 on LiveCodeBench.' Section 7 discusses limitations of the approach and when it underperforms."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: PLANSEARCH hurts pass@1 (Section 6), PLANSEARCH does not significantly outperform until k≥4 (Section 7), and gains over o1-mini are marginal (Appendix K). Appendix H shows ablations where removing components had minimal effect."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about pass@200 of 77.0% on LiveCodeBench are confirmed in Table 1. Claims about outperforming baselines and diversity correlation are supported by Figure 6 and the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims PLANSEARCH improves performance through increased diversity. Ablation studies (Appendix H) provide controlled single-variable manipulations. The backtranslation experiment (Section 3.2) and conditioning experiment (Section 3.3) provide additional causal support for the role of idea diversity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'LLM Search For Code Generation' broadly, but the paper only evaluates on competitive programming benchmarks (HumanEval+, MBPP+, LiveCodeBench) involving self-contained function-level problems. Real-world software engineering tasks, multi-file projects, and non-Python languages are untested. The paper does not bound its claims to competitive programming."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 investigates whether diversity explains the performance gains (rather than just generating more samples). Appendix D provides compute-normalized comparisons to control for the additional compute used by PLANSEARCH. Appendix O discusses the non-independence issue with the estimator."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements — it measures pass@k on specific benchmarks and claims improvement in pass@k. The diversity metric is explicitly labeled as a proxy and its correlation with performance is measured, not assumed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names without specific API versions or snapshot dates: 'Claude 3.5 Sonnet', 'GPT-4o-mini', 'GPT-4o', 'DeepSeek-Coder-V2', 'o1-mini'. Per schema criteria, marketing names like 'GPT-4o' without a snapshot date do not count as specified versions. Model behavior changes across API updates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix M (Sections M.1–M.4), including backtranslation prompts, repeated sampling prompts, IdeaSearch prompts, and all PLANSEARCH prompts (observation generation, combining observations, etc.)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2: 'All models are run with temperature 0.9 and top-p of 0.95. (o1-mini was run with temperature 1.0 and top-p of 1.0 because of API constraints.)' Appendix F shows the temperature sweep used to select this value."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The PLANSEARCH pipeline is described in detail in Section 4.3: observation generation (4.3.1), deriving new observations via combinatorial subsets (4.3.2), translation to ideas then pseudocode then code (4.3.3). Figure 2 provides a visual diagram of the full pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 documents formatting requirements, code extraction, and test execution. Section 5.1 describes the temporal filtering of LiveCodeBench (May–September 2024). Section 3.3 describes the filtering criteria for the conditioning experiment (removing 0% and 100% solve-rate problems)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7, titled 'Limitations and Future Work,' provides substantive discussion of limitations including pass@1 degradation, compute costs, need for verifiers, and domains outside code generation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: PLANSEARCH hurts pass@1, doesn't outperform until k≥4, requires a verifier for filtering, and may have diminishing returns when stacked on search models like o1. Appendix O discusses the biased estimator concern for non-independent samples."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly enumerate what the results do NOT show. While Section 7 mentions 'domains outside of code generation that are out of scope,' it does not state that results are limited to competitive programming (not general software engineering), Python only, or API-accessible models only. The title and abstract frame claims broadly as 'LLM Search For Code Generation.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (individual model outputs, per-problem pass/fail results, generated codes) is made available. Only aggregated pass@k statistics are reported in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes the data collection procedure: models are sampled at specified temperatures, code is extracted from outputs, and run against test suites. The number of samples and benchmarks used are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. All data comes from standard public benchmarks (HumanEval+, MBPP+, LiveCodeBench) and LLM API outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: generate n completions per model/method → extract code from model output → run against all test cases → compute pass@k using unbiased estimator (Equation 4). For filtering experiments, the additional public-test-filtering step is described in Section 5.4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The Acknowledgements section (Section 8) thanks individuals for comments and discussion but does not mention any grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Scale AI, California Institute of Technology, Northeastern University, and Anysphere. Two authors are noted as having conducted work while at Scale AI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Most authors are affiliated with Scale AI, a company that sells AI data and evaluation services. Scale AI has a commercial interest in demonstrating that inference-time compute scaling (which could drive demand for their services) is effective. No explicit funding disclosure allows independence assessment."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included. Several authors are from Scale AI (a well-funded AI company) and Anysphere (an AI startup), but no patent, equity, or financial interest disclosures are provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1: 'Claude 3.5 Sonnet has a knowledge cutoff of April 2024.' The paper uses May 2024 as the cutoff date for LiveCodeBench problems to avoid contamination."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly addresses this: 'LiveCodeBench differentiates itself from other benchmarks by taking care to segregate problems by date to avoid data contamination concerns. For this paper, we use only the subset of problems between May 2024 and September 2024.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 addresses contamination by using only post-cutoff LiveCodeBench problems and noting that 'coding data is often highly upsampled during pre-training.' The temporal split specifically mitigates contamination risk for LiveCodeBench. HumanEval+ and MBPP+ contamination is not directly addressed but the paper relies on LiveCodeBench as its primary contamination-free evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All experiments are automated benchmark evaluations of LLM code generation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates LLMs on code benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D reports '244 generated tokens per completion per problem for Repeated Sampling, and 1,428 generated tokens per completion per problem for PLANSEARCH.' Figure 18 provides a compute-normalized comparison. However, dollar costs are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper mentions 'prohibitively high costs' for o1-mini evaluation but does not quantify total API spend, GPU hours, or total tokens consumed across all experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results are reported across multiple random seeds. Each model/method configuration appears to be run once, with the stochasticity coming from sampling temperature rather than seed variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states that Repeated Sampling and IdeaSearch generate exactly n codes, while PLANSEARCH generates 300–400 codes per problem. The pass@k estimator uses these sample counts explicitly."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F describes a temperature sweep 'over temperature increments of 0.1 from 0.0 to 1.2, inclusive, with top-p of 0.95, on Repeated Sampling and IdeaSearch.' This gives 13 configurations per method. Appendix H provides ablation budgets for S and L parameters."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Temperature selection is justified via the sweep in Appendix F. Ablation results in Appendix H justify the choices of S=2 and L=2 by showing performance across alternatives."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Scale AI authors implement and evaluate their own PLANSEARCH method against their own implementations of baselines (Repeated Sampling, IdeaSearch). No acknowledgment of potential author-evaluation bias, no independent evaluation, and no comparison against third-party implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D / Figure 18 provides an explicit compute-normalized comparison showing 'average tokens used per problem' on the x-axis and solve-rate on the y-axis for Repeated Sampling vs. PLANSEARCH across all models."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 discusses why coding is a powerful domain for search (verifiable via test execution) and Section 5.1 discusses the properties of each benchmark. The paper acknowledges saturation effects on HumanEval+ and MBPP+ and uses LiveCodeBench as the primary evaluation for this reason."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is structured so that the same model is used across all search methods (Repeated Sampling, IdeaSearch, PLANSEARCH). The search method IS the variable being tested, and the model is controlled. Results in Table 1 show per-model comparisons within the same scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1: LiveCodeBench problems are restricted to May–September 2024, after Claude 3.5 Sonnet's April 2024 training cutoff, explicitly to prevent temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses public test filtering (Section 5.4) as a feature to improve performance but does not discuss whether public tests provide hints that leak information about the hidden test structure or expected solution approach."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities, come from similar sources, or have near-duplicate relationships. HumanEval+ and MBPP+ problem independence is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splits are used as a concrete leakage prevention method: LiveCodeBench problems are filtered to only include those created after May 2024, and the paper explicitly states this is to avoid contamination."
       }
     }
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "PLANSEARCH achieves pass@200 of 77.0% on LiveCodeBench with Claude 3.5 Sonnet, nearly double the best pass@1 (41.4%) and outperforming standard repeated sampling pass@200 (60.6%).",
       "evidence": "Table 1 and Figure 1 show pass@200 = 77.0% for Claude 3.5 Sonnet with PLANSEARCH on LiveCodeBench. Repeated Sampling pass@200 is 55.6% for the same model, and the best non-search pass@1 across all models is 41.4%.",
       "supported": "strong"
     },
     {
       "claim": "Searching over plans in natural language produces significantly more diverse code outputs than baseline search methods.",
       "evidence": "Section 6.1 and Figure 6 show a strong positive correlation between idea diversity scores (measured via LLM-as-judge pairwise comparison) and relative performance gains from pass@1 to pass@200 across all models and methods.",
       "supported": "moderate"
     },
     {
       "claim": "Most variance in solving a problem is explained by whether the solution sketch is correct, not implementation details.",
       "evidence": "Figure 4b shows that when conditioning on a specific sketch, solve rates polarize toward 0% or 100%, suggesting that the sketch quality dominates implementation variance. The backtranslation experiment (Figure 4a) shows even 10 tokens of a correct sketch significantly boosts accuracy.",
       "supported": "moderate"
     },
     {
       "claim": "PLANSEARCH outperforms all baselines across all models and benchmarks considered.",
       "evidence": "Table 1 shows PLANSEARCH achieves the highest pass@200 for every model-benchmark combination: HumanEval+ (up to 99.5%), MBPP+ (up to 93.7%), LiveCodeBench (up to 77.0%). Figures 7–9 show full pass@k curves.",
       "supported": "strong"
     },
     {
       "claim": "Instruction-tuned models can have less diversity than their base model counterparts, with base models sometimes exceeding instruct models at high k.",
       "evidence": "Figure 3 shows DeepSeek-Coder-V2-Lite-Base surpassing its instruct counterpart at high k on MBPP+. Appendix I (Figure 31) provides extended comparisons across multiple model families. However, this trend is 'not all models and benchmarks' per the authors.",
       "supported": "moderate"
     },
     {
       "claim": "Running PLANSEARCH on a small model (GPT-4o-mini) outperforms larger models without search after merely 4 attempts.",
       "evidence": "This claim is stated in the introduction. Supporting evidence in Appendix A figures shows GPT-4o-mini with PLANSEARCH crossing larger models' pass@1 at low k, though 'merely 4' is not precisely demonstrated in the main text.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PLANSEARCH, a search algorithm that explores plans in natural language rather than directly searching over code, achieves pass@200 of 77.0% on LiveCodeBench with Claude 3.5 Sonnet — nearly double the best single-sample performance (41.4%). The paper demonstrates that idea diversity, measured via LLM-as-judge pairwise comparisons, is highly predictive of search performance gains across all models and methods. Backtranslation experiments show that even a 10-token correct solution sketch significantly boosts accuracy, and conditioning on specific sketches polarizes solve rates toward 0% or 100%, suggesting solution planning is the critical bottleneck rather than code implementation.",
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All main results (Table 1, Figures 1, 5) are reported as point estimates without confidence intervals, error bars, or variance across runs. For a paper making strong comparative claims across methods, the absence of any uncertainty quantification makes it impossible to assess whether observed differences are statistically meaningful."
     },
     {
       "flag": "Company evaluating its own method",
       "detail": "Most authors are affiliated with Scale AI, which has commercial interests in AI evaluation and inference-time compute scaling. The paper evaluates their own PLANSEARCH method against their own implementations of baselines. No independent evaluation or third-party replication is provided."
     },
     {
       "flag": "Variable sample counts across methods",
       "detail": "PLANSEARCH generates a variable number of codes (300–400) per problem, while Repeated Sampling and IdeaSearch generate exactly n codes. The unbiased pass@k estimator assumes remaining generations did not pass when k > n, but the authors acknowledge in Appendix O that PLANSEARCH's non-independent samples may bias the estimator."
     },
     {
       "flag": "LLM-as-judge circular dependency for diversity measurement",
       "detail": "The diversity metric (Section 6.1) uses GPT-4o-mini to judge pairwise similarity of code solutions. Using an LLM to measure diversity of LLM outputs introduces a circular dependency — the judge model's own biases may systematically mischaracterize diversity in ways that align with the paper's hypothesis."
     },
     {
       "flag": "Contamination risk unaddressed for HumanEval+ and MBPP+",
       "detail": "While LiveCodeBench uses temporal splits, the paper's results on HumanEval+ and MBPP+ (published 2021/2023) are reported without addressing contamination risk from models trained after these benchmarks were released. The paper notes saturation but does not discuss whether inflated scores result from data leakage."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language monkeys: Scaling inference compute with repeated sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich", "Ronald Clark", "Quoc V. Le", "Christopher Ré", "Azalia Mirhoseini"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Directly relevant as a study on scaling inference-time compute for LLM code generation through repeated sampling."
     },
     {
       "title": "Scaling llm test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Foundational work on test-time compute scaling laws for LLMs, directly motivating the inference-time search paradigm evaluated in this paper."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao", "Izhak Shafran", "Thomas L. Griffiths", "Yuan Cao", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "A key related approach to search-based LLM reasoning that PLANSEARCH builds upon and differentiates from."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Brian Ichter", "Fei Xia", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique that PLANSEARCH extends by explicitly searching over reasoning plans rather than using a single chain-of-thought."
     },
     {
       "title": "Livecodebench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu", "Wen-Ding Li", "Fanjia Yan", "Tianjun Zhang", "Sida Wang", "Armando Solar-Lezama", "Koushik Sen", "Ion Stoica"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Primary contamination-free benchmark used to evaluate PLANSEARCH, designed with temporal splits to avoid data leakage."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and the pass@k metric used throughout this paper's evaluation."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Created HumanEval+ and MBPP+ with additional test cases for more rigorous evaluation, both used as benchmarks in this paper."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung", "Nate Kushman"],
       "year": 2022,
       "relevance": "Pioneered large-scale sampling for competitive programming code generation, a direct precursor to the search approaches evaluated here."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Edward Berman", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Related agentic approach using verbal feedback to improve LLM performance, part of the broader inference-time compute scaling landscape."
     },
     {
       "title": "STaR: Bootstrapping Reasoning With Reasoning",
       "authors": ["Eric Zelikman", "Yuhuai Wu", "Jesse Mu", "Noah D. Goodman"],
       "year": 2022,
       "arxiv_id": "2203.14465",
       "relevance": "Training-time approach to improving reasoning that contrasts with PLANSEARCH's inference-time approach to diversity."
     },
     {
       "title": "Smaller, weaker, yet better: Training llm reasoners via compute-optimal sampling",
       "authors": ["Hritik Bansal", "Arian Hosseini", "Rishabh Agarwal", "Vinh Q. Tran", "Mehran Kazemi"],
       "year": 2024,
       "arxiv_id": "2408.16737",
       "relevance": "Studies compute-optimal sampling strategies for LLM reasoning, showing small models with more sampling can outperform larger models."
     },
     {
       "title": "Rest-mcts*: Llm self-training via process reward guided tree search",
       "authors": ["Dan Zhang", "Sining Zhoubian", "Ziniu Hu", "Yisong Yue", "Yuxiao Dong", "Jie Tang"],
       "year": 2024,
       "arxiv_id": "2406.03816",
       "relevance": "Uses MCTS for LLM search over individual tokens, contrasting with PLANSEARCH's approach of searching over natural language plans."
     },
     {
       "title": "Quiet-star: Language models can teach themselves to think before speaking",
       "authors": ["Eric Zelikman", "Georges Harik", "Yijia Shao", "Varuna Jayasiri", "Nick Haber", "Noah D. Goodman"],
       "year": 2024,
       "arxiv_id": "2403.09629",
       "relevance": "Trains models to generate internal reasoning tokens, a training-time alternative to PLANSEARCH's inference-time diversity approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Code is released and method can be applied on top of existing LLM APIs for code generation, though it requires significant inference compute (300-400 samples per problem)."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that scaling model parameters is sufficient — demonstrates that searching over natural language plans can nearly double performance, and that instruction-tuned models may be less diverse than base models."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on improving code generation accuracy."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with existing work; positions itself as a complementary contribution to existing search methods."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released at GitHub (scaleapi/plansearch), usable with LLM APIs, though running experiments requires API credits and compute."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Scale AI is a well-known AI company; the paper evaluates on prominent models including Claude 3.5 Sonnet and GPT-4o, attracting attention from those ecosystems."
     }
   }
 }