ai-research-survey

scan.json (19433B)

---

 {
   "paper": {
     "title": "Controlled Self-Evolution for Algorithmic Code Optimization",
     "authors": ["Tu Hu", "Ronghao Chen", "Shuo Zhang", "Jianghao Yin", "Mou Xiao Feng", "Jingping Liu", "Shaolei Zhang", "Andy Wang", "Wenqi Jiang", "Yuqi Fang", "Sen Hu", "Huacan Wang", "Yi Xu"],
     "year": 2026,
     "arxiv_id": "2601.07348"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is publicly available at https://github.com/QuantaAlpha/EvoControl.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They evaluate on EffiBench-X, a publicly available benchmark (Qing et al., 2025). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup with library versions is described in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Implementation details are given (Appendix C) but no README-level commands or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CSE outperforms all baselines but provides no statistical significance tests. Comparisons are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw percentage scores are reported. No effect sizes (Cohen's d, etc.) are provided. While baseline context exists (e.g., Direct vs. CSE numbers), no formal effect size measures are used."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 623 problems but no justification is given for why this sample size is adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C mentions 5 repeated runs per candidate with outlier removal for the reward signal, but no variance or standard deviation is reported for the final metrics across runs or problems in Table 1."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: Direct, Self-Reflection, SE-Agent, and AlphaEvolve (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include AlphaEvolve (Novikov et al., 2025) and SE-Agent (Lin et al., 2025), which are recent and competitive methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents ablation removing each of the three components (Planning, Evolution, Memory). Figure 4 further analyzes Memory's synergy with other components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: Execution Time (ET), Memory Peak (MP), and Memory Integral (MI)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of code quality is included. All evaluation is automated via test cases and efficiency metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "EffiBench-X provides comprehensive test cases per problem (100 test cases with time/memory limits per Appendix C). The benchmark itself serves as the held-out evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (Python vs. C++) and by metric (ET, MP, MI) in Table 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses evolution dynamics including failure lessons in local memory (Section 4.3). The case study in Section 5.4 shows the full evolution trajectory including failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows that adding Memory to Planning alone provides negligible benefit (delta MI = -0.23), an honest negative result about component interactions."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CSE 'consistently outperforms all baselines across various LLM backbones' and 'achieves higher efficiency from early generations.' Table 1 and Figure 3 support these claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'CSE's effectiveness arises from the synergistic interplay of all three components' are supported by controlled ablation studies (Table 2) where single components are removed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Algorithmic Code Optimization' is reasonably scoped, but results are only on EffiBench-X (competitive programming). The paper does not bound generalization to this domain — it implies the framework works for code optimization generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The improvements could partly be due to the increased effective prompt length from memory injection, but this is not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'DeepSeek-V3-0324', 'Qwen3-235B-A22B', 'Claude-4.5-Sonnet', and 'GPT-5'. DeepSeek-V3-0324 includes a date suffix. However, Claude-4.5-Sonnet and GPT-5 lack snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt strategies conceptually but does not provide the actual prompt text used for planning, mutation, crossover, or reflection. Appendix C mentions a 'decomposition template' (Figure 9) but the full prompts are not shown in the paper text provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C reports: T=30 iterations, Ninit=5 plans, Nq=3 queries, Km=3 retrievals, epsilon=0.001, k=5 clipping, alternating mutation/crossover schedule, and embedding model (Qwen3-8B-Embedding)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: planning initialization (Section 4.1), genetic evolution with mutation and crossover (Section 4.2), hierarchical memory with local/global components and retrieval (Section 4.3), and Algorithm 1 provides the full workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the evaluation protocol: efficiency measured only on tasks solved by Direct, fallback to Direct baseline, 100 test cases per problem, 5 repeated runs with outlier removal."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after Section 6 (Conclusion), discussing the lack of amortization into the base model."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section only discusses one future direction (distilling trajectories into RL training). It does not discuss specific threats to validity of the current results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of what settings, tasks, or domains are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-problem scores, evolution trajectories) is released. Only aggregate results in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "EffiBench-X aggregates 623 problems from AtCoder, Codeforces, and LeetCode (Section 5.1). The benchmark source and evaluation protocol are described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses automated benchmarks only."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the full pipeline: problem selection from EffiBench-X, Direct baseline solving, efficiency measurement only on solved tasks, fallback protocol, 5-run averaging with outlier removal."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: NJU, PKU, Midea-AIRC, ECNU, SYSU, RUC, QuantaAlpha, QuantML."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Authors from Midea-AIRC and QuantaAlpha (industry affiliations) could have commercial interests."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors are affiliated with QuantaAlpha and QuantML which may have commercial interests in code optimization."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-5, Claude-4.5-Sonnet, DeepSeek-V3, and Qwen3 on EffiBench-X problems sourced from competitive programming platforms, but states no training cutoff dates for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "EffiBench-X problems come from AtCoder, Codeforces, and LeetCode — publicly available sources likely in model training data. No discussion of potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Competitive programming problems from public platforms are likely in training data of all evaluated models. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The method runs 30 LLM calls per task across 623 problems with 4 models, but no API costs, token counts, or wall-clock time is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is stated despite significant compute requirements (4 models x 623 problems x 30 iterations)."
       }
     }
   },
   "claims": [
     {
       "claim": "CSE consistently outperforms all baselines across various LLM backbones on EffiBench-X.",
       "evidence": "Table 1 shows CSE achieves highest average MI across all four models (Qwen3: 50.17%, DeepSeek-V3: 55.09%, Claude-4.5-Sonnet: 74.41%, GPT-5: 67.47%).",
       "supported": "moderate"
     },
     {
       "claim": "CSE achieves higher efficiency from early generations and maintains continuous improvement throughout evolution.",
       "evidence": "Figure 3 shows best-so-far MI curves. Table 3 shows CSE has more improvement events (#Imp.=1.79) and later Iter@Best (12.06) than baselines.",
       "supported": "moderate"
     },
     {
       "claim": "All three components (Planning, Evolution, Memory) are essential, with Memory showing the largest impact.",
       "evidence": "Table 2 ablation: removing Memory drops MI from 68.10% to 63.08% (largest drop). Figure 4 shows Memory's gain is context-dependent.",
       "supported": "strong"
     },
     {
       "claim": "Memory amplifies controlled evolutionary processes rather than providing universal improvement.",
       "evidence": "Figure 4 shows Memory + Planning alone gives delta MI = -0.23, but Memory + Evolution gives +2.68, and all three gives +5.02.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CSE proposes three mechanisms (diversified planning initialization, genetic evolution with controlled mutation/crossover, and hierarchical evolution memory) to improve exploration efficiency in LLM-based code optimization. On EffiBench-X across four LLM backbones and two programming languages, CSE outperforms baselines including AlphaEvolve and SE-Agent on efficiency metrics. Ablation studies show all three components contribute, with memory providing the largest impact but only when combined with evolution, demonstrating a synergistic rather than additive effect.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims are based on comparing point estimates without any significance tests, confidence intervals, or variance reporting. Margins between CSE and baselines are often small (e.g., 0.36% ET difference for Qwen3 Python)."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "EffiBench-X problems come from AtCoder, Codeforces, and LeetCode — public platforms almost certainly in the training data of GPT-5, Claude-4.5-Sonnet, etc. This is not discussed."
     },
     {
       "flag": "No cost reporting despite heavy compute",
       "detail": "Running 4 LLMs x 623 problems x 30 iterations represents substantial compute. No costs, token counts, or wall-clock times are reported, making practical applicability impossible to assess."
     },
     {
       "flag": "Industry affiliations without conflict disclosure",
       "detail": "Authors are affiliated with Midea-AIRC, QuantaAlpha, and QuantML (industry entities) but no conflicts of interest or funding sources are disclosed."
     }
   ],
   "cited_papers": [
     {
       "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery",
       "authors": ["Alexander Novikov"],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "Key baseline; represents population-based evolutionary code optimization with LLMs."
     },
     {
       "title": "SE-Agent: Self-Evolution Trajectory Optimization in Multi-Step Reasoning with LLM-Based Agents",
       "authors": ["Jiaye Lin"],
       "year": 2025,
       "arxiv_id": "2508.02085",
       "relevance": "Key baseline; trajectory-level self-evolving agent for code generation."
     },
     {
       "title": "AfterBurner: Reinforcement Learning Facilitates Self-Improving Code Efficiency Optimization",
       "authors": ["Mingzhe Du"],
       "year": 2025,
       "arxiv_id": "2505.23387",
       "relevance": "Self-reflection baseline for code efficiency optimization."
     },
     {
       "title": "EffiBench-X: A Multi-Language Benchmark for Measuring Efficiency of LLM-Generated Code",
       "authors": ["Yuhao Qing"],
       "year": 2025,
       "arxiv_id": "2505.13004",
       "relevance": "The primary evaluation benchmark used; measures code efficiency across languages."
     },
     {
       "title": "EffiLearner: Enhancing Efficiency of Generated Code via Self-Optimization",
       "authors": ["Dong Huang"],
       "year": 2024,
       "relevance": "Prior self-optimization method for code efficiency; evaluation protocol adopted from this work."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "relevance": "Foundational self-refinement method for LLM outputs."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn"],
       "year": 2023,
       "relevance": "Self-reflection framework for LLM agents using verbal feedback."
     },
     {
       "title": "Mathematical discoveries from program search with large language models",
       "authors": ["Bernardino Romera-Paredes"],
       "year": 2023,
       "relevance": "FunSearch; foundational work on LLM-driven evolutionary program search."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper; foundational LLM code generation evaluation."
     },
     {
       "title": "A survey on code generation with LLM-based agents",
       "authors": ["Yihong Dong"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based code generation agents relevant to the survey scope."
     }
   ]
 }