ai-research-survey

scan.json (19684B)

---

 {
   "paper": {
     "title": "EVOR: Evolving Retrieval for Code Generation",
     "authors": ["Hongjin Su", "Shuyang Jiang", "Yuhang Lai", "Haoyuan Wu", "Boao Shi", "Che Liu", "Qian Liu", "Tao Yu"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2402.12317"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our model, code, and data are available at https://arks-codegen.github.io.' A URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and data are available at the project page. They also compile a new benchmark EVOR-BENCH with four datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section listing library versions is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper itself. The project page is referenced but the paper does not contain a README or reproducing results section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-4 and figures are reported as point estimates (e.g., '37.9') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims EVOR 'significantly' outperforms baselines but provides no statistical significance tests (no p-values, t-tests, etc.)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage improvements with baseline context, e.g., 'EVOR outperforms DocPrompting by 18.6% on average using CodeLlama' and provides full baseline numbers in Table 2."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are reported (142, 45, 107, 113 problems) but no justification is given for why these sizes are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or spread measures are reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baselines are compared: Vanilla, MPSC, ExeDec, Reflexion, and DocPrompting (Section 3.1, Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: Reflexion (2024), DocPrompting (2023), MPSC (2023), ExeDec (2023). These are contemporary and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 (Table 3) ablates query evolution vs. knowledge evolution vs. both. Section 4.2 (Table 4) ablates knowledge source types."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only execution accuracy (pass@1) as the metric: 'By default, we use the execution accuracy (pass@1) as the metric throughout the paper.' Section 4.4 also uses pass@t but this is the same metric at different token budgets."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via execution accuracy. Human evaluation could assess code quality beyond pass/fail."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "EVOR-BENCH is a newly compiled benchmark with manually written ground truth solutions. The benchmark problems are separate from any development data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dataset breakdowns across all four datasets (Scipy-M, Tensorflow-M, Ring, Pony) rather than just averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis or qualitative discussion of failure cases is presented. The paper does not show where EVOR fails or discuss specific failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that web search 'only marginally improves the results' (Section 3.2) and achieves less than 1% improvement when used alone without query evolution (Section 4.2, Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '2 to 4 times execution accuracy' which is supported by Table 2 (e.g., EVOR 35.3% vs Reflexion 13.9% with ChatGPT). Claims about flexibility and combination with other methods are supported in Section 4.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by controlled ablation studies in Sections 4.1 and 4.2, where individual components are systematically added/removed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Evolving Retrieval for Code Generation' is broad, but the evaluation is limited to 4 specific datasets (2 modified Python libraries, 2 long-tail languages) and 2 models. The paper does not explicitly bound its generalization claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, whether the gains come primarily from more LLM calls/tokens rather than the retrieval evolution is not adequately addressed beyond Section 4.4."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'gpt-3.5-turbo-1106' (footnote 5), 'CodeLlama-34b-Instruct-hf' (footnote 6), 'GPT-4-1106' and 'Claude-3-opus' for SWE-bench experiments."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompting approaches but does not provide full prompt text. Query evolution and code generation prompts are described in natural language without the actual text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Maximum iterations set to 30, termination condition (same feedback 3 consecutive iterations), maximum context length of 4096, INSTRUCTOR-xl as retrieval model (Section 3.2). However, temperature/sampling settings are not explicitly stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 provides a detailed formal description of the EVOR pipeline including query evolution, retrieval, generation, execution feedback, and knowledge base evolution steps."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 describes dataset curation: modified Python libraries to simulate updates, adapted DS-1000 problems, selected Ring and Pony for long-tail languages, manual ground truth annotation. More details in Appendix A."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' discusses iterative process leading to longer latency and increased energy consumption."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is generic, mentioning only latency and energy consumption concerns. No specific threats to validity for the experimental results are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings/populations are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The project page (https://arks-codegen.github.io) claims to make model, code, and data available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 describes how datasets were compiled: modifying Scipy/Tensorflow libraries, adapting DS-1000 problems, selecting LeetCode problems for Ring/Pony, manual ground truth writing."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data is from standard benchmarks and programmatically constructed datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides dataset statistics. The curation process is described in Section 2.4 with further details in Appendix A, including how problems were adapted and solutions annotated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section mentioning grants or sponsors is visible in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Hong Kong, Fudan University, Sea AI Lab."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training cutoff dates for gpt-3.5-turbo-1106 and CodeLlama are not explicitly stated, though the paper addresses contamination concerns through benchmark design."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 explicitly addresses this: 'We do not use a real library update version because it is potentially exposed to LLM training data.' The benchmark is designed to avoid overlap by modifying libraries and using long-tail languages excluded from StarCoder training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark design (EVOR-BENCH) is motivated by avoiding contamination: modified libraries simulate unseen updates, and Ring/Pony are chosen because they 'have little public data and are excluded from the StarCoder training set' (Section 2.4)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 analyzes token consumption at different budgets (4k-24k tokens) and compares EVOR's efficiency against DocPrompting. Figure 3 shows pass@t at different token levels."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, GPU hours, or total computational budget is reported. The token analysis in Section 4.4 shows per-example token budgets but not total experiment cost."
       }
     }
   },
   "claims": [
     {
       "claim": "EVOR achieves 2-4x execution accuracy compared to existing methods like Reflexion and DocPrompting",
       "evidence": "Table 2: EVOR achieves 35.3% avg with ChatGPT vs Reflexion 13.9% (~2.5x) and DocPrompting 19.2% (~1.8x). With CodeLlama, EVOR 32.2% vs DocPrompting 16.0% (2x).",
       "supported": "strong"
     },
     {
       "claim": "Synchronous evolution of both queries and knowledge is consistently better than evolving either alone",
       "evidence": "Table 3 (Section 4.1): EVOR (evolve both) achieves 35.3% vs evolve query only 28.4% vs evolve knowledge only 23.8% with ChatGPT. Similar pattern with CodeLlama.",
       "supported": "strong"
     },
     {
       "claim": "EVOR can be combined with existing methods (MPSC, ExeDec, Reflexion) for further improvement",
       "evidence": "Table 2: EVOR + Reflexion achieves 37.9% vs EVOR alone 35.3% with ChatGPT, up to 2.6% additional gain on average.",
       "supported": "strong"
     },
     {
       "claim": "EVOR is a more effective approach to using tokens, achieving superior results at all token budgets",
       "evidence": "Figure 3 (Section 4.4): EVOR achieves higher pass@t than DocPrompting at all token consumption levels from 4k to 24k for both models.",
       "supported": "moderate"
     },
     {
       "claim": "Diverse knowledge sources enhance RACG performance, with larger improvements under evolution",
       "evidence": "Table 4 (Section 4.2): CodeLlama with Exec+Code+Doc achieves 32.2% with evolution vs 20.4% without. Adding documentation to Exec+Code improves by 6.9% with evolution but only 4.5% without.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EVOR, a retrieval-augmented code generation pipeline that synchronously evolves both queries and knowledge bases, achieves 2-4x execution accuracy over existing methods on a new benchmark (EVOR-BENCH) covering updated libraries and long-tail programming languages. Ablation studies show that evolving both queries and knowledge is consistently superior to evolving either alone. The approach is composable with existing methods like Reflexion and SWE-agent for additional gains. Diverse knowledge sources (documentation, code snippets, execution feedback) provide complementary benefits that are amplified by the evolution mechanism.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, standard deviations, or confidence intervals. Given that LLM outputs are stochastic, results could vary across runs."
     },
     {
       "flag": "No significance tests despite repeated claims of 'significant' improvement",
       "detail": "The paper uses the word 'significantly' multiple times to describe improvements but provides no statistical significance tests."
     },
     {
       "flag": "Small dataset sizes for some benchmarks",
       "detail": "Tensorflow-M has only 45 problems. Performance differences of a few percentage points on such small datasets may not be meaningful."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Key benchmark for evaluating LLM agents on real-world software engineering tasks, used in EVOR's repo-level evaluation."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn"],
       "year": 2024,
       "relevance": "Iterative LLM self-improvement framework used as baseline; represents agent-based code generation approaches."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark for code generation, foundational for LLM code evaluation."
     },
     {
       "title": "DocPrompting: Generating Code by Retrieving the Docs",
       "authors": ["Shuyan Zhou"],
       "year": 2022,
       "relevance": "Retrieval-augmented code generation using documentation; key baseline and predecessor to EVOR."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Uses execution feedback to refine LLM code generation, related approach to EVOR's execution feedback component."
     },
     {
       "title": "StarCoder: May the Source Be with You!",
       "authors": ["Raymond Li"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM; its training set exclusions informed EVOR-BENCH's choice of long-tail languages."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Rigorous evaluation methodology for LLM code generation quality."
     },
     {
       "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation",
       "authors": ["Yuhang Lai"],
       "year": 2023,
       "relevance": "Source benchmark from which EVOR-BENCH's Scipy and Tensorflow problems were adapted."
     },
     {
       "title": "Active Retrieval Augmented Generation",
       "authors": ["Zhengbao Jiang"],
       "year": 2023,
       "relevance": "Active RAG approach that iteratively decides when to retrieve; related to EVOR's evolving retrieval paradigm."
     }
   ]
 }