ai-research-survey

scan.json (24216B)

---

 {
   "paper": {
     "title": "CodeCoR: An LLM-Based Self-Reflective Multi-Agent Framework for Code Generation",
     "authors": ["Ruwei Pan", "Hongyu Zhang", "Chao Liu"],
     "year": 2025,
     "venue": "ACM (preprint)",
     "arxiv_id": "2501.07811",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Data Availability) provides a link to source code, experimental data, and prompts: https://anonymous.4open.science/r/CodeCoR-3EFC. Note this is an anonymized link, which may be temporary for review."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (HumanEval, HumanEval-ET, MBPP, MBPP-ET) and states experimental data is available at their anonymous repository (Section 7)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup with library versions is mentioned in the paper. The paper mentions using Python and psutil but does not provide sufficient detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper points to an anonymous repository but does not include step-by-step reproduction instructions in the paper itself. No README with commands or a 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-6 are reported as point estimates (e.g., '86.6%') with no confidence intervals or error bars, despite the paper stating they ran 10 rounds of experiments (Section 5.4)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeCoR 'significantly outperforms' baselines multiple times but provides no statistical significance tests (no p-values, t-tests, or other tests). Differences are compared by raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baselines: e.g., CodeCoR achieves 86.6% vs MapCoder's 80.5% on HumanEval, and provides average Pass@1 of 77.8% vs MapCoder's 72.8% (Section 4.5). This provides sufficient context to judge magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why HumanEval (164 problems), MBPP, or the first 10 problems for cost analysis were chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states they 'conducted 10 rounds of experiments for each trial and averaged the results' (Section 5.4), but no standard deviations, variance, or spread measures are reported anywhere in the tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists 22 baseline models including code LLMs (InCoder, CodeGeeX, StarCoder, etc.) and multi-agent frameworks (MapCoder, CodeCoT, MetaGPT, ChatDev). Comprehensive comparisons in Tables 2 and 6."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include MapCoder (2024), CodeCoT (2023), MetaGPT (2023), ChatDev (2024), and GPT-4 — all recent and competitive at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 (RQ2) presents ablation results removing each of the four major components: w/o Prompt Agent, w/o Test Agent, w/o Repair Agent, and w/o Pruning Method, demonstrating each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three metrics: Pass@1, Edit Distance, and BLEU score (Section 4.4, Tables 2-3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code is included. All evaluation is automated via test case execution and automated metrics (Pass@1, Edit Distance, BLEU). Human evaluation could assess code readability and maintainability."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks HumanEval and MBPP have standard test suites that are separate from the task descriptions. The paper evaluates against these standard test cases."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (HumanEval, HumanEval-ET, MBPP, MBPP-ET) in Tables 2 and 4, allowing readers to see variation across different benchmark types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 (Section 5.2) shows a specific failure case where code without the Repair Agent contains a semantic error (unnecessary conditional check), and discusses how the Repair Agent fixes it."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 4) shows that removing components degrades performance, which constitutes negative results. Section 5.3 shows that performance varies across repair rounds, with best at 3 rounds and worse at others."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims an average Pass@1 of 77.8% and that CodeCoR 'significantly outperforms existing baselines (e.g., CodeCoT and MapCoder).' Table 2 confirms these numbers: average across 4 datasets is indeed 77.8% (86.6+80.5+79.2+65.2)/4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (Table 4, Section 4.6): removing each component reduces performance, demonstrating each component's contribution via controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims a general 'Framework for Code Generation' but all experiments are on Python-only benchmarks (HumanEval, MBPP). The paper does not bound its claims to Python or to the specific function-level code generation tasks tested. The broad title and claims about 'code generation' generally are not bounded to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 (Threats to Validity) discusses generic methodological concerns (execution environment variability, experimenter bias, metric suitability) but does not discuss specific alternative explanations for why CodeCoR outperforms baselines — e.g., whether the improvement comes from simply making more API calls or generating more candidates."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-3.5-turbo', 'GPT-4', and 'CodeLlama (34B)' without specifying exact model versions or snapshot dates (e.g., no 'gpt-3.5-turbo-0613' or API version). Marketing names without version identifiers do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows example prompts and outputs for all four agents, and Figure 5 shows the pruning prompts with the actual workflow. Section 7 states that 'concrete examples of prompts and generated code' are available in the repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API hyperparameters are reported anywhere in the paper, despite using LLM APIs for all four agents."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail in Section 3 (Methodology), including the four agents, their interactions, the five-phase workflow (Figure 2), pruning methods (Section 3.3, Figures 4-5), and the overall algorithm (Algorithm 1)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the four datasets used and their characteristics. The paper uses standard benchmarks without modification, and Section 4.3 notes that 'results of the baselines are obtained from previous studies under the same experimental setting.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 'Threats to Validity' is a dedicated subsection discussing four categories of validity threats with multiple bullet points."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The threats in Section 5.4 are largely generic: 'minor fluctuations in execution environments could still introduce variability,' 'experimenter bias and errors remain potential threats,' 'the suitability of these metrics could be questioned.' These are boilerplate concerns, not specific threats to this particular study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what results do NOT show. The final bullet in Section 5.4 acknowledges 'specific datasets and settings used might limit the broader applicability' but does not specify what was not tested (e.g., non-Python languages, complex multi-file projects, real-world software engineering tasks)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states that 'source code, experimental data, and concrete examples of prompts and generated code' are available at the anonymous repository. The underlying benchmarks (HumanEval, MBPP) are also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the four datasets. Section 4.3 explains that baseline results were obtained from previous studies. Section 4.7 (RQ3) describes using psutil on the first 10 HumanEval problems on a dedicated server."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper uses standard code generation benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 documents the full pipeline from task description input through all five phases to final code output. The flow of data between agents is clearly documented in Section 3 and Figures 2, 4, 5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no mention of grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed as affiliated with Chongqing University, China. Their email addresses ({panruwei,hyzhang,liuchao}@cqu.edu.cn) are provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding disclosure is a concern — the paper does not state whether it is unfunded or simply omits this information."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-3.5-turbo, GPT-4, and CodeLlama on HumanEval and MBPP benchmarks but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021) and MBPP (published 2021) are well-known public benchmarks. GPT-3.5-turbo and GPT-4 were trained after these benchmarks were published, creating contamination risk. The paper does not discuss this at all."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP have been publicly available since 2021, well before the training cutoffs of GPT-3.5-turbo, GPT-4, and CodeLlama. The paper does not address the risk that these models may have seen the benchmark problems during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of code generation systems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (RQ3) reports runtime (123.69s), CPU usage (0.8%), memory (0.01 GB), disk I/O, and network I/O for CodeCoR vs baselines on the first 10 HumanEval problems. However, no API cost in dollars or tokens consumed is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Table 5 reports per-problem cost metrics for 10 problems, the total computational budget for all experiments (total API calls, total tokens, total dollar cost) is not stated. The cost analysis covers only 10 problems, not the full experimental suite."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeCoR achieves an average Pass@1 score of 77.8% across four datasets (HumanEval, HumanEval-ET, MBPP, MBPP-ET) with GPT-3.5-turbo, significantly outperforming MapCoder (72.8%) and CodeCoT.",
       "evidence": "Table 2 shows CodeCoR scores of 86.6%, 80.5%, 79.2%, 65.2% on the four datasets respectively. MapCoder scores 80.5%, 77.4%, 78.9%, 54.4%. Average for CodeCoR is 77.8% vs MapCoder's 72.8%.",
       "supported": "moderate"
     },
     {
       "claim": "CodeCoR achieves Pass@1 of 94.5% on HumanEval and 83.5% on HumanEval-ET when using GPT-4.",
       "evidence": "Table 6 reports these numbers for GPT-4. CodeCoR outperforms MapCoder (93.9%, 82.9%) and all other methods listed.",
       "supported": "moderate"
     },
     {
       "claim": "Each major component (Prompt Agent, Test Agent, Repair Agent, Pruning Method) is necessary for CodeCoR's performance.",
       "evidence": "Table 4 (RQ2) shows ablation results: removing Test Agent causes the largest drop (45.1% on HumanEval vs 86.6% full), removing Prompt Agent drops to 77.4%, removing Repair Agent to 75.6%, and removing Pruning to 77.4%.",
       "supported": "moderate"
     },
     {
       "claim": "CodeCoR incurs lower runtime cost than other representative code generation frameworks.",
       "evidence": "Table 5 shows CodeCoR at 123.69s vs MapCoder 166.45s, SCoT 251.79s, Self-Planning 242.92s, though CodeChain is faster at 121.80s. Measured on only the first 10 HumanEval problems.",
       "supported": "weak"
     },
     {
       "claim": "The code generated by CodeCoR shows higher textual similarity (BLEU) and shorter edit distance to ground-truth code compared to baselines.",
       "evidence": "Table 3 shows CodeCoR achieves mean BLEU of 0.314 vs MapCoder's 0.295 and mean edit distance of 272.70 vs MapCoder's 282.49.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeCoR is a self-reflective multi-agent framework for code generation using four LLM-based agents (Prompt, Coding, Test, Repair) with pruning methods to filter low-quality outputs at each stage. On four Python code generation benchmarks using GPT-3.5-turbo, CodeCoR achieves an average Pass@1 of 77.8%, outperforming MapCoder (72.8%) and CodeCoT. Ablation studies confirm each component's necessity, with the Test Agent being most critical. The framework also generalizes to GPT-4 (94.5% on HumanEval) and CodeLlama, though all evaluations are limited to Python function-level code generation.",
   "red_flags": [
     {
       "flag": "No statistical significance tests despite 'significantly outperforms' claims",
       "detail": "The paper repeatedly claims CodeCoR 'significantly outperforms' baselines but provides no statistical tests (p-values, confidence intervals). The difference on MBPP between CodeCoR (79.2%) and MapCoder (78.9%) is only 0.3 percentage points, well within likely variance. Despite running 10 rounds, no standard deviations are reported."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "HumanEval and MBPP were published in 2021. GPT-3.5-turbo and GPT-4 were trained after this date and may have seen these benchmarks in training data. The paper does not acknowledge or address this contamination risk, which could inflate all results."
     },
     {
       "flag": "Cost analysis on only 10 problems",
       "detail": "The cost comparison in Table 5 uses only the first 10 HumanEval problems, which is a tiny sample. No API cost in dollars or tokens is reported, making it impossible to assess practical cost at scale."
     },
     {
       "flag": "Baseline results taken from other papers",
       "detail": "Section 4.3 states 'results of the baselines are obtained from previous studies under the same experimental setting.' This makes it impossible to verify identical conditions (same API endpoints, same model versions, same time period) across all comparisons."
     },
     {
       "flag": "No model version specification",
       "detail": "GPT-3.5-turbo and GPT-4 are referenced without specific version identifiers. Model behavior changes across versions, making results non-reproducible. Baseline results from other papers may use different model snapshots."
     },
     {
       "flag": "Generalization claims exceed evidence",
       "detail": "The paper presents itself as a general 'Code Generation' framework but all evaluations are on Python-only, function-level benchmarks. No testing on other programming languages, multi-file projects, or real-world software engineering tasks."
     }
   ],
   "cited_papers": [
     {
       "title": "MapCoder: Multi-Agent Code Generation for Competitive Problem Solving",
       "authors": ["M. A. Islam", "M. E. Ali", "M. R. Parvez"],
       "year": 2024,
       "arxiv_id": "2405.11403",
       "relevance": "State-of-the-art multi-agent code generation framework that CodeCoR directly compares against and claims to outperform."
     },
     {
       "title": "CodeCoT and Beyond: Learning to Program and Test like a Developer",
       "authors": ["D. Huang", "Q. Bu", "H. Cui"],
       "year": 2023,
       "arxiv_id": "2308.08784",
       "relevance": "Multi-agent code generation framework using chain-of-thought; one of CodeCoR's primary baselines."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["S. Hong", "X. Zheng", "J. Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Foundational multi-agent framework for software development with SOP-based coordination."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["C. Qian", "W. Liu", "H. Liu"],
       "year": 2024,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent software development framework using communicative agents with role specialization."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["N. Shinn", "F. Cassano", "A. Gopinath"],
       "year": 2024,
       "relevance": "Self-reflective agent framework using verbal feedback; baseline for code generation comparison."
     },
     {
       "title": "Self-Collaboration Code Generation via ChatGPT",
       "authors": ["Y. Dong", "X. Jiang", "Z. Jin", "G. Li"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Early multi-agent code generation framework with analyst-coder-tester role assignment."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, the primary evaluation dataset used in this paper."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["X. Chen", "M. Lin", "N. Schärli", "D. Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Self-debugging approach for LLM code generation; baseline in evaluation."
     },
     {
       "title": "CodeChain: Towards Modular Code Generation through Chain of Self-Revisions with Representative Sub-Modules",
       "authors": ["H. Le", "H. Chen", "A. Saha"],
       "year": 2023,
       "arxiv_id": "2310.08992",
       "relevance": "Iterative self-revision approach to code generation; used as baseline for both accuracy and cost comparison."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Rozière", "J. Gehring", "F. Gloeckle"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used to evaluate CodeCoR's generalizability beyond proprietary models."
     },
     {
       "title": "Is Self-Repair a Silver Bullet for Code Generation?",
       "authors": ["T. X. Olausson", "J. P. Inala", "C. Wang"],
       "year": 2023,
       "relevance": "Examines limitations of self-repair in code generation, directly relevant to CodeCoR's repair agent design."
     },
     {
       "title": "INTERVENOR: Prompt the Coding Ability of Large Language Models with the Interactive Chain of Repairing",
       "authors": ["H. Wang", "Z. Liu", "S. Wang"],
       "year": 2023,
       "arxiv_id": "2311.09868",
       "relevance": "Interactive repair framework for LLM code generation; baseline in evaluation."
     }
   ]
 }