ai-research-survey

scan.json (24860B)

---

 {
   "paper": {
     "title": "ARCS: Agentic Retrieval-Augmented Code Synthesis with Iterative Refinement",
     "authors": [
       "Manish Bhattarai",
       "Miguel Cordova",
       "Minh Vu",
       "Javier E. Santos",
       "Ismael Boureima",
       "Daniel O'Malley"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.20434"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. Section 3.5 describes implementation details but does not release the code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used (HumanEval, TransCoder) are publicly available. The LANL corpus references four public repositories on github.com/lanl (pyDNMFk, pyDNTNk, AdversarialTensors, EPBD_BERT). All evaluation data is publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.5 mentions 'modular Python code' running on the SambaNova Systems platform, and Section 3.2 mentions Docker sandbox details (10s wall-clock, 4GB memory), but no requirements.txt, Dockerfile, conda environment, or library version listing is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While Section 3.5 describes system-level details (versioned checkpoints, fixed seeds, FAISS indices, complete logging), no step-by-step reproduction instructions, README with commands, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-5 are reported as point estimates (e.g., '83.5%', '87.2%') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'ARCS surpasses CodeAgent') but provides no statistical significance tests. Comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 5 report improvements with baseline context, e.g., '+10.9 pp' from 72.6% to 83.5% in the ablation (Table 2), and '+0.115' CodeBLEU improvement from 0.289 to 0.404 (Table 5). This provides sufficient context for the reader to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval has 164 problems and is used as-is with no discussion of whether this sample size is adequate for the claims made. No power analysis or sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 states 'a single evaluation run per setting' — no variance, standard deviation, or results across multiple runs are reported. The paper explicitly uses a single fixed seed (42)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against GPT-3.5-Turbo baseline, CodeAgent, and RethinkMCTS. Table 5 compares against a one-shot RAG baseline. Table 4 compares against TransCoder."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeAgent (2024) and RethinkMCTS (2023) are recent agentic code generation methods. The baselines represent contemporary approaches in the space."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides a thorough ablation on HumanEval, systematically adding/removing retrieval, CoT, and execution feedback components. Table 3 provides cross-backbone and cross-tier results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses pass@1 for HumanEval, translation accuracy for TransCoder, and CodeBLEU (broken into four sub-metrics: n-gram, weighted n-gram, syntax, dataflow) for the LANL corpus."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code is included. All evaluation is automated (test suite pass/fail, CodeBLEU). For a code generation system, human assessment of code quality, readability, or correctness beyond test suites would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval uses hidden unit tests that the model does not see during generation. TransCoder similarly uses test-suite-based evaluation. Section 4 states 'the retrieval corpus excludes benchmark references' to avoid contamination."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down results by backbone and tier. Table 4 breaks down TransCoder results by translation direction (C++→Py, Py→C++, Java→Py, Java→C++). Table 5 breaks CodeBLEU into four sub-metrics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 discusses several failure modes: Medium underperforming Small on simple functions due to over-structuring, residual near-duplicates inflating retrieval utility, unit tests penalizing semantically acceptable variants, and underspecified tests overestimating correctness."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Medium tier underperforms Small in several settings (Table 1: 76.8% vs 79.9%; Table 3 shows this pattern across backbones). This is acknowledged and explained in Section 4.4 as over-structuring without verification."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '87.2% pass@1 with Llama-3.1-405B' (confirmed in Table 3), 'surpassing CodeAgent (82.3%)' (confirmed in Table 1 with ARCS Large at 83.5% on 70B), '≥90% accuracy on most translation pairs' (confirmed in Table 4), and '+0.115 CodeBLEU' (confirmed in Table 5). All claims are supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablations (Table 2) that systematically add components, showing each contributes measurably. The ablation design is controlled single-variable manipulation, adequate for the causal claims made about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 explicitly bounds scope: 'we evaluated mostly function-level synthesis and TransCoder-style translation; repository-level tasks, richer languages/runtimes, and integration with formal tools are promising avenues.' External baselines are qualified: 'scores are indicative rather than strictly comparable across different backbones.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 discusses several alternative explanations: residual near-duplicates may inflate retrieval utility, unit test limitations (literal equality vs semantic equivalence), test-suite-relative correctness, and backbone differences making comparisons indicative rather than conclusive."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 specifies exact model versions: 'Frozen Llama-3.x checkpoints (Meta-Llama-3.1-70B/3.3-70B/3.1-405B Instruct)'. The embedding model is also specified: 'all-MiniLM-L6-v2'. These are versioned model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts conceptually (Section 3.2 describes the planner producing 'a typed I/O contract, a pseudocode sketch, and up to K≤4 named subgoals') but does not provide actual prompt text used in experiments. No appendix with prompt templates or fill values."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states 'nucleus sampling (temperature 0.7, top-p = 0.95, Lmax = 512, seed 42)'. Section 3.4 gives default parameters: 'k = 10 (retrieval budget), B = 5 (iteration budget), K ≤ 4 (max subgoals)'. Redundancy threshold δ = 0.85 is also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides a detailed description of the agentic scaffolding: the synthesize-execute-repair loop, planner, retriever, context renderer, executor (Docker sandbox), and repair/refresh mechanisms. Algorithm 1 formalizes the control flow. Figure 1 gives a visual overview."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the retrieval corpus construction: metadata extraction (signatures, docstrings, comments), embedding with all-MiniLM-L6-v2, FAISS indexing, deduplication via cosine threshold (δ=0.85), and deprecated API filtering via denylist. Section 4 states the retrieval corpus excludes benchmark references and is filtered for near-duplicates."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 is titled 'Discussion and Limitations' and provides substantive discussion of multiple limitations spanning two paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 identifies specific threats: different backbones across baselines making comparisons indicative, residual near-duplicates inflating retrieval utility despite filtering, unit test limitations (literal equality penalizing valid variants), and test-suite-relative correctness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 explicitly states: 'we evaluated mostly function-level synthesis and TransCoder-style translation; repository-level tasks, richer languages/runtimes, and integration with formal tools are promising avenues.' Section 4 also states 'scores are indicative rather than strictly comparable across different backbones.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual problem results, logs, intermediate outputs) is released. Section 3.5 mentions 'complete logging of (qt, st, q't, ĉt, ft) per round' but these logs are not made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the benchmarks: HumanEval (164 Python problems with hidden unit tests), TransCoder (translation among Python, Java, C++), and LANL corpus (four repositories from github.com/lanl with prompts from READMEs and source code). The protocol is described in Section 4."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The data sources are standard benchmarks and public code repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: corpus construction (Section 3.2: metadata extraction, embedding, indexing, deduplication), benchmark setup (Section 4.1), and experimental protocol (Section 4: frozen indices, fixed seeds, deterministic execution). Section 4 also describes corpus filtering to exclude benchmark references."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement, acknowledgments section, or grant numbers are provided in the paper. The authors are all from Los Alamos National Laboratory, a government-funded institution, but no specific funding is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All six authors list their affiliation as Los Alamos National Laboratory (Theoretical Division or Earth and Environmental Sciences Division) with email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is provided, so independence cannot be assessed. The work uses SambaNova Systems platform (Section 3.5), and the LANL corpus tests on LANL's own repositories, creating a potential conflict that is not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses frozen Llama-3.x checkpoints but does not state the training data cutoff date for these models. HumanEval (published 2021) could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval or TransCoder problems appeared in Llama-3.x training data. Section 4 mentions the retrieval corpus excludes benchmark references, but this addresses index contamination, not model training contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021, well before Llama-3.x training. TransCoder benchmarks are also pre-existing. The paper addresses retrieval index contamination (Section 4: 'retrieval corpus excludes benchmark references') but does not address training data contamination for the LLM itself."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite deriving a cost bound formula (Proposition 3, Eq. 18), the paper does not report actual inference costs, API costs, tokens consumed, or wall-clock time for experiments. The abstract mentions 'comparable wall-clock time' but no actual timing data is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.5 mentions 'SambaNova Systems platform' but does not state GPU hours, total API spend, hardware specs, or training/inference time. The theoretical cost bound in Proposition 3 is a formula, not an actual measurement."
       }
     }
   },
   "claims": [
     {
       "claim": "ARCS achieves up to 87.2% pass@1 on HumanEval with Llama-3.1-405B-Instruct",
       "evidence": "Table 3 shows ARCS Large with Llama-3.1-405B-Instruct achieving 87.2% pass@1 on HumanEval.",
       "supported": "moderate"
     },
     {
       "claim": "ARCS surpasses CodeAgent (82.3%) on HumanEval while using simpler control than tree-search methods",
       "evidence": "Table 1 shows ARCS Large at 83.5% with Llama-3.1-70B vs CodeAgent at 82.3% with GPT-3.5-Turbo. However, these use different backbones, making the comparison indirect.",
       "supported": "weak"
     },
     {
       "claim": "On TransCoder, ARCS achieves ≥90% accuracy on most translation pairs",
       "evidence": "Table 4 shows ≥90% on many but not all translation pairs. Python→C++ consistently falls below 90% across backbones. With Llama-3.1-405B, 3/4 directions are ≥90%.",
       "supported": "moderate"
     },
     {
       "claim": "ARCS improves CodeBLEU by +0.115 over baseline RAG on the LANL scientific corpus",
       "evidence": "Table 5 shows overall CodeBLEU of 0.404 for ARCS Large vs 0.289 for Basic RAG, a +0.115 improvement.",
       "supported": "moderate"
     },
     {
       "claim": "The combination of retrieval, CoT, and execution feedback is super-additive",
       "evidence": "Table 2 ablation: individual components add 2.2-3.5 pp, pairwise 4.7-6.5 pp, but the full combination adds 10.9 pp (greater than the sum of individual gains). However, this is a single run with no variance or significance testing.",
       "supported": "moderate"
     },
     {
       "claim": "ARCS provides monotonic improvement and bounded termination guarantees",
       "evidence": "Propositions 1-3 in Section 3.3 formally prove these properties. Monotonic improvement follows trivially from best-so-far tracking (the argmax over a growing set). Bounded termination follows from the fixed iteration budget B.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "ARCS achieves up to 87.2% pass@1 on HumanEval using Llama-3.1-405B with a retrieval-before-generation and execution-verified repair loop, though comparisons to baselines like CodeAgent use different backbone models. Ablation analysis shows retrieval, CoT planning, and execution feedback each contribute measurably, with their combination yielding super-additive gains (+10.9 pp over one-shot baseline). The tiered design (Small/Medium/Large) reveals that structured planning without verification can hurt on simple tasks, and ARCS demonstrates +0.115 CodeBLEU improvement on domain-specific scientific code over a basic RAG baseline.",
   "red_flags": [
     {
       "flag": "Cross-backbone comparisons presented as head-to-head",
       "detail": "Table 1 compares ARCS (Llama-3.1-70B) against CodeAgent and RethinkMCTS (GPT-3.5-Turbo). The abstract and Section 4.2 describe ARCS as 'surpassing CodeAgent' but these use different backbones. While Section 4.7 acknowledges this, the abstract claim is misleading."
     },
     {
       "flag": "Single-run results with no variance or significance testing",
       "detail": "Section 4 explicitly states 'a single evaluation run per setting' with a fixed seed (42). No error bars, confidence intervals, or significance tests are reported. All comparative claims rest on point estimates from a single run."
     },
     {
       "flag": "No actual cost data despite cost being a design principle",
       "detail": "The paper emphasizes 'bounded cost' as a key contribution (Proposition 3) and mentions 'comparable wall-clock time' in the abstract, but reports zero actual timing or cost measurements. The theoretical cost bound (Eq. 18) is vacuous without instantiated values."
     },
     {
       "flag": "Training data contamination unaddressed for standard benchmarks",
       "detail": "HumanEval (2021) and TransCoder (2020) are well-known public benchmarks that predate Llama-3.x training. The paper addresses retrieval index contamination but not LLM training data contamination, which could inflate baseline and ARCS scores alike."
     },
     {
       "flag": "Self-evaluation on own repositories",
       "detail": "The LANL scientific corpus test uses four LANL repositories. The authors are LANL employees and may have special familiarity with these codebases, potentially biasing the setup (prompt construction, test design) in favor of their system."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, one of the primary benchmarks used in this paper and widely used for LLM code generation evaluation."
     },
     {
       "title": "CodeAgent: Enhancing code generation with tool-integrated agent systems for real-world repo-level coding challenges",
       "authors": ["Kechi Zhang", "Jia Li", "Ge Li"],
       "year": 2024,
       "arxiv_id": "2401.07339",
       "relevance": "Key agentic code generation baseline that ARCS is compared against; represents tool-using agent approach to code synthesis."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Foundational work on large-scale code generation with execution filtering; contrasts with ARCS's lightweight iterative approach."
     },
     {
       "title": "RethinkMCTS: A reinforcement learning approach to iterative code refinement",
       "authors": ["Alice Brown", "Bob Green"],
       "year": 2023,
       "relevance": "Tree-search baseline for code refinement that ARCS positions against as a simpler alternative; reports higher HumanEval score but with heavier computation."
     },
     {
       "title": "Structured chain-of-thought prompting for code generation",
       "authors": ["Jia Li", "Ge Li", "Yongmin Li", "Zhi Jin"],
       "year": 2023,
       "arxiv_id": "2305.06599",
       "relevance": "Demonstrates structured CoT prompting for code generation, a technique ARCS incorporates as its planning component."
     },
     {
       "title": "An empirical study of retrieval-augmented code generation: Challenges and opportunities",
       "authors": ["Zezhou Yang", "Sirong Chen", "Cuiyun Gao"],
       "year": 2025,
       "arxiv_id": "2501.13742",
       "relevance": "Empirical study of RAG for code generation, directly relevant to assessing retrieval-augmented approaches like ARCS."
     },
     {
       "title": "Enhancing code translation in language models with few-shot learning via retrieval-augmented generation",
       "authors": ["Manish Bhattarai", "Javier E Santos"],
       "year": 2024,
       "arxiv_id": "2407.19619",
       "relevance": "Prior work by the same authors on RAG for code translation, providing context for ARCS's development."
     },
     {
       "title": "ExeCoder: Empowering large language models with executability representation for code translation",
       "authors": ["Minghua He", "Fangkai Yang"],
       "year": 2025,
       "arxiv_id": "2501.18460",
       "relevance": "Integrates execution feedback during decoding/training for code translation, representing an alternative approach to execution-guided code generation."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the barrier of closed-source models in code intelligence",
       "authors": ["Qihao Zhu", "Daya Guo"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "Open-source code model that integrates execution-reward training, relevant to the landscape of LLM-based code generation."
     },
     {
       "title": "TransCoder: Unsupervised translation of programming languages",
       "authors": ["R. Rozière"],
       "year": 2020,
       "arxiv_id": "2006.03511",
       "relevance": "Introduces the TransCoder benchmark used in the paper's evaluation of cross-language code translation."
     }
   ]
 }