ai-research-survey

scan.json (29269B)

---

 {
   "paper": {
     "title": "GraphCodeAgent: Dual Graph-Guided LLM Agent for Retrieval-Augmented Repo-Level Code Generation",
     "authors": [
       "Jia Li",
       "Xianjie Shi",
       "Kechi Zhang",
       "Ge Li",
       "Zhi Jin",
       "Lei Li",
       "Huangzhao Zhang",
       "Jia Li",
       "Fang Liu",
       "Yuwei Zhang",
       "Zhengwei Tao",
       "Yihong Dong",
       "Yuqi Zhu",
       "Chongyang Tao"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.10046"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GraphCodeAgent proposes a dual graph-guided LLM agent combining a Requirement Graph (RG) and Structural-Semantic Code Graph (SSCG) for retrieval-augmented repo-level code generation. On DevEval, it achieves 58.14% Pass@1 with GPT-4o, a 43.81% relative improvement over Dense RACG. The advantage is most pronounced for non-standalone code with complex cross-file dependencies, where it nearly doubles the best baseline's performance. Ablation shows the SSCG traversal tool is the most critical component (12.17% relative drop when removed).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Data Availability' states: 'The code implementation and data are publicly available at figshare anonymous link' with URL https://figshare.com/s/4148a1c56d08804cd75a."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks DevEval and CoderEval, and the figshare link claims to include data. Both benchmarks are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or dependency specifications are provided in the paper. The paper mentions tools (tree-sitter, Neo4j, stella_en_400M_v5, Black, DuckDuckGo API) but does not provide version-specific environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The figshare link is mentioned but the paper itself contains no instructions on how to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4, 5, 6, and 7 are reported as point estimates (e.g., '58.14' Pass@1) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper repeatedly claims GraphCodeAgent 'significantly outperforms' baselines but provides no statistical significance tests (no p-values, t-tests, or any formal tests). All claims are based on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports relative improvements with baseline context, e.g., '43.81% relative improvement over the strongest baseline Dense RACG' (58.14 vs 40.43), '94.30% relative gain' for cross-file tasks (43.31 vs 22.29). Absolute values are provided in all tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. DevEval has 1,825 examples and CoderEval's Python task count is not stated. No power analysis or discussion of whether these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.2 states 'we execute each method two times and report the average experimental results' but no standard deviation, variance, or spread measure is reported across those runs. Temperature is set to 0, so runs should be near-deterministic, but this is not explicitly discussed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are compared: ScratchCG, Sparse RACG, Dense RACG, RepoCoder, GraphCoder, and CodeAgent, covering text-based, graph-based, and agent-based RACG approaches (Section 4.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include RepoCoder (2023), GraphCoder (2024), CodeAgent (2024), and CodexGraph (2024). These are recent and represent the state of the art in RACG."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 presents a systematic ablation study removing each tool individually (SSCGTraverse, WebSearch, CodeTesting) and reporting the impact on Pass@1 with GPT-4o on DevEval."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 is used as the evaluation metric. The paper defines Pass@k generally (Equation 1) but only reports k=1. No other metrics (e.g., CodeBLEU, edit similarity) are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality. The PhD candidates verified RG construction (Section 3.2.2) but did not evaluate the system's code generation outputs. All evaluation is automated via test-case execution."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "DevEval and CoderEval are established benchmarks with designated test sets. DevEval contains 1,825 test examples from 117 repositories (Section 4.3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides detailed breakdowns by dependency type: Standalone (502), Non-standalone subdivided into Local-file (455), Cross-file (157), and Local&Cross-file (571)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1 provides a case study showing a successful generation but no failure cases are analyzed. There is no error analysis or discussion of where the approach breaks down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments show GraphCodeAgent outperforming baselines. The ablation (Table 5) shows all components contribute positively. No failed approaches or configurations are reported. WebSearch's minimal contribution (0.51% drop) is spun positively."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '43.81% with GPT-4o and 39.15% with Gemini-1.5-Pro on DevEval, and 31.91% with GPT-4o and 8.25% with Gemini-1.5-Pro on CoderEval' all match Table 4 exactly."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 5) uses controlled single-variable manipulation (removing one tool at a time) to support causal claims about component contributions. The comparative evaluation holds the backbone LLM constant while varying the approach."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Repo-Level Code Generation' broadly, but evaluation is limited to Python benchmarks (DevEval and CoderEval). The paper does not bound its claims to Python or to these specific benchmark types. Section 8 concludes with unbounded claims about 'real-world coding challenges.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.2 discusses threats to validity but does not consider alternative explanations for the improvements. For example, GraphCodeAgent may simply provide more context tokens to the LLM than baselines — the paper claims equal retrieved code elements but not equal information volume (graph structure adds information). This is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@1 (test-case execution correctness) and claims this measures code generation quality. The claims match the granularity of measurements without broader framing — they do not claim to measure developer productivity or software quality more broadly."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is specified as 'GPT-4o-2024-08-06' (specific version), but Gemini is listed as 'Gemini-1.5-Pro-latest' which is not a pinned version (changes over time). QWQ-32B is given without a specific version date. Not all models have exact versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes tool functionality in natural language (Section 3.4) and mentions instructions 'shown in the link provided at Section 9' (figshare). The actual agent prompts, ReAct reasoning prompts, and tool descriptions sent to the LLM are not provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports: temperature=0, maximum generation length=500 tokens, similarity threshold ε=0.8, embedding model stella_en_400M_v5, RepoCoder iterations=2. Key hyperparameters are stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes the agent scaffolding in detail: five tools (RGRetrieval, DualGraphMapping, SSCGTraverse, WebSearch, CodeTesting) with inputs/outputs (Table 2), ReAct reasoning strategy, workflow order, and stopping conditions. Figure 4 provides an overview diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.2.2 and 3.3.2 document how repositories are parsed with tree-sitter, how functions/classes/methods are extracted, how RG and SSCG are constructed, and how semantic similarity edges are computed. Section 4.5 describes prompt length truncation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Threats to Validity' discusses both internal validity (hyperparameter settings, fairness of comparison) and external validity (benchmark quality, baseline selection, LLM choice, metric selection)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses study-specific threats: the use of DeepSeek-V2.5 for RG construction with manual verification by PhD candidates, the adaptation of RepoCoder to repo-level generation, and the specific embedding model choice (stella_en_400M_v5)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of which languages, repository types, or task complexities are excluded. The threats section discusses potential validity issues but does not state explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While a figshare link is provided, the paper does not indicate that raw per-task results (individual pass/fail outcomes) are available. Only aggregate Pass@1 scores are reported in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "DevEval is described as containing '1,825 test examples from 117 repositories' covering '10 domains such as Internet and Database' (Section 4.3). CoderEval is described as originating from 'open-source projects from various domains' with contextual dependencies."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (DevEval and CoderEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from repository parsing (tree-sitter) through graph construction (RG and SSCG) to retrieval and code generation is documented across Sections 3.2-3.4. The evaluation pipeline (single generation, temperature 0, test execution) is described in Section 4.5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Wuhan University, Peking University, University of Hong Kong, Tsinghua University, Beihang University, Institute of Software Chinese Academy of Sciences, Academy of Military Sciences. Authors are not affiliated with the evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Without knowing the funding source, independence cannot be assessed. The authors use commercial APIs (GPT-4o, Gemini) but are not affiliated with those companies."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-4o, Gemini-1.5-Pro, or QwQ-32B. The models' training data periods are not mentioned."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether DevEval or CoderEval examples could have appeared in the training data of the models used. DevEval and CoderEval are sourced from public repositories that may be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DevEval (2024) and CoderEval (2024) use code from public repositories. GPT-4o-2024-08-06 could have been trained on these repositories. No contamination analysis is performed or discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The PhD candidates verifying RG construction are not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'The retrieval process of our approach typically takes only a few seconds on each task' but does not report API costs, token consumption, or total inference cost. The multi-turn LLM agent calls (average 2.3 SSCGTraverse invocations per task) would incur significant API cost which is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No GPU hours, API spend, or hardware specifications for running the experiments are provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0, making generation near-deterministic, but no seed sensitivity analysis is reported. The paper does not discuss whether results are sensitive to any source of randomness."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 states: 'we execute each method two times and report the average experimental results.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The similarity threshold ε=0.8 for SSCG and RepoCoder iterations=2 appear chosen without reporting how many configurations were tried. No hyperparameter search budget is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "ε=0.8 is used without justification for why this threshold was chosen. RepoCoder iterations=2 is justified by citing the original paper, but other hyperparameters are not justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own system against their own implementations of baselines. No acknowledgment of author-evaluation bias (Lucic et al. 2018). They mention keeping the same retrieved code count for fairness but do not discuss the inherent bias of authors implementing their own baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GraphCodeAgent uses an iterative multi-tool agent (average 2.3 SSCG traversals + other tool calls per task) requiring substantially more LLM calls than non-agent baselines. This compute difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Pass@1 on DevEval and CoderEval actually measures repo-level code generation capability in practice. The paper takes benchmark validity as given without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GraphCodeAgent uses a different scaffold (RG+SSCG-guided agent with 5 tools) than CodeAgent (5 programming tools with different strategies). Performance differences are attributed to the approach without controlling for or discussing the scaffolding confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. DevEval and CoderEval are sourced from public repositories that may predate model training. The models could have seen the solution code during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The WebSearch tool potentially provides external hints not discussed in terms of leakage. The paper mentions blocking 'websites that may lead to data leakage' but does not explain how."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples share structural similarities or come from repositories in the models' training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. The mention of blocking leakage-prone websites in WebSearch (Section 3.4.1) is not elaborated and does not constitute a detection method for benchmark contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "GraphCodeAgent achieves 43.81% relative improvement with GPT-4o on DevEval (Pass@1: 58.14% vs 40.43% for Dense RACG) and 39.15% relative improvement with Gemini-1.5-Pro (54.74% vs 39.34%)",
       "evidence": "Table 4 shows full results on DevEval across all baselines and both LLMs. Section 5.1 discusses the results.",
       "supported": "moderate"
     },
     {
       "claim": "GraphCodeAgent achieves 31.91% relative improvement with GPT-4o on CoderEval (Pass@1: 53.91% vs 40.87% for Sparse RACG) and 8.25% with Gemini-1.5-Pro (45.65% vs 42.17%)",
       "evidence": "Table 4 shows CoderEval results. The improvement on Gemini-1.5-Pro is notably smaller (8.25%) compared to DevEval.",
       "supported": "moderate"
     },
     {
       "claim": "SSCGTraverse is the most critical component, causing 12.17% relative performance drop when removed (Pass@1 from 58.14 to 51.83)",
       "evidence": "Table 5 ablation study on GPT-4o/DevEval. Only one model and one benchmark tested for ablation.",
       "supported": "moderate"
     },
     {
       "claim": "GraphCodeAgent's advantage is particularly significant for non-standalone code with complex cross-file dependencies, achieving 94.30% relative gain on cross-file tasks",
       "evidence": "Table 6 breaks down DevEval by dependency type. Cross-file: 43.31 vs 22.29 for RepoCoder. Local&Cross-file: 45.18 vs 32.07 for GraphCoder.",
       "supported": "moderate"
     },
     {
       "claim": "GraphCodeAgent generalizes to reasoning models, achieving 10.65% relative improvement over the best baseline on QwQ-32B",
       "evidence": "Table 7 shows QwQ-32B results on DevEval only. GraphCodeAgent: 54.14 vs RepoCoder: 48.93.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims 'significantly outperforms' repeatedly but provides no statistical tests (no p-values, confidence intervals, or formal tests). All comparisons are based on raw point estimates. With temperature=0 and only 2 runs averaged, the reliability of reported differences is unknown."
     },
     {
       "flag": "Relative improvement framing inflates perceived gains",
       "detail": "The paper prominently reports relative improvements (e.g., '43.81%') rather than absolute differences. The absolute improvement on DevEval with GPT-4o is ~18 percentage points (58.14 vs 40.43). The 8.25% relative improvement with Gemini on CoderEval is only ~3.5pp absolute."
     },
     {
       "flag": "No contamination analysis despite using public benchmark code",
       "detail": "DevEval and CoderEval source code from public repositories. GPT-4o and Gemini may have seen this code during training. No temporal analysis, contamination checks, or decontamination is performed."
     },
     {
       "flag": "Ablation study limited to one model and one benchmark",
       "detail": "The ablation (Table 5) is only performed with GPT-4o on DevEval. It is unclear whether component contributions hold for Gemini-1.5-Pro or on CoderEval."
     },
     {
       "flag": "Uncontrolled compute budget across methods",
       "detail": "GraphCodeAgent uses multiple LLM calls per task (RGRetrieval + DualGraphMapping + average 2.3 SSCGTraverse + 0.4 WebSearch + 0.8 CodeTesting), while simpler baselines use a single generation call. The compute and cost difference is never quantified or discussed."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only Pass@1 is reported. Code generation quality has multiple dimensions (readability, maintainability, efficiency, style consistency) that are not measured. The paper claims suitability for 'practical application in complex software development workflows' based solely on test-case pass rates."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation evaluation, foundational benchmark in the field."
     },
     {
       "title": "CodeAgent: Enhancing code generation with tool-integrated agent systems for real-world repo-level coding challenges",
       "authors": ["Kechi Zhang", "Jia Li", "Ge Li"],
       "year": 2024,
       "arxiv_id": "2401.07339",
       "relevance": "Pioneer LLM-based agent framework for repo-level code generation, key baseline in this paper."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Introduces iterative retrieval-generation pipeline for repo-level code completion, important RACG baseline."
     },
     {
       "title": "GraphCoder: Enhancing repository-level code completion via code context graph-based retrieval and language model",
       "authors": ["Wei Liu", "Ailun Yu"],
       "year": 2024,
       "arxiv_id": "2406.07003",
       "relevance": "Graph-based retrieval-augmented code completion framework using code context graphs."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks from GitHub issues."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "authors": ["Jia Li", "Ge Li"],
       "year": 2024,
       "arxiv_id": "2405.19856",
       "relevance": "Primary evaluation benchmark in this paper for repo-level code generation with 1,825 examples from 117 repositories."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Alternative approach to LLM-based software engineering that preprocesses repository structure without agentic scaffolding."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Foundational reasoning-and-acting framework used as the agent reasoning strategy in GraphCodeAgent."
     },
     {
       "title": "CodexGraph: Bridging large language models and code repositories via code graph databases",
       "authors": ["Xiangyan Liu", "Bo Lan"],
       "year": 2024,
       "arxiv_id": "2408.03910",
       "relevance": "Uses static analysis to extract code graphs from repositories for LLM-based code tasks."
     },
     {
       "title": "Locagent: Graph-guided LLM agents for code localization",
       "authors": ["Zhaoling Chen", "Xiangru Tang"],
       "year": 2025,
       "arxiv_id": "2503.09089",
       "relevance": "Combines graph-based repository representation with agentic LLM for code localization, closely related approach."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reasoning model relevant to understanding LLM capability improvements through RL training."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu", "Bo Shen"],
       "year": 2024,
       "relevance": "Second primary evaluation benchmark in this paper, assessing pragmatic code generation with contextual dependencies."
     }
   ]
 }