ai-research-survey

scan.json (28290B)

---

 {
   "paper": {
     "title": "Collaborative Agents for Automated Program Repair in Ruby",
     "authors": [
       "Nikta Akbarpour",
       "Mahdieh Sadat Benis",
       "Fatemeh Hendijani Fard",
       "Ali Ouni",
       "Mohamed Aymen Saied"
     ],
     "year": 2025,
     "venue": "arXiv preprint (submitted to ACM conference)",
     "arxiv_id": "2511.03925"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a Figshare replication package link: https://figshare.com/s/829875edc8c876c50de5. Section 8 states 'Full replication package and experimental data are available at' this URL. The introduction also states 'Releasing all scripts and experimental results as open source to support replication and future research.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the publicly available xCodeEval benchmark (Khan et al. 2024). The replication package on Figshare includes experimental data. The benchmark dataset is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 mentions GPU hardware (NVIDIA Tesla V100, NVIDIA H100 SXM5) and 4-bit quantization, but there is no mention of a requirements.txt, Dockerfile, conda environment, or specific library versions. The environment specification is insufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the replication package is provided on Figshare, the paper itself does not include step-by-step reproduction instructions, a README description of how to run experiments, or specific commands. The paper only provides a link to the replication package without detailing what it contains or how to use it."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 67.0% pass@1) without confidence intervals or error bars. No uncertainty quantification is provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RAMP 'outperforms' baselines based solely on comparing pass@1 numbers (e.g., 67.0% vs 61.7%) without any statistical significance tests. Section 3.4 explicitly states 'As this is a deterministic generation with no randomness, the results are comparable without requiring statistical tests,' but this justification is weak given the small sample sizes (34 questions in RQ1)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports percentage point differences with baseline context. For example, ablation studies report 'pass@1 dropping from 66.5% to 48.4% (−18.1 points)' and comparative results give both raw values (e.g., RAMP 67.0% vs LANTERN 61.7%). This provides enough context to understand effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The RQ1 comparison uses only 34 Ruby samples (10% of validation set). The paper acknowledges this was 'due to resource constraints' (Section 6) but does not justify whether 34 samples provide adequate statistical power for the claims made. No power analysis is presented."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 states they use greedy decoding (deterministic, single sample), so there is no variance across runs. However, this means the paper reports single-run results with no measure of variability. No standard deviation or variance is reported for any experiment."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes six baselines: LANTERN, ChatRepair, Self-Planning, Self-Collaboration, Few-Shot, and Zero-Shot prompting. Table 1 compares all methods on Ruby."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LANTERN (2025), ChatRepair (2024), Self-Planning (2024), Self-Collaboration (2024), and RepairAgent (2025) are all recent approaches. The baselines are contemporary and represent the current state of the art in LLM-based APR."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Table 2) presents ablation studies removing test generation, self-reflection, first reflection, all reflection, and adding specification inference. RQ3 (Table 4) ablates prompt components (I/O specs, time/memory limits, sample I/O). These ablations are conducted across two models."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only pass@1 as its evaluation metric. Section 3.4 justifies this choice but acknowledges it 'may miss qualities such as efficiency and readability' (Section 6). No other metrics (e.g., exact match, edit distance, code quality) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The paper evaluates repairs solely through automated test execution. Given the paper's claims about practical repair utility, human evaluation of repair quality (e.g., readability, correctness beyond test passing) would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The xCodeEval benchmark provides hidden test cases that are used only for final evaluation (Section 3.2, Step 5). Generated tests are used during the iterative repair process, and hidden benchmark tests are reserved for final validation, providing a clear separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 provides breakdowns by difficulty level (Figure 4 left), bug execution outcome type (Figure 5 right), and problem domain tags (Figure 5 left). Table 3 provides per-iteration breakdowns of test case outcomes."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 discusses failure categories: RAMP struggles with TIME_LIMIT_EXCEEDED (40.0% success), advanced categories (binary search, bitmasks, matrices at 0%), and higher-difficulty problems. Section 4.4 and the Discussion section address these limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: SCoT performs worse than CoT (Table 4), specification-inference reflection reduces DeepSeekCoder performance (-2.7 points in Table 2), adding sample I/O pairs decreases performance (-2.7 points in Table 4), and Figure 3 left shows regressions where previously correct repairs become incorrect in later iterations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims RAMP achieves 'pass@1 of 67% on Ruby, outperforming prior approaches' (supported by Table 1), converges within five iterations (supported by Figure 2 left and Figure 3 right), and that test generation and self-reflection are key drivers (supported by Table 2 ablations). All claims match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., 'removing test generation...substantially degrades performance'). The ablation design—controlled single-variable manipulation—is adequate for these causal claims. Each component is removed individually while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Automated Program Repair in Ruby' but the evaluation is on competitive-programming-style tasks from xCodeEval (short, single-file programs with well-defined I/O). While Section 6 acknowledges this limitation, the abstract and title do not bound the claims to competitive programming. The paper also claims RAMP provides 'new insights into multi-agent repair strategies' and is 'a practical solution for APR in Ruby' without bounding to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section considers alternative explanations: SCoT's lower performance may be due to 'rigid structures constrain[ing] the model's flexibility' (Section 5). The threats to validity section discusses prompt sensitivity, parameter choices, and benchmark representativeness as factors that could affect results. Section 4.2 discusses why Qwen Coder may be insensitive to ablations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 specifies 'DeepSeek-Coder 6.7B-Instruct' and 'Qwen2.5-Coder-7B-Instruct' with specific parameter counts. These are specific open-source model versions with known architectures and weights, unlike proprietary API-based models where version ambiguity is problematic."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 states 'Due to space limitations, the prompts used for each LLM-based agent are included in the replication package.' The prompts are not in the paper or appendix itself. While they may be in the Figshare package, the paper describes prompts only in natural language (e.g., 'the model is prompted to generate only a natural language explanation')."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 reports: temperature 0.8 for code generation, 0.1 for other agents, top_p=0.95, single candidate per bug, 4-bit quantization, and 11 iterations. For baselines, 'we use the hyperparameters specified in their original implementations.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail in Section 3.1: four agents (Feedback Integrator, Test Designer, Programmer, Test Executor), their roles, data flow between them (Figure 1), the iterative loop, and stopping conditions. Section 3.1.1-3.1.4 describes each agent's specific function."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the benchmark selection: xCodeEval validation set, 5,068 samples across 11 languages, 343 Ruby samples. Section 4 explains the 10% sampling strategy for RQ1 ('sampled to preserve the original language and difficulty distribution') with 34 Ruby questions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated, substantive section covering internal, external, construct, and conclusion validity threats across multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: 'RAMP relies on LLM-generated test cases for intermediate feedback; these test cases may not always be correct,' 'the behavior of individual agents...is sensitive to prompt design,' 'We conducted experiments on the XCodeEval benchmark, which covers competitive-programming-style tasks. These problems may not fully reflect the complexity or diversity of real-world Ruby projects,' and 'RQ1 evaluations were performed on a 10% subset.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states: 'These problems may not fully reflect the complexity or diversity of real-world Ruby projects.' Section 2 explicitly scopes: 'our focus is on competitive-programming style tasks: short, single-file programs with well-defined I/O-based test cases.' Section 6 also notes: 'we did not systematically investigate' generalization to other languages."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 provides a Figshare link with 'full replication package and experimental data.' The underlying xCodeEval benchmark is also publicly available. Per-task identifiers are mentioned in Section 6."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the data source: xCodeEval benchmark validation set with 5,068 samples across 11 languages, 343 Ruby samples. Each instance includes buggy code, problem description, unit tests, reference solution, execution outcome, problem tags, and difficulty level."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from a standard benchmark (xCodeEval), so recruitment methods do not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: xCodeEval validation set → Ruby subset (343 samples) → 10% subsample for RQ1 (34 samples, preserving language and difficulty distribution) → full set for RQ2-RQ4. Section 3.2 and Section 4 describe these steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no mention of funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of British Columbia (Canada), Ecole de Technologie Superieure (Canada), Laval University (Canada). These are academic institutions with no apparent conflict regarding the evaluated tools."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses DeepSeek-Coder 6.7B-Instruct and Qwen2.5-Coder-7B-Instruct but does not state the training data cutoff dates for either model. Since these models are evaluated on xCodeEval (a public benchmark), knowing the training cutoff is important for assessing contamination."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the xCodeEval benchmark data may have been in the training data of DeepSeek-Coder or Qwen2.5-Coder. The xCodeEval paper was published in 2024, and the models were trained on code data that may include competitive programming problems."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "xCodeEval includes competitive programming problems that may have been available online before the models' training cutoffs. No contamination analysis is performed. This is particularly relevant because the models are code-specialized and likely trained on competitive programming data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a benchmark evaluation of an automated program repair system."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section reports wall-clock time for all methods: RAMP takes ~6.6 hours, RAMP with early stopping ~5.1 hours, Self-Planning ~2.4 hours, Few-Shot ~6.4 hours, Zero-Shot is very fast, and LANTERN ~147 hours. GPU memory snapshots are also provided (24,229.25 MB GPU for full run). Figure 6 left visualizes the time-accuracy trade-off."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 specifies hardware: NVIDIA Tesla V100 (32GB) for RAMP with DeepSeekCoder, NVIDIA H100 SXM5 (80GB) for Qwen experiments and baselines. The Discussion section reports GPU memory usage (24,229.25 MB / 29,269.25 MB), CPU utilization (4.5% / 10.4%), and RAM (1,748.66 MB / 2,215.67 MB)."
       }
     }
   },
   "claims": [
     {
       "claim": "RAMP achieves the highest pass@1 on Ruby (67.0%), outperforming LANTERN (61.7%), Self-Planning (56.0%), and other baselines.",
       "evidence": "Table 1 shows pass@1 across all methods on the Ruby subset of xCodeEval (10% validation set, 34 samples). RAMP achieves 67.0% vs LANTERN 61.7%, Self-Planning 56.0%, Few-Shot 47.5%, Zero-Shot 24.1%, ChatRepair 17.6%, Self-Collaboration 0.0%.",
       "supported": "moderate"
     },
     {
       "claim": "RAMP converges quickly, reaching peak performance within five iterations.",
       "evidence": "Figure 2 left and Figure 3 right show cumulative pass@1 rising sharply in early iterations and plateauing by iteration 5. The text in Section 4.1 states 'performance rises from 55.0% at iteration 0 to 67.0% by iteration five, after which results plateau.'",
       "supported": "strong"
     },
     {
       "claim": "Test generation and self-reflection are key drivers of RAMP's performance, with removal causing up to 19.3 percentage point drops.",
       "evidence": "Table 2 shows ablation results: removing test generation drops DeepSeekCoder pass@1 from 66.5% to 48.4% (−18.1 points); removing self-reflection drops it from 66.5% to 47.2% (−19.3 points). However, Qwen Coder shows minimal sensitivity (<1.5 points change).",
       "supported": "moderate"
     },
     {
       "claim": "RAMP is particularly effective at repairing WRONG_ANSWER (68.5%), COMPILATION_ERROR (66.7%), and RUNTIME_ERROR (60.4%) cases.",
       "evidence": "Figure 5 right and Section 4.4 report pass@1 per execution outcome category on the full Ruby validation set (343 samples). WRONG_ANSWER: 68.5%, COMPILATION_ERROR: 66.7%, RUNTIME_ERROR: 60.4%, TIME_LIMIT_EXCEEDED: 40.0%.",
       "supported": "strong"
     },
     {
       "claim": "RAMP offers the best accuracy-time trade-off among evaluated methods for Ruby repair.",
       "evidence": "Figure 6 left and the Discussion section show RAMP achieves 67% pass@1 in ~6.6 hours, while LANTERN achieves 61.7% in ~147 hours. Self-Planning is faster (~2.4h) but lower accuracy (56.0%). However, LANTERN's time includes multi-language processing, making the comparison not fully apples-to-apples.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "RAMP, a multi-agent framework for automated program repair in Ruby, achieves 67.0% pass@1 on the xCodeEval benchmark, outperforming prior methods including LANTERN (61.7%) on a 34-sample subset. Ablation studies reveal that test generation and self-reflection are critical for DeepSeekCoder (up to 19.3 percentage point impact) but have minimal effect on Qwen Coder. RAMP converges within five iterations and is most effective on WRONG_ANSWER and COMPILATION_ERROR cases, while struggling with resource-related failures and advanced algorithmic categories. The framework demonstrates practical portability by achieving competitive results on C++ with minimal adaptation.",
   "red_flags": [
     {
       "flag": "Very small comparison sample size",
       "detail": "The head-to-head comparison with baselines (RQ1) uses only 34 Ruby samples (10% subset). At this sample size, a 5.3 percentage point difference between RAMP (67.0%) and LANTERN (61.7%) could easily be due to chance—this corresponds to roughly 1-2 additional problems solved. No statistical testing is performed."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "The paper claims RAMP 'substantially outperforms' baselines based on raw percentage comparisons without any significance tests. The justification that 'deterministic generation' eliminates the need for statistics ignores the sampling variability from the benchmark subset and the small sample size."
     },
     {
       "flag": "Inconsistent ablation results across models",
       "detail": "Ablation results are highly model-dependent: test generation and self-reflection produce 18-19 point improvements for DeepSeekCoder but <1.5 points for Qwen Coder (Table 2). This suggests the claimed benefits of the multi-agent design may be model-specific rather than general, but the paper's conclusions don't adequately bound this."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "DeepSeek-Coder and Qwen2.5-Coder are trained on large code corpora that likely include competitive programming problems. xCodeEval draws from such problems. No contamination analysis is performed despite this being a known risk for code LLMs evaluated on public benchmarks."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only pass@1 is reported. No metrics for code quality, readability, efficiency, or similarity to reference solutions are used. A repaired program that passes tests but is poorly written or inefficient would still count as a success."
     },
     {
       "flag": "Greedy decoding with no variance estimation",
       "detail": "Using greedy (deterministic) decoding means results are based on a single run with no variance estimation. While deterministic, the results are fragile—temperature variations or different sampling could yield meaningfully different outcomes, especially on 34 samples."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "doi": "10.1145/3650212.3680384",
       "relevance": "Autonomous multi-agent framework for repository-level program repair using AST navigation and fault localization from GitHub issues."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2025,
       "doi": "10.1109/ICSE55347.2025.00157",
       "relevance": "LLM-based autonomous agent for program repair, representative of the newest wave of multi-agent APR systems."
     },
     {
       "title": "MAGIS: LLM-based multi-agent framework for GitHub issue ReSolution",
       "authors": ["Wei Tao", "Yucheng Zhou", "Yanlin Wang", "Wenqiang Zhang", "Hongyu Zhang", "Yu Cheng"],
       "year": 2025,
       "relevance": "Multi-agent LLM framework targeting GitHub issue resolution, relevant to agentic software engineering workflows."
     },
     {
       "title": "SWE-Search: Enhancing Software Agents with Monte Carlo Tree Search and Iterative Refinement",
       "authors": ["Antonis Antoniades", "Albert Örwall", "Kexun Zhang", "Xixi Xie", "Anirudh Goyal", "William Yang Wang"],
       "year": 2025,
       "relevance": "Combines MCTS with multi-agent iterative refinement for software engineering tasks, advancing agentic search strategies."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "relevance": "Open platform for AI software engineering agents, relevant to understanding agentic coding tool infrastructure."
     },
     {
       "title": "Demystifying LLM-Based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2025,
       "doi": "10.1145/3715754",
       "relevance": "Analysis of LLM-based SE agents, directly relevant to understanding the agentic AI paradigm for software tasks."
     },
     {
       "title": "A Survey of LLM-based Automated Program Repair: Taxonomies, Design Paradigms, and Applications",
       "authors": ["Boyang Yang", "Zijian Cai", "Fengling Liu", "Bach Le", "Lingming Zhang"],
       "year": 2025,
       "relevance": "Comprehensive survey of LLM-based APR covering taxonomy and design paradigms, relevant as a survey-of-surveys candidate."
     },
     {
       "title": "Automated program repair via conversation: Fixing 162 out of 337 bugs for $0.42 each using chatgpt",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2024,
       "relevance": "ChatRepair demonstrates conversational LLM-based repair with cost reporting, a key baseline for agentic repair approaches."
     },
     {
       "title": "FixAgent: A Unified Debugging Approach via LLM-Based Multi-Agent Synergy",
       "authors": ["Cheryl Lee", "Chunqiu Steven Xia", "Jen tse Huang"],
       "year": 2024,
       "arxiv_id": "2404.17153",
       "relevance": "Multi-agent debugging framework aligning with cognitive models, relevant to understanding collaborative agent designs for SE."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2023,
       "relevance": "Foundational work on self-reflection in language agents, directly relevant to the reflection mechanism used in RAMP."
     },
     {
       "title": "A systematic literature review on large language models for automated program repair",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yang Xie"],
       "year": 2024,
       "relevance": "Systematic review of LLM-based APR revealing language coverage skew and evaluation gaps."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "authors": ["Daya Guo", "Qi Zhu", "Zhi Du"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Technical report for DeepSeek-Coder, one of the backbone models used in the RAMP evaluation."
     }
   ]
 }