ai-research-survey

scan.json (31361B)

---

 {
   "paper": {
     "title": "RepoGenReflex: Enhancing Repository-Level Code Completion with Verbal Reinforcement and Retrieval-Augmented Generation",
     "authors": [
       "Jicheng Wang",
       "Yifeng He",
       "Hao Chen"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2409.13122",
     "doi": "10.48550/arXiv.2409.13122"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "RepoGenReflex proposes an iterative RAG framework enhanced with verbal reinforcement learning for repository-level code completion, using a Reflector component to provide feedback between iterations. The paper finds that a general-purpose model (Meta-Llama-3-8B) outperforms a code-specialized model (CodeGen-Mono-6B) as the Reflector, and claims the full framework outperforms CodeT5+ 2B, CodeLlama-7b-hf, StarCoder, and CodeGemma on both RepoEval and their new RepoGenEval benchmark. However, the framework's Evaluator component uses ground-truth EM/ES scores during the iterative loop — oracle access that baselines do not receive — making the comparison fundamentally unfair.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'Our source code and benchmark will be publicly available' — future tense promise, not an actual release. No repository URL is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The RepoGenEval benchmark is described but not released. The promise is future tense ('will be publicly available'). The underlying repos are public GitHub projects, but the curated 1600-sample benchmark is not available. RepoEval is a pre-existing public benchmark they did not create."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, hardware descriptions, or dependency lists are provided anywhere in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. There is no README, no scripts, and no step-by-step guide to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (Tables 2-5) report only point estimates for EM and ES. No confidence intervals, error bars, or uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'significant improvements' and 'superior performance' but provides no statistical significance tests (no p-values, t-tests, or any other test). All comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2-5 report absolute EM and ES scores for all models and baselines, allowing direct computation of improvement magnitudes. For example, RepoGenReflex achieves 0.480 EM vs CodeGemma's 0.463 on RepoEval (Table 3), providing enough context to assess the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 200 randomly selected lines per repository (1600 total) with no justification for why this sample size is adequate. No power analysis or discussion of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviations, variance across seeds, or spread measures are reported anywhere."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against four baselines: CodeT5+ 2B, CodeLlama-7b-hf, StarCoder, and CodeGemma (Section 4.3, Tables 3-4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include CodeGemma (2024), StarCoder (2023), CodeLlama (2023), and CodeT5+ (2023). These are reasonably contemporary models for a 2024 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 presents an ablation study with three configurations: Full Model, without Reflector+Experience, and without Evaluator (Table 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used throughout: Exact Match (EM) and Edit Similarity (ES), as defined in Section 3.1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via EM and ES metrics. Human evaluation could have assessed code quality, usefulness, or correctness in context."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The Evaluator component computes EM/ES against ground truth during the iterative loop (Algorithm 1, lines 7-16). The ground truth labels are used to guide the optimization process (stopping criteria and Reflector feedback), meaning test data leaks into the generation pipeline. There is no separation between evaluation data and optimization data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-repository breakdowns are provided in all results tables (Tables 2-5), showing performance on each of the 8 repositories in both RepoEval and RepoGenEval."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are discussed. The paper presents only aggregate metrics without any qualitative analysis of where the framework fails or produces poor completions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement. The paper reports no configurations that failed, no approaches that were tried and abandoned, and no settings where the framework underperformed baselines."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'significant improvements' but no statistical significance tests are performed. The abstract also claims 'superior performance and effectiveness across standard code completion tasks' — the improvements over baselines are small (1-2% EM) and could be entirely attributable to oracle access via the Evaluator, not the framework's design."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation study (Section 4.4) makes causal claims about component contributions ('disabling Reflector and Experience led to a significant decline'). While the ablation design is structurally adequate, the Evaluator's oracle access to ground truth confounds all causal claims — improvements may be due to label leakage rather than component quality."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Repository-Level Code Completion' generally, but all experiments are on Python-only repositories. The abstract claims 'robustness and adaptability' without qualification. Results are on only 16 repos total (8 RepoEval + 8 RepoGenEval), all Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed improvements. The most obvious alternative — that the framework's advantage stems from oracle access to ground truth EM/ES during iteration rather than from the retrieval-generation design — is never acknowledged."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures EM and ES and claims these reflect code completion accuracy and quality. The metrics directly measure what is claimed (exact match and edit similarity to ground truth), so there is no significant proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided: CodeGen-Mono-6B, Meta-Llama-3-8B, CodeT5+ 2B, CodeLlama-7b-hf, StarCoder, and CodeGemma 7B. These are identifiable model checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figures 2 and 3 show schematic illustrations of the prompt construction and example feedback, but the actual prompt templates and fill values used in experiments are not provided. The reader cannot reconstruct the exact prompts sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM generation hyperparameters are reported (temperature, top-p, max tokens). Algorithm 1 specifies framework parameters (max_iter=10, no_improvement_threshold=3), but model inference settings are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The iterative framework (Retriever, Actor, Evaluator, Reflector, Experience) is described in detail in Section 3 with a workflow diagram (Figure 1), component descriptions, mathematical formulations, and Algorithm 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the RepoGenEval construction criteria: non-forked, open-source, 700-40000 stars, >90% Python, explicit unit tests, created no earlier than 2023. Line selection: 200 random lines per repo, unique, non-comment (1600 total samples)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section, threats-to-validity section, or any substantive discussion of the work's limitations anywhere in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not mention any specific concerns about its methodology, evaluation design, or generalizability."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No scope boundaries are stated. The paper does not explicitly discuss what the results do NOT show or what settings/populations are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (model outputs, per-sample results, benchmark files) are available for verification. Only aggregate metrics in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data collection for RepoGenEval: GitHub REST API and manual keyword searches for Python projects meeting specific criteria (stars, Python ratio, unit tests, creation date). Table 1 lists the 8 selected repositories with metadata."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are public GitHub repositories and a pre-existing benchmark (RepoEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While repo selection criteria are documented, the pipeline from selected repos to final benchmark samples is sparse. How random line selection was implemented, how 'unique' was defined, how lines were extracted for completion tasks, and how ground truth was constructed are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, grants, or sponsorship are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are from University of California, Davis. They do not evaluate a commercial product from their own institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be verified. Without a funding disclosure statement, we cannot assess whether any funder had a stake in the results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (CodeGen-Mono-6B, Meta-Llama-3-8B, CodeT5+ 2B, CodeLlama-7b-hf, StarCoder, CodeGemma)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the test repositories or code lines appeared in any model's training data. The RepoEval benchmark repos are from GitHub and could be in training sets of models trained on GitHub data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RepoEval was published in 2023 and uses GitHub repos; models like Meta-Llama-3 and CodeGemma trained on internet data may have seen it. RepoGenEval uses GitHub repos created in 2023, which could also be in training sets. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of code completion models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported. The iterative process runs up to 10 iterations per example, each involving Retriever, Actor, Evaluator, and Reflector LLM calls — potentially 40+ LLM calls per sample — but cost is never mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. No GPU hours, hardware specifications, or total experiment time are provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. All results appear to be from single runs without any sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single runs or averaged across multiple attempts."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Framework parameters (max_iter=10, no_improvement_threshold=3) appear chosen without justification, and no search over alternatives is documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 transparently compares two Reflector model choices (CodeGen-Mono-6B vs Meta-Llama-3-8B) and reports results for both (Table 2), justifying the selection of Meta-Llama-3-8B based on superior performance."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite making many cross-model and cross-repository comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own framework against baselines without acknowledging the self-comparison bias. Baselines are used as-is (from their JSONL files) while RepoGenReflex uses the full iterative pipeline, but this asymmetry is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RepoGenReflex uses up to 10 iterations with multiple LLM calls per iteration, while baselines use single-pass inference. This massive compute difference (potentially 10-40x more LLM calls) is never discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether EM/ES on random single-line completions actually measures useful code completion ability. No analysis of construct validity for either RepoEval or RepoGenEval."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RepoGenReflex wraps CodeGen-Mono-6B in an elaborate iterative scaffold (Retriever + Actor + Evaluator + Reflector + Experience) while baselines are standalone models. The comparison attributes differences to the framework but does not isolate the scaffold effect from the model effect. The baselines 'only utilized the JSONL file of our benchmark' while 'our framework leveraged all elements in the benchmark and the iterative loop' (Section 4.3)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark repositories (some from 2023) could have been in model training data. RepoGenEval uses repos created after 2023, which may help for some models, but this is not discussed as a leakage mitigation strategy."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Evaluator computes EM/ES against ground truth during the iterative loop and feeds these scores to the Reflector (Figure 2, Algorithm 1). This gives the framework oracle access to how close its output is to the answer — a direct form of label leakage during inference that baselines do not receive. This is never acknowledged."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether code in test repositories shares patterns, dependencies, or near-duplicates with other data used in the process."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference tests, n-gram overlap analysis, or temporal splits."
       }
     }
   },
   "claims": [
     {
       "claim": "Meta-Llama-3-8B is a better Reflector than CodeGen-Mono-6B for the RepoGenReflex framework",
       "evidence": "Table 2 shows Meta-Llama-3-8B consistently achieves higher EM and ES scores across all 8 RepoGenEval repositories (e.g., Auto-GPT-ZH EM 0.325 vs 0.280, ES 0.483 vs 0.438).",
       "supported": "moderate"
     },
     {
       "claim": "RepoGenReflex outperforms SOTA models (CodeT5+ 2B, CodeLlama-7b-hf, StarCoder, CodeGemma) on both RepoEval and RepoGenEval benchmarks",
       "evidence": "Tables 3-4 show RepoGenReflex achieving the highest EM and ES scores across repositories on both benchmarks. However, the comparison is fundamentally unfair: RepoGenReflex uses ground-truth EM/ES scores during its iterative loop (oracle access) while baselines perform single-pass inference.",
       "supported": "weak"
     },
     {
       "claim": "Each component (Reflector, Experience, Evaluator) is critical for the framework's performance",
       "evidence": "Ablation study in Table 5 shows performance degrades when Reflector+Experience are removed (EM drops ~0.12) and when Evaluator is removed (EM drops ~0.04). However, the ablation data for 'Model A' shows EM values decreasing in perfect 0.005 arithmetic steps across repos (0.365, 0.360, 0.355, 0.350, 0.345, 0.340), which is inconsistent with real experimental results.",
       "supported": "weak"
     },
     {
       "claim": "RepoGenReflex achieves 'significant improvements' in code completion accuracy",
       "evidence": "The improvements over the best baseline (CodeGemma) are small: ~1.7% EM on RepoEval (Table 3), ~1.4% EM on RepoGenEval (Table 4). No statistical significance tests are performed. The word 'significant' is used without statistical backing.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Oracle access to ground truth during inference",
       "detail": "The Evaluator component computes EM/ES against the ground truth answer during the iterative optimization loop (Algorithm 1, lines 7-16) and feeds these scores to the Reflector (Figure 2). This gives the framework direct knowledge of how close its output is to the correct answer — information that baselines do not receive. This makes the comparison fundamentally unfair and means the framework cannot work in practice (where the ground truth is unknown). The entire performance advantage could be attributed to this oracle signal rather than the retrieval-generation design."
     },
     {
       "flag": "Suspiciously regular experimental results",
       "detail": "In Table 5 (ablation), Model A EM values decrease in perfect 0.005 arithmetic steps across the first 6 repositories: 0.365, 0.360, 0.355, 0.350, 0.345, 0.340. Model A ES shows the identical pattern: 0.665, 0.660, 0.655, 0.650, 0.645, 0.640. Real experimental results on different repositories do not produce perfect arithmetic progressions. In Table 3, all models show remarkably tight clustering across 8 different repos (e.g., RepoGenReflex EM ranges only 0.476-0.481). These patterns are inconsistent with genuine experimental measurement."
     },
     {
       "flag": "Unfair baseline comparison (scaffold confound)",
       "detail": "RepoGenReflex uses an iterative pipeline with up to 10 iterations of Retriever + Actor + Evaluator + Reflector calls per sample, while baselines perform single-pass inference. The paper acknowledges this asymmetry ('Unlike the SOTA models, which only utilized the JSONL file of our benchmark, our framework leveraged all elements') but does not control for the massive difference in compute and oracle access."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All results are point estimates from apparently single runs. Claims of 'significant improvements' and 'superior performance' are made without any statistical tests, confidence intervals, or variance measures across the entire paper."
     },
     {
       "flag": "No limitations discussion",
       "detail": "The paper contains no limitations section, no threats to validity, and no discussion of the framework's weaknesses. The most critical limitation — that the Evaluator requires ground truth and thus the framework cannot function in real deployment — is never acknowledged."
     },
     {
       "flag": "No cost analysis despite expensive iterative design",
       "detail": "The iterative process involves up to 10 iterations per sample, each requiring multiple LLM calls (Retriever, Actor, Evaluator, Reflector). This could mean 40+ LLM calls per completion versus 1 for baselines. No cost, latency, or compute comparison is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["Fengji Zhang", "Bei Chen", "Yue Zhang", "Jin Liu", "Daoguang Zan", "Yi Mao", "Jian-Guang Lou", "Weizhu Chen"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Direct baseline for repository-level code completion using iterative RAG; RepoGenReflex extends this approach with verbal reinforcement learning."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2024,
       "relevance": "Foundational work on verbal reinforcement learning (VRL) that RepoGenReflex adapts for code completion; shows iterative LLM self-refinement without weight updates."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks; relevant to the survey's scope of agentic AI evaluation."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "Fabio Petroni", "Vladimir Karpukhin", "Naman Goyal"],
       "year": 2020,
       "relevance": "Foundational RAG paper that RepoGenReflex builds upon; establishes the retrieval-augmented generation paradigm used across LLM applications."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi", "Lifu Tu", "Huan Wang", "Yingbo Zhou", "Silvio Savarese", "Caiming Xiong"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "CodeGen-Mono-6B is the primary Actor model in the RepoGenReflex framework; key code generation model in the LLM programming space."
     },
     {
       "title": "RLCoder: Reinforcement Learning for Repository-Level Code Completion",
       "authors": ["Yanlin Wang", "Yanxian Wang", "Daya Guo", "Jiachi Chen", "Ruikai Zhang", "Yuchi Ma", "Zibin Zheng"],
       "year": 2024,
       "arxiv_id": "2407.19487",
       "relevance": "Concurrent work on reinforcement learning for repository-level code completion; directly relevant to the survey scope of LLM-based code generation."
     },
     {
       "title": "REPOFORMER: Selective Retrieval for Repository-Level Code Completion",
       "authors": ["Di Wu", "Wasi Uddin Ahmad", "Dejiao Zhang", "Murali Krishna Ramanathan", "Xiaofei Ma"],
       "year": 2024,
       "arxiv_id": "2403.10059",
       "relevance": "Addresses selective retrieval for repository-level code completion using RAG; directly comparable approach to RepoGenReflex."
     },
     {
       "title": "Competition-Level Code Generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung", "Nate Kushman", "Julian Schrittwieser"],
       "year": 2022,
       "relevance": "Major code generation system from DeepMind; demonstrates large-scale code generation capabilities relevant to the LLM programming survey."
     },
     {
       "title": "Repository-Level Prompt Generation for Large Language Models of Code",
       "authors": ["Disha Shrivastava", "Hugo Larochelle", "Daniel Tarlow"],
       "year": 2023,
       "relevance": "Proposes prompt generation methods for repository-level code completion; directly relevant to the repository-level code completion research area."
     },
     {
       "title": "LongCoder: A Long-Range Pre-Trained Language Model for Code Completion",
       "authors": ["Daya Guo", "Canwen Xu", "Nan Duan", "Jian Yin", "Julian McAuley"],
       "year": 2023,
       "relevance": "Addresses the long-range context challenge in code completion; relevant to understanding LLM approaches to repository-level code understanding."
     },
     {
       "title": "Cognitive Architectures for Language Agents",
       "authors": ["Theodore R. Sumers", "Shunyu Yao", "Karthik Narasimhan", "Thomas L. Griffiths"],
       "year": 2023,
       "arxiv_id": "2309.02427",
       "relevance": "Foundational work on cognitive architectures for LLM-based agents; relevant to the survey's coverage of agentic AI frameworks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The concept of iterative refinement for code completion is practically relevant, but the framework requires ground truth during inference, making it unusable in practice, and no code is released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that iterative refinement with feedback improves model outputs; does not challenge any conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security implications; this is a code completion accuracy paper."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with existing work or claims."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or benchmark released despite promises; nothing for anyone to try."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From UC Davis, not a major AI lab; does not evaluate or involve any high-profile product."
     }
   }
 }