ai-research-survey

scan.json (29746B)

---

 {
   "paper": {
     "title": "Prompt-Based Code Completion via Multi-Retrieval Augmented Generation",
     "authors": [
       "Hanzhuo Tan",
       "Qi Luo",
       "Ling Jiang",
       "Zizheng Zhan",
       "Jing Li",
       "Haotian Zhang",
       "Yuqun Zhang"
     ],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2405.07530",
     "doi": "10.1145/3725812"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ProCC, a multi-retrieval augmented code completion framework using prompt engineering and contextual multi-armed bandits (LinUCB), outperforms prior RAG-based approaches by 8.6% and 10.1% Exact Match on open-source and private-domain benchmarks respectively. Prompt-based multi-retrievers (lexical semantics, hypothetical line, code summarization) achieve comparable performance to dedicated encoding models without additional training. The LinUCB adaptive selection algorithm significantly outperforms simpler alternatives including direct concatenation, max similarity, and logistic regression. Applying ProCC to a fine-tuned Code Llama yields an additional 5.6% EM improvement.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states 'we provide the repository [1] for all the other available materials, including the source code of the artifact and the open-source dataset.' Reference [1] links to https://github.com/anonepo/issta2024pcc. However, this appears to be an anonymized review-period URL and may not be permanently accessible."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The open-source benchmark dataset (20 GitHub repositories) is stated to be included in the repository. The private-domain benchmark from the e-commerce company is explicitly withheld: 'the dataset containing the private-domain code of the company shall remain undisclosed' (Section 8)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8 NVIDIA A100-80GB GPUs', 'Text-Generation-Inference framework', Python, and Faiss, but provides no requirements.txt, Dockerfile, or detailed dependency listing with library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While a repository is referenced, the paper itself provides no commands, README description, or scripts for replicating the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (Tables 1, 3, 5, 6) report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper repeatedly claims ProCC 'outperforms' and 'significantly improves' over baselines based solely on comparing point estimates. No p-values, t-tests, or any statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context throughout, e.g., '14.1% EM improvement over the vanilla model (from 47.90 to 54.66)' and '8.6% EM improvement over the previous SOTA technique RepoCoder (from 50.31 to 54.66)' in Section 4.2.1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set sizes (3317 open-source, 3074 private-domain) are stated but not justified. No power analysis or justification for why these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The Threats to Validity section states 'we averaged results over five runs, reducing variance,' but no standard deviations, interquartile ranges, or spread measures are reported anywhere in the results tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against BM25, ReACC (ACL 2022), and RepoCoder (EMNLP 2023), plus a vanilla base model, across both benchmark suites (Tables 1, 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "RepoCoder (EMNLP 2023) and GTE-large (August 2023) for the ReACC re-implementation are contemporary baselines for a 2024 paper. The paper states RepoCoder is the SOTA technique."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 4.2.2) provides extensive ablations: individual retriever perspectives (Table 4), perspective combinations (Table 5), and selection algorithm comparisons (Table 6). The Venn diagram (Figure 3) shows unique contributions of each retriever."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used throughout: Exact Match (EM) and Edit Similarity (ES), as stated in Section 4.1.4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. All evaluation is automated using EM and ES metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 describes a clear split: '10% of the files as the test set, 10% for validation used in training the adaptive retrieval selection algorithm, and the rest as retrieval data.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task type (Function Body vs. Random Line) in Tables 1 and by domain (open-source vs. private-domain) in Tables 1 and 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While Figure 1 shows motivating scenarios, no systematic error analysis or failure case discussion is provided for ProCC's outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every configuration and combination of ProCC shows improvement. The paper notes that direct concatenation ('Union') provides only marginal gain (Table 6) and that expanding to 6 perspectives yields marginal improvement over 3, but these are framed as validation of design choices rather than negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 8.6% improvement on open-source (Table 1: 50.31→54.66), 10.1% on private-domain (Table 3: 41.84→46.07), and 5.6% over fine-tuned (Figure 4: 60.75→64.17) are all supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation studies (Tables 5, 6) that systematically add/remove components and selection algorithms, providing adequate controlled single-variable manipulation for claims like 'incorporating varied perspectives enriches representations.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title claims 'Code Completion' broadly, yet the evaluation covers only Java code repositories (visible from code examples). The programming languages used in the 20 open-source and 58 private repositories are never explicitly stated, and no scope boundaries on language or domain are articulated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Threats to Validity section (Section 5) addresses implementation bugs, benchmark selection, and randomness, but does not consider alternative explanations for the results (e.g., whether improvements come from additional context length rather than multi-perspective semantics, or whether the LinUCB simply learns input difficulty)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Exact Match and Edit Similarity and claims 'code completion effectiveness' — these metrics directly measure code completion quality at the token level without overreaching into broader framing like 'developer productivity.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 specifies 'StarCoderBase 15.5B' and 'Code Llama 13b-Instruct' — these are open-source models with specific architecture identifiers (parameter count + variant), obtained from Hugging Face."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Section 3.2 and Table 4: 'Embedding the following code snippets: [code]', '<PRE> [Prefix] <SUF> [Suffix] <MID>', and 'This code snippets of [code] means'. The placeholders ([code], [Prefix], [Suffix]) represent the deterministic input (the incomplete code), so the full prompts are reconstructible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.5 reports: LinUCB α=0.1, greedy decoding, fine-tuning batch_size=12, learning_rate=2e-5, AdamW optimizer, 2 epochs. Greedy decoding minimizes generation randomness."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. ProCC is a retrieval-augmented generation pipeline with no agent loops, tool use, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 documents: following RepoCoder's protocol to crawl 20 GitHub repos, 10%/10%/80% test/validation/retrieval split, random line selection (3 per file), function extraction for all test files, yielding 3317 and 3074 test instances."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' provides a substantive discussion organized into internal, external, and construct validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific mitigations: using original source code and identical hyperparameters for baselines (internal), including industry data unknown to LLMs alongside established benchmarks (external), averaging over five runs to reduce randomness (external), and using two established metrics (construct)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show — no mention of untested languages, model families, or completion scenarios that fall outside the evaluation scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The open-source benchmark data is stated to be available in the GitHub repository [1]. Private-domain data is withheld but this limitation is explicitly disclosed in Section 8."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 describes: crawling 20 high-quality GitHub repositories following RepoCoder's protocol, same protocol applied to 58 private-domain repositories from an e-commerce company with ~1 billion MAU."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are GitHub repositories (standard public data) and private company repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 documents the pipeline: crawl repos → split files (80/10/10) → select test cases (3 random lines per file for RL, all functions for FB) → final counts (3317 open-source, 3074 private-domain instances)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section, acknowledgments, or grant information appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Southern University of Science and Technology, The Hong Kong Polytechnic University, and Kwai Inc. The Kwai affiliation is relevant given the private-domain evaluation likely uses Kwai-related data."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two authors (Zizheng Zhan, Haotian Zhang) are affiliated with Kwai Inc., and the private-domain benchmark comes from 'an e-commerce company with around one billion Monthly Active Users' — likely Kwai or a partner. This creates a potential conflict where the company has interest in showing its data benefits from the proposed technique."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states StarCoder is 'trained on 1 trillion tokens from GitHub' and Code Llama on '500B code data' but does not specify training data cutoff dates for either model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 explicitly acknowledges: 'Given that LLMs are pre-trained on expanded GitHub datasets, it might inadvertently encompass elements from our test set and lead to the risk of test set contamination.' They construct a private-domain benchmark as mitigation."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper constructs a private-domain benchmark from proprietary code specifically to mitigate contamination risk: 'To alleviate this issue, we construct another benchmark based on private-domain code from an e-commerce company' (Section 4.1.3)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 reports: 'ProCC is considerably more resource-efficient and operable on a single A100 GPU' with 'embedding and search processes aggregating to approximately 0.5 seconds on the same device.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.5 states: 'all experiments are performed on a cluster equipped with 8 NVIDIA A100-80GB GPUs.' Fine-tuning takes 'approximately 9.5 hours on the 8×A100 cluster' (Section 4.2.3)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'averaged results over five runs' (Section 5) but reports no variance, standard deviation, or sensitivity analysis across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (External Validity) states: 'we averaged results over five runs, reducing variance.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "LinUCB α is set to 0.1 and fine-tuning hyperparameters are specified, but no search budget is reported. The paper mentions 'multiple reasonable training setups' with results in the GitHub repo (footnote 1) but does not quantify the search in the paper."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 describes a clear validation set (10% of files) used for training the LinUCB algorithm, separate from the test set used for final evaluation."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement ReACC with a different encoding model (GTE-large instead of the original) since 'it does not provide complete reproducible encoding models for retrieval' (Section 4.1.2). This substitution could bias results but is not acknowledged as a potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While the paper compares ProCC's cost (~0.5s on 1 A100) to fine-tuning (9.5h on 8×A100), performance is not systematically reported as a function of compute budget, and no performance-compute curves are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Exact Match and Edit Similarity without discussing whether these metrics capture meaningful code completion quality (e.g., whether exact match is too strict or whether edit similarity handles semantic equivalence)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. ProCC is a retrieval pipeline, not a scaffold-dependent system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges contamination risk but does not analyze temporal relationships between when the 20 GitHub repositories were created/published and the model training data cutoff dates."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context (e.g., whether the retrieval database provides hints that wouldn't be available in real deployment)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the training/retrieval data and test data from the same repositories share structural similarities or near-duplicate code patterns that could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied. The private-domain benchmark is a contamination mitigation strategy but not a detection method for the open-source benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "ProCC outperforms state-of-the-art code completion technique by 8.6% on the open-source benchmark in terms of Exact Match.",
       "evidence": "Table 1 shows ProCC achieves 54.66 average EM on Code Llama vs RepoCoder's 50.31 on the open-source benchmark suite (Section 4.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "ProCC outperforms SOTA by 10.1% on the private-domain benchmark in terms of Exact Match.",
       "evidence": "Table 3 shows ProCC achieves 46.07 average EM on Code Llama vs BM25's 41.84 on the private-domain benchmark suite (Section 4.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "Prompt-based retrievers are robust across templates and comparable to external encoders.",
       "evidence": "Table 4 shows prompt variations produce EM scores between 50.41-51.61, comparable to ReACC (50.31) and RepoCoder (50.31) which use dedicated encoding models (Section 4.2.2).",
       "supported": "moderate"
     },
     {
       "claim": "Combining multiple retrieval perspectives outperforms individual perspectives.",
       "evidence": "Table 5 shows combinations (52.43-54.88 EM) consistently outperform the best single perspective (51.61 EM for hypothetical line). Table 6 shows LinUCB (54.66) beats Union (51.97), Max Similarity (52.37), and Logistic Regression (53.21).",
       "supported": "moderate"
     },
     {
       "claim": "ProCC improves fine-tuned models by 5.6% in a plug-and-play manner.",
       "evidence": "Figure 4 shows fine-tuned Code Llama achieves 60.75 EM; adding ProCC increases it to 64.17 on the open-source benchmark (Section 4.2.3).",
       "supported": "moderate"
     },
     {
       "claim": "ProCC shows even more significant enhancement on private-domain data where LLMs are limited.",
       "evidence": "Table 3 shows a 36.8% EM increase over baseline (33.67 to 46.07) on private-domain, compared to 14.1% on open-source (Section 4.2.1). This is consistent with RAG benefiting more when model knowledge is limited.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims ProCC 'significantly improves' and 'outperforms' baselines throughout, but provides no statistical tests (p-values, confidence intervals, or significance tests). All comparisons are based on point estimates from averaged runs without reported variance, making it impossible to determine whether differences are statistically meaningful."
     },
     {
       "flag": "Variance not reported despite averaging over multiple runs",
       "detail": "The paper states results are 'averaged over five runs' but never reports standard deviations or variance. This makes it impossible to assess result stability or whether the reported improvements are within the noise margin."
     },
     {
       "flag": "Baseline re-implementation with different model",
       "detail": "ReACC is re-implemented using GTE-large instead of its original encoding model because 'it does not provide complete reproducible encoding models for retrieval.' This substitution could systematically bias the comparison, as the original model may perform differently."
     },
     {
       "flag": "Potential industry conflict of interest",
       "detail": "Two authors are from Kwai Inc., and the private-domain benchmark appears to come from Kwai's codebase. The paper shows larger improvements on private-domain data (10.1%) than open-source (8.6%), which is the more favorable narrative for industry adoption. No conflict of interest statement is provided."
     },
     {
       "flag": "Anonymized repository URL may be inaccessible",
       "detail": "The code repository URL (https://github.com/anonepo/issta2024pcc) appears to be a temporary anonymized repository for review, raising concerns about long-term artifact availability."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and the HumanEval benchmark, foundational for LLM code generation evaluation."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM used as a base model in this paper's experiments."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used as a base model in this paper's experiments, featuring fill-in-the-middle pre-training."
     },
     {
       "title": "ReACC: A Retrieval-Augmented Code Completion Framework",
       "authors": ["Shuai Lu", "Nan Duan"],
       "year": 2022,
       "relevance": "Pioneering RAG-based code completion technique using dual-encoder retrieval, key baseline."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["Fengji Zhang", "Bei Chen"],
       "year": 2023,
       "relevance": "State-of-the-art RAG code completion technique using iterative retrieval, primary baseline and benchmark protocol source."
     },
     {
       "title": "CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion",
       "authors": ["Yangruibo Ding", "Zijian Wang"],
       "year": 2023,
       "relevance": "Cross-file code completion benchmark relevant to repository-level evaluation."
     },
     {
       "title": "RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems",
       "authors": ["Tianyang Liu", "Canwen Xu"],
       "year": 2023,
       "relevance": "Repository-level code completion benchmark for evaluating completion systems."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive nlp tasks",
       "authors": ["Patrick Lewis", "Ethan Perez"],
       "year": 2020,
       "relevance": "Foundational RAG paper establishing the retrieval-augmented generation paradigm used in this work."
     },
     {
       "title": "InCoder: A Generative Model for Code Infilling and Synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan"],
       "year": 2022,
       "relevance": "Code infilling model that established fill-in-the-middle benchmarks for code completion."
     },
     {
       "title": "UniXcoder: Unified Cross-Modal Pre-training for Code Representation",
       "authors": ["Daya Guo", "Shuai Lu"],
       "year": 2022,
       "relevance": "Code representation model used as the encoding model for RepoCoder baseline."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "relevance": "Pioneering pre-trained code model for code understanding tasks."
     },
     {
       "title": "Codet5+: Open code large language models for code understanding and generation",
       "authors": ["Yue Wang", "Hung Le"],
       "year": 2023,
       "arxiv_id": "2305.07922",
       "relevance": "Open-source code LLM advancing code understanding and generation capabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "ProCC is a directly usable RAG technique for code completion that works with off-the-shelf models and requires no additional training of encoding models."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that prompts can replace dedicated encoding models for retrieval is modestly novel, but multi-perspective retrieval being better than single is expected."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on code completion accuracy."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; a straightforward technical contribution."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released via GitHub but requires significant infrastructure (A100 GPU, model downloads) to run."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses well-known models (Code Llama, StarCoder) but authors are from relatively less prominent institutions (SUSTech, PolyU, Kwai)."
     }
   }
 }