ai-research-survey

scan.json (24537B)

---

 {
   "paper": {
     "title": "CODEPROMPTZIP: Code-specific Prompt Compression for Retrieval-Augmented Generation in Coding Tasks with LMs",
     "authors": ["Pengfei He", "Shaowei Wang", "Tse-Hsun (Peter) Chen"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2502.14925"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides an anonymous repository link (https://anonymous.4open.science/r/CodePromptZip-6B2B) in Appendix A ('Data Availability') and the Ethical Considerations section states 'Both the source code and data are released as free and open-source software.' However, this is an anonymized link for review — whether it is a fully working URL is uncertain, but it constitutes a provided link."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in the Ethical Considerations section that 'Both the source code and data are released as free and open-source software and are made available in the public domain.' The datasets used (Assertion Generation, Bugs2Fix/CodexGLUE, Code Suggestion) are also based on publicly available benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies the base model (CodeT5, 775M parameters) and mentions using AdamW optimizer with specific hyperparameters, but does not provide a requirements.txt, Dockerfile, or detailed environment setup section listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides an anonymous repository link but does not include step-by-step reproduction instructions in the paper itself. No README with commands or 'Reproducing Results' section is present in the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, 5, and 6 are reported as point estimates without confidence intervals, error bars, or any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements of 23.4%, 28.7%, and 8.7% over baselines but provides no statistical significance tests (no p-values, t-tests, or other tests). Differences are compared by raw number comparison only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements over baselines with baseline context. For example, '23.4%, 28.7%, and 8.7% over the best baseline' and Table 2 shows the actual metric values (e.g., CODEPROMPTZIP: 42.1% Exact Match vs. LongLLMLingua: 34.1%), providing enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper randomly samples 2,000 instances from validation and test sets 'due to computational resource constraints' (Sec. 3.2) but does not justify why 2,000 is sufficient or discuss whether this sample is representative."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs (temperature set to 0 for BLMs, but no mention of multiple training runs for the compressor model)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against four baselines: LLMLingua, LongLLMLingua, LLMLingua-2, and RECOMP, as well as w/o retrieval and Oracle conditions (Table 2, Sec. 5.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are contemporary: LLMLingua (2023), LongLLMLingua (2023), LLMLingua-2 (2024), and RECOMP (2024). These represent the state of the art in prompt compression at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes an ablation study comparing CODEPROMPTZIP with and without the copy mechanism (Table 2: 'CODEPROMPTZIP w/o Copy' vs. 'CODEPROMPTZIP'), demonstrating the contribution of the copy mechanism across all three tasks."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Different metrics are used for different tasks (Exact Match for Assertion Generation, CodeBleu for Bugs2Fix and Code Suggestion), and token count/compression ratio are reported alongside. However, each individual task uses only one quality metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated using Exact Match and CodeBleu metrics. For code compression quality, human judgment of compressed code readability or usefulness would be informative but is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses original train/validation/test splits from each dataset (Sec. 3.2: 'we follow the original split of the dataset into Train, Validation, and Test partitions'). The validation set is used for ablation analysis and the test set for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per task (Assertion Generation, Bugs2Fix, Code Suggestion) in all tables. Per-model breakdowns are provided in RQ4 (Figure 6). Varying compression ratios are shown in Table 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure cases or error analysis is provided. The paper does not show examples of where compression produced poor results or where the compressor failed to meet the desired ratio."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the w/o Copy variant struggles with compression ratio control (Sec. 6.3, Table 6), and that cross-task compressors show degraded performance (Appendix D, Table 5). It also notes performance degradation vs. uncompressed prompts."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims improvements of 23.4%, 28.7%, and 8.7% over the best baseline for the three tasks. These numbers are confirmed in Table 2 (comparing CODEPROMPTZIP to the best baseline in each task: 42.1 vs 34.1, 61.9 vs 48.1, 23.7 vs ~21.8)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation (removing the copy mechanism degrades performance). The ablation design is adequate: controlled single-variable manipulation comparing w/ and w/o copy on the same datasets and settings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and abstract claim compression 'for Retrieval-Augmented Generation in Coding Tasks' broadly, but experiments are only on Java, method-level tasks, and 3 specific benchmarks. The Limitations section acknowledges this (Java only, method-level), but the title and abstract do not qualify the scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For example, it does not consider whether the improvements are due to the type-aware priority ranking vs. the copy mechanism architecture vs. the training data construction, beyond the basic ablation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-3.5-turbo' and 'Gemini-1.0-pro' without specifying snapshot dates or API versions. It also uses 'CodeLlama-13B' which is more specific but still lacks version details. The compressor is 'CodeT5 (Wang et al., 2021), 775M' which is reasonably specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 (Appendix) shows the prompt templates for all three coding tasks with the actual structure including placeholders (e.g., {header}, {body}, {method under test}). The fill values are the actual code examples from the datasets, which are publicly available."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training hyperparameters are reported: 'AdamW optimizer with a batch size of 16, a learning rate of 5e-5, and 1,000 warmup steps for 10 epochs' (Sec. 4.2). BLM temperature is set to 0 (Sec. 3.2). Compression ratios are explicitly stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a straightforward pipeline: retrieve examples, compress with trained model, pass to BLM for generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: code examples are categorized into parsable/unparsable using JavaParser (Table 1 shows counts), 2,000 instances randomly sampled from validation/test sets (Sec. 3.2), token types categorized via AST construction, and the compression dataset construction is detailed in Algorithm 1 and Appendix C."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is titled 'Limitations' and contains substantive discussion of two specific limitations: the need for extra training for new coding tasks and the limited generalizability to Java/method-level tasks only."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section discusses specific threats: the approach requires retraining for different coding tasks because removal priorities are task-dependent, and experiments are limited to Java and method-level tasks. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states: 'This study focuses exclusively on Java and method-level tasks' and notes that applying to repository-level tasks would require additional training. It encourages 'further studies to explore additional base LMs and a broader range of programming languages.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package is stated to be available (Appendix A) and the underlying datasets (CodexGLUE/Bugs2Fix, Assertion Generation, Code Suggestion) are public benchmarks. The paper states 'Both the source code and data are released.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data sources are clearly described: Assertion Generation from Nashid et al. 2023, Bugs2Fix from CodexGLUE (Lu et al. 2021), Code Suggestion from Chen et al. 2024. Table 1 provides dataset statistics with knowledge base, test, and validation sizes. The compression dataset construction is described in detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks, making this criterion not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented: code examples parsed with JavaParser, tokens categorized by type, ablation analysis to determine removal priorities, Algorithm 1 for constructing compressed training examples, and the training/validation/test split (8:1:1 ratio, Appendix C Table 4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. There is no acknowledgments section disclosing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Manitoba (He, Wang) and Concordia University (Chen). These are academic institutions, not companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of funder cannot be assessed. The absence of funding disclosure makes this a NO."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-3.5-turbo, Gemini-1.0-pro, and CodeLlama-13B as base LMs for evaluation but does not state their training data cutoff dates. This is relevant because the benchmarks used could have been in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the LLMs' training data and the benchmark test sets. CodexGLUE/Bugs2Fix was published in 2021, well before GPT-3.5-turbo's training, raising contamination concerns that are not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks (CodexGLUE, etc.) were published before the models' training cutoffs, making contamination a real concern, but the paper does not discuss this risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper motivates the work partly by referencing GPT-4 API costs ($2.50 per million tokens) in the introduction, and reports token counts and compression ratios, but does not report the actual inference cost or latency of running the compressor itself or the end-to-end pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about GPU hours, total API spend, or hardware used for training the compressor or running the BLM experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "CODEPROMPTZIP improves by 23.4%, 28.7%, and 8.7% over the best baseline for Assertion Generation, Bugs2Fix, and Code Suggestion respectively.",
       "evidence": "Table 2 shows CODEPROMPTZIP achieves 42.1% Exact Match vs 34.1% (LongLLMLingua) for Assertion Generation, 61.9% CodeBleu vs 48.1% (LLMLingua-2) for Bugs2Fix, and 23.7% CodeBleu vs ~21.8% (LLMLingua) for Code Suggestion.",
       "supported": "moderate"
     },
     {
       "claim": "The copy mechanism consistently contributes to performance enhancement across all tasks.",
       "evidence": "Table 2 ablation shows w/o Copy vs w/ Copy: 40.9% vs 42.1% Exact Match (Assertion Generation), 56.7% vs 61.9% CodeBleu (Bugs2Fix), 20.5% vs 23.7% CodeBleu (Code Suggestion). Consistent improvement across all tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Token type removal priorities are task-specific yet model-agnostic, applicable across different LMs.",
       "evidence": "Figure 1 shows priority rankings are consistent across GPT-3.5-turbo and Gemini-1.0-pro for all three tasks, but the rankings differ between tasks (e.g., Invocation highest priority for Assertion Generation, Signature highest for Code Suggestion).",
       "supported": "moderate"
     },
     {
       "claim": "Fewer, less-compressed examples generally outperform more, highly-compressed examples within a fixed token budget.",
       "evidence": "Figure 4 (RQ2) shows that 1-shot with lower compression ratio generally achieves better performance than 3 or 5 shots with higher compression, across all three tasks.",
       "supported": "moderate"
     },
     {
       "claim": "CODEPROMPTZIP generalizes across different base LMs (CodeLlama-13B, Gemini-1.0-pro).",
       "evidence": "Figure 6 (RQ4) shows CODEPROMPTZIP consistently outperforms baselines across CodeLlama-13B and Gemini-1.0-pro on all three tasks.",
       "supported": "moderate"
     },
     {
       "claim": "CODEPROMPTZIP handles unparsable code effectively with minimal performance degradation.",
       "evidence": "Table 3 (RQ5) shows only slight decreases when code is artificially made unparsable: 42.1% to 42.0% (1% removed) and 42.1% to 41.7% (3% removed) for Assertion Generation. Oracle is N/A for unparsable code.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CODEPROMPTZIP introduces a code-specific prompt compression framework for RAG-based coding tasks that uses type-aware token priority ranking and a copy-enhanced CodeT5 compressor. The framework outperforms existing NL-focused compression methods (LLMLingua, LLMLingua-2, RECOMP) by 8.7-28.7% across three Java coding tasks. The copy mechanism is critical for both compression quality and ratio control. The approach generalizes across different base LMs and handles unparsable code, though it is limited to Java method-level tasks and requires task-specific retraining.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All claimed improvements are based on raw number comparisons with no significance tests, confidence intervals, or variance reporting. The improvements could be within noise, especially for Code Suggestion (8.7% over baseline)."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "CodexGLUE (Bugs2Fix) was published in 2021 and could be in the training data of GPT-3.5-turbo and Gemini-1.0-pro. This is never discussed, yet the absolute performance numbers (which determine whether compression 'maintains' quality) depend on this."
     },
     {
       "flag": "No inference cost analysis despite cost motivation",
       "detail": "The paper motivates the work by citing GPT-4 costs ($2.50/M tokens) but never reports the actual cost of running the compressor, the end-to-end latency, or the GPU resources needed to train/deploy the 775M parameter compressor. The cost savings from compression could be offset by compressor costs."
     },
     {
       "flag": "Subsampled test sets without justification",
       "detail": "Only 2,000 instances are randomly sampled from test sets (originally 6,545-18,027) 'due to computational resource constraints' with no justification that this sample is representative or large enough for reliable conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "LLMLingua: Compressing Prompts for Accelerated Inference of Large Language Models",
       "authors": ["Huiqiang Jiang", "Qianhui Wu", "Chin-Yew Lin", "Yuqing Yang", "Lili Qiu"],
       "year": 2023,
       "relevance": "Key baseline for prompt compression; entropy-based approach for reducing prompt length while preserving LM performance."
     },
     {
       "title": "LLMLingua-2: Data Distillation for Efficient and Faithful Task-Agnostic Prompt Compression",
       "authors": ["Zhuoshi Pan", "Qianhui Wu", "Huiqiang Jiang"],
       "year": 2024,
       "relevance": "Knowledge distillation baseline for prompt compression using transformer encoder for token classification."
     },
     {
       "title": "RECOMP: Improving Retrieval-Augmented LMs with Context Compression and Selective Augmentation",
       "authors": ["Fangyuan Xu", "Weijia Shi", "Eunsol Choi"],
       "year": 2024,
       "relevance": "Distillation-based baseline that trains T5 on GPT-3.5-turbo summaries for retrieval context compression."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven CH Hoi"],
       "year": 2021,
       "relevance": "Base model architecture used for the code compressor; foundational pre-trained model for code tasks."
     },
     {
       "title": "Retrieval-based Prompt Selection for Code-related Few-shot Learning",
       "authors": ["Noor Nashid", "Mifta Sintaha", "Ali Mesbah"],
       "year": 2023,
       "relevance": "Provides the Assertion Generation task and RAG prompt template used in evaluation."
     },
     {
       "title": "CodexGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation",
       "authors": ["Shuai Lu", "Daya Guo", "Shuo Ren"],
       "year": 2021,
       "relevance": "Provides the Bugs2Fix benchmark dataset used in evaluation; foundational code benchmark."
     },
     {
       "title": "Code Search is All You Need? Improving Code Suggestions with Code Search",
       "authors": ["Junkai Chen", "Xing Hu", "Zhenhao Li", "Cuiyun Gao", "Xin Xia", "David Lo"],
       "year": 2024,
       "relevance": "Provides the Code Suggestion task and dataset used in evaluation."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used as one of the base LMs for evaluating transferability of compressed prompts."
     },
     {
       "title": "Less is More: Docstring Compression in Code Generation",
       "authors": ["Guang Yang", "Yu Zhou", "Wei Cheng"],
       "year": 2024,
       "arxiv_id": "2410.22793",
       "relevance": "Related work on compressing natural language parts (docstrings) in coding task prompts, complementary to code compression."
     },
     {
       "title": "Natural is the Best: Model-Agnostic Code Simplification for Pre-trained Large Language Models",
       "authors": ["Yan Wang", "Xiaoning Li", "Tien N Nguyen", "Shaohua Wang"],
       "year": 2024,
       "relevance": "Provides the token type taxonomy (Symbol, Signature, Invocation, Identifier, Structure) used to categorize code tokens."
     }
   ]
 }