ai-research-survey

scan.json (32010B)

---

 {
   "paper": {
     "title": "ShadowCode: Towards (Automatic) External Prompt Injection Attack against Code LLMs",
     "authors": [
       "Yuchen Yang",
       "Yiming Li",
       "Hongwei Yao",
       "Bingrun Yang",
       "Yiling He",
       "Tianwei Zhang",
       "Dacheng Tao",
       "Zhan Qin"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2407.09164"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ShadowCode generates concise 12-token non-functional perturbations that manipulate Code LLMs into producing attacker-specified malicious code during completion, achieving up to 97.9% ASR on open-source models (CodeGemma-2b/7b, CodeGeeX2-6b) and over 90% on commercial tools (GitHub Copilot, CodeGeeX application). The attack transfers effectively from open-source surrogate models to black-box commercial systems, and resists potential defenses including semantic analysis and character removal. Ablation studies show that forward reasoning enhancement and keyword-based perturbation design are both critical components.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The code is available at https://github.com/LianPing-cyber/ShadowCodeEPI' providing a working repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: Humaneval, Humaneval-x, CodeXGLUE, MBPP, and Eval-Plus (Table 1). The 13 malicious objectives and 31 threat cases are described in Section 4.1 and Appendix A with enough detail to reconstruct."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'NVIDIA RTX A6000 Graphics Card' and 'float16 precision format without quantization' but provides no requirements.txt, Python version, library versions, or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and hyperparameters are detailed in Section 4.1, the paper itself contains no step-by-step reproduction instructions, no README-style commands, and no 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-4 and Figures 6, 10 are reported as point estimates (e.g., '82.0%', '86.6%') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ShadowCode 'significantly outperforms the other baseline methods' (Section 4.2) based solely on comparing ASR numbers. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute ASR values with baseline context throughout, e.g., Table 2 shows ShadowCode at 82.0-86.6% vs C-GCG at 41.9-44.6% and IPI-M at 11.2-26.2%, and Table 3 provides NBR/STF comparisons across methods."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 14,273 data items across 7 sub-datasets (Table 1) and 620 manually tested items for commercial evaluation, but never justifies why these sizes were chosen or discusses statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported. The random seed is fixed at 100 (Section 4.1) suggesting single-run results. No multi-run aggregation is described."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes two baselines: C-GCG (adapted from GCG [26]) and IPI-M (manually designed IPI based on [18]). A 'No Attack' reference is also included in Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GCG (Zou et al., 2023) and indirect prompt injection (Greshake et al., 2023) are recent and represent the closest existing work for this attack paradigm."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 provides ablation studies for keyword-based design (0 vs 1 vs 2 keywords, Figure 10), forward reasoning enhancement (Figure 11), and Top-k value effects. Each component's contribution is measured independently."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses five evaluation metrics: ASR (attack success rate), NBR (not bad cases rate), STF (selective threat factor), NIT (number of injected tokens), and NIC (number of injected chars), defined in Section 4.1 and Equation 7."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated. ASR is measured by functional equivalence to target code (Section 4.1). No human evaluation of attack outputs is performed. The commercial evaluation (Section 4.3) involves manual testing of 620 items but this is manual execution, not human judgment of output quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Perturbations are optimized on simulated code contexts (Section 3.3) using a surrogate model, then tested on separate code completion benchmark datasets (Humaneval, MBPP, etc.). The optimization and evaluation data are distinct."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks results down by model, dataset, and programming language. Table 3 provides per-language NBR/STF. Figure 14 shows per-case ASR across all 13 objectives. Table 4 separates gray-box and black-box scenarios."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 notes 'ShadowCode is relatively less effective on CodeXGLUE' due to data complexity. Appendix B discusses that ST2 is 'the most challenging' case and CWE22 is 'difficult for attackers to achieve.' Figure 14 shows per-case variation including low-performing cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports that 0-keyword perturbation 'struggles to achieve an ASR above 80%' even with high computational power. Figure 11 shows low ASR (<20%) without FRE even with low loss. Section 4.2 discusses reduced effectiveness on complex datasets."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'up to 97.9% ASR' on open-source models and 'over 90% ASR' on commercial applications are supported by Table 2 (CodeGemma-7b Python at 97.9%) and Table 4 (CodeGeeX Python at 93.3%, Copilot Python at 90.4%). The '12-token' claim is supported by the default 10-token perturbation + 2 keywords."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies in Section 4.5 make causal claims (FRE and keyword-based design improve ASR). The ablation design uses controlled single-variable manipulation: Figures 10-11 isolate individual components while holding others constant, which is adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion states 'Our work alerted to a new serious security threat in the use of Code LLMs' broadly, but testing covers only 3 open-source models from 2 families and 2 commercial applications. The title claims about 'Code LLMs' generally, beyond the 5 systems tested. Modern models like GPT-4, Claude, or Llama-based code models are not evaluated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for why the attack succeeds. For example, it does not discuss whether the success is due to specific model architectures, training data composition, or inherent properties of autoregressive generation. Section 5 discusses limitations but not alternative explanations for the observed results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "ASR measures whether generated code has equivalent functionality to the target malicious code at the correct position (Section 4.1). This directly corresponds to the paper's claim of achieving malicious objectives — the proxy and outcome are tightly aligned."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 specifies exact model versions: CodeGemma-2b, CodeGemma-7b, and CodeGeeX2-6b. For commercial applications, CodeGeeX and GitHub Copilot are named, though the paper acknowledges the exact underlying model versions are unknown (which is the point of gray/black-box evaluation)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides example code contexts (Figures 3, 5, 7, 13) but not the full set of code contexts/inputs used across all 31 threat cases and 14,273 data items. The code simulation structure is described algorithmically but complete prompt texts are not provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports all key hyperparameters: Top-k=400, r=0.4, h=2, keyword count=2, perturbation length=10, random seed=100, 5 noisy code samples, float16 precision, beam search size k for Algorithm 1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates direct code completion by Code LLMs, not multi-step agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents code contextual simulation construction (conditional code, noise code, position/target code). Section 4.1 and Table 1 detail how malicious objectives are converted into 31 threat cases across 3 languages, with dataset breakdowns (14,273 total items: 2,132 C/C++, 7,965 Python, 4,176 Java)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Potential Limitations and Future Work' provides substantive discussion of three specific limitations: keyword selection strategy, lack of natural-looking perturbation enforcement, and restriction to code completion scenarios."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: keyword tokens selected only from output tuple via simple grid search 'may overlook potentially better solutions'; perturbations are not mandated to be entirely natural-looking; the attack is only evaluated on code completion, not other programming tasks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states what was NOT tested: other code tasks ('code summarization and code translation'), natural-looking perturbation generation, and more efficient keyword optimization. Section 4.1 bounds the evaluation to 3 languages and specific model families."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregated ASR, NBR, and STF values are reported. Per-item attack results, generated code outputs, and detailed logs are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data sources: 13 malicious objectives from CWE top-10 KEV and prior work, standard benchmarks (Humaneval, MBPP, etc.). Table 1 provides item counts per language. Appendix A details each threat case (Table 6)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard publicly available benchmarks and CWE vulnerability definitions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from malicious objective → output tuple (Section 3.3) → code contextual simulation → perturbation optimization (Algorithm 1) → evaluation on benchmark datasets is fully documented. Dataset counts and language breakdowns are provided in Table 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgments section thanks two professors for comments but does not disclose any funding sources, grants, or sponsoring agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Zhejiang University and Nanyang Technological University. The evaluated models (CodeGemma by Google, CodeGeeX by Tsinghua-affiliated, Copilot by GitHub/OpenAI) are from different organizations than the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funders cannot be assessed. Without a funding statement, this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests an attack method against Code LLMs, not model capability on benchmarks. The benchmarks serve as realistic code contexts for attack evaluation, not as capability measures. Similar to a red-teaming study."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates attack success rates, not model knowledge. Whether models have seen benchmark code is a confound for attack evaluation but not a contamination concern in the standard sense."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "As an attack evaluation rather than a model capability benchmark, contamination is not the primary concern. The attack aims to manipulate outputs regardless of whether the model has seen the input code before."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports perturbation generation time: 'approximately 0.5-1 hours for CodeGemma-2b and 2-3 hours for both CodeGemma-7b and CodeGeeX-6b' on NVIDIA RTX A6000."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'We run Algorithm 1 on an NVIDIA RTX A6000 Graphics Card... employing a float16 precision format without quantization' with generation times for each model."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states 'the random seed for initializing induced perturbations is set to 100' — a single fixed seed. No results across multiple seeds or seed sensitivity analysis is provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state how many experimental runs produced the reported ASR values. The fixed seed of 100 suggests single-run results, but this is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Default hyperparameters are stated (Top-k=400, r=0.4, h=2, etc.) but the paper does not report how these defaults were selected, how many configurations were tried, or the total compute spent on hyperparameter tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation study (Section 4.5) varies Top-k and keyword count but the selection of default values (Top-k=400, 2 keywords, r=0.4) is described as 'default parameter values' without justification for why these specific values were chosen."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Both baselines (C-GCG and IPI-M) are the authors' own implementations/adaptations of existing methods. The paper does not acknowledge potential bias from implementing competitor baselines, as documented by Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "ShadowCode uses 12 tokens (10 perturbation + 2 keyword) vs C-GCG at 12 prefix tokens vs IPI-M at <30 tokens, controlling for perturbation size. However, the computational cost of generating perturbations (0.5-3 hours on A6000) is not compared across methods."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ASR on benchmark code completion datasets (Humaneval, MBPP, etc.) actually represents real-world attack success in production code environments with more complex contexts and workflows."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The evaluation tests direct code completion without multi-step scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the Code LLMs were trained on the benchmark datasets used for evaluation (Humaneval published 2021, MBPP 2021, etc.), which could affect attack success rates if models have strong priors about 'correct' completions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides the model with information that would not be available in a real-world attack scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether benchmark examples share structural similarities or whether optimization on simulated contexts could overfit to patterns present in the evaluation benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. The paper does not test for overlap between training data and evaluation benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "ShadowCode achieves up to 97.9% attack success rate on open-source Code LLMs using 12-token non-functional perturbations.",
       "evidence": "Table 2 shows CodeGemma-7b achieves 97.9% ASR on Humaneval (Python) and 97.4% on Eval-Plus (Python). Average ASR across all models and datasets ranges from 82.0% to 86.6%.",
       "supported": "strong"
     },
     {
       "claim": "ShadowCode successfully transfers to commercial Code LLM-integrated applications with over 90% ASR.",
       "evidence": "Table 4 reports 93.3% ASR on CodeGeeX (gray-box, Python) and 90.4% on GitHub Copilot (black-box, Python). Java and C/C++ results range from 67.7% to 80.4%.",
       "supported": "moderate"
     },
     {
       "claim": "ShadowCode significantly outperforms existing attack methods (C-GCG and IPI-M) across all evaluation settings.",
       "evidence": "Table 2 shows ShadowCode average ASR of 82.0-86.6% vs C-GCG at 41.9-44.6% and IPI-M at 11.2-26.2%. Table 3 shows ShadowCode leads in both NBR and STF metrics across all languages.",
       "supported": "moderate"
     },
     {
       "claim": "Forward reasoning enhancement (FRE) and keyword-based perturbation design (KPD) are both necessary for ShadowCode's effectiveness.",
       "evidence": "Section 4.5 and Figures 10-11: removing keywords drops ASR by ~10% (1-keyword) or prevents reaching 80% ASR even with Top-k=600 (0-keyword). Without FRE, ASR stays below 20% even when loss is lower than with FRE.",
       "supported": "strong"
     },
     {
       "claim": "ShadowCode perturbations are semantically indistinguishable from benign code via embedding-layer analysis.",
       "evidence": "Section 4.4 and Figure 8: t-SNE visualization of 1,640 code segments (820 benign, 820 injected) at CodeGemma-2b layer 17 shows features are 'almost completely intertwined.'",
       "supported": "weak"
     },
     {
       "claim": "ShadowCode resists character removal defenses, maintaining sufficient ASR under 50% removal rate.",
       "evidence": "Figure 9 shows ASR remains high at 50% character removal rate. Table 5 shows ASR of 65-90% when perturbations use non-comment forms (variable assignment, output content).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across Tables 2-4 and Figures 6, 10 are single point estimates. No confidence intervals, standard deviations, or multi-run statistics are provided despite the stochastic nature of the optimization process (random seed, greedy search)."
     },
     {
       "flag": "Single random seed evaluation",
       "detail": "All experiments use a fixed random seed of 100 (Section 4.1). No seed sensitivity analysis is conducted, so it is impossible to assess how robust the results are to different initializations."
     },
     {
       "flag": "Self-implemented baselines",
       "detail": "Both baselines (C-GCG and IPI-M) are the authors' own adaptations of existing methods (GCG and indirect prompt injection). The C-GCG adaptation removes code simulation and enhancement modules, potentially weakening the baseline. IPI-M is a simplified version of existing IPI. No independent baseline implementations are used."
     },
     {
       "flag": "Selective reporting of best-case results",
       "detail": "The abstract highlights 'up to 97.9% ASR' and 'over 90% ASR' which are best-case language-specific results (Python). Average results across all languages are lower (84.5% for open-source, 67.7-90.4% for commercial), and some individual cases have considerably lower ASR (Figure 14)."
     },
     {
       "flag": "Weak defense evaluation",
       "detail": "The defense evaluation (Section 4.4) only considers t-SNE visualization of embeddings (not a trained classifier) and random character removal. No evaluation against practical defenses like instruction-following guardrails, output safety filters, or structured code analysis tools."
     },
     {
       "flag": "Commercial evaluation sample size and methodology",
       "detail": "Commercial application testing (Table 4) uses only 620 manually tested items from Humaneval/Humaneval-x, which are simpler code contexts. Manual testing introduces potential subjectivity in ASR determination. No description of how 'randomly selected' samples were chosen."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "relevance": "Foundational work on indirect prompt injection against LLM-integrated applications; directly inspires ShadowCode's external prompt injection paradigm."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["A. Zou", "Z. Wang", "J. Z. Kolter", "M. Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG attack method that serves as the basis for the C-GCG baseline and informs the greedy search optimization approach."
     },
     {
       "title": "You autocomplete me: Poisoning vulnerabilities in neural code completion",
       "authors": ["R. Schuster", "C. Song", "E. Tromer", "V. Shmatikov"],
       "year": 2021,
       "relevance": "Pioneering work on backdoor attacks against code completion models; provides the ST0 and ST1 malicious objectives used in ShadowCode evaluation."
     },
     {
       "title": "Stealthy backdoor attack for code models",
       "authors": ["Z. Yang", "B. Xu", "J. M. Zhang", "H. J. Kang", "J. Shi", "J. He", "D. Lo"],
       "year": 2024,
       "relevance": "Stealthy backdoor attack via dataset poisoning for code models; represents the category of training-time attacks that ShadowCode claims to improve upon."
     },
     {
       "title": "Multi-target backdoor attacks for code pre-trained models",
       "authors": ["Y. Li", "S. Liu", "K. Chen", "X. Xie", "T. Zhang", "Y. Liu"],
       "year": 2023,
       "relevance": "Model poisoning-based backdoor attack on code LLMs; represents training-time attack methods that require model access."
     },
     {
       "title": "An LLM-assisted easy-to-trigger backdoor attack on code completion models: Injecting disguised vulnerabilities against strong detection",
       "authors": ["S. Yan", "S. Wang", "Y. Duan", "H. Hong", "K. Lee", "D. Kim", "Y. Hong"],
       "year": 2024,
       "relevance": "LLM-assisted backdoor attack on code completion that injects disguised CWE vulnerabilities; closely related attack vector targeting code security."
     },
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["J. He", "M. Vechev"],
       "year": 2023,
       "relevance": "Evaluates LLMs for code security including adversarial testing with CWE vulnerabilities; directly relevant to security assessment of code LLMs."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Q. Zhan", "Z. Liang", "Z. Ying", "D. Kang"],
       "year": 2024,
       "relevance": "Benchmark for indirect prompt injection in LLM agents; shares the attack scenario of compromising users through injected instructions."
     },
     {
       "title": "CodeAttack: Code-based adversarial attacks for pre-trained programming language models",
       "authors": ["A. Jha", "C. K. Reddy"],
       "year": 2023,
       "relevance": "Adversarial attacks on code models via identifier renaming; represents the adversarial attack category that ShadowCode contrasts with."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the HumanEval benchmark for code generation used as a primary evaluation dataset in this paper."
     },
     {
       "title": "\"Do Anything Now\": Characterizing and evaluating in-the-wild jailbreak prompts on large language models",
       "authors": ["X. Shen", "Z. Chen", "M. Backes", "Y. Shen", "Y. Zhang"],
       "year": 2024,
       "relevance": "Comprehensive study of jailbreak prompts against LLMs; represents the broader prompt injection attack landscape that ShadowCode extends to code domain."
     },
     {
       "title": "CodeGemma: Open code models based on Gemma",
       "authors": ["C. Team", "H. Zhao", "J. Hui"],
       "year": 2024,
       "arxiv_id": "2406.11409",
       "relevance": "CodeGemma model series used as primary evaluation targets (CodeGemma-2b, CodeGemma-7b) in ShadowCode experiments."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Demonstrates a real attack vector against widely-used code completion tools (Copilot, CodeGeeX) that practitioners should be aware of, but executing the attack requires ML expertise and GPU access."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Prompt injection is a known concern; extending it to code completion via non-functional perturbations is incremental rather than paradigm-shifting."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates 90%+ success rate injecting malicious code (including 'rm -rf /') into GitHub Copilot completions via just 12 tokens hidden in code comments — directly alarming for developer security."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Shows popular commercial tools are vulnerable but does not frame this as controversy or conflict with the tool developers."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub (https://github.com/LianPing-cyber/ShadowCodeEPI) but requires GPU hardware and ML setup to run."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Attacks GitHub Copilot (OpenAI/GitHub) and CodeGeeX — well-known products — but the authors and their institutions are not high-profile in the broader AI discourse."
     }
   }
 }