ai-research-survey

scan.json (28546B)

---

 {
   "paper": {
     "title": "STELP: Secure Transpilation and Execution of LLM-Generated Programs",
     "authors": [
       "Swapnil Shinde",
       "Sahil Wadhwa",
       "Andy Luo",
       "Akshay Gupta",
       "Mohammad Shahed Sorower"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.05467",
     "doi": "10.48550/arXiv.2601.05467"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "STELP is a transpiler-inspired engine that intercepts LLM-generated Python code, validates its AST against a configurable safe grammar subset, and executes a secured version with runtime controls. On the authors' InjectedHumanEval benchmark (634 samples, 12 CWE types), STELP achieves a perfect True Block Rate (1.0) and 0.981 True Allow Rate, significantly outperforming Meta's CodeShield (TBR 0.68, TAR 0.93). STELP adds median 0.19ms latency overhead and achieves 100% correctness on a filtered subset of the Python-Code-Execution-Output dataset. A feedback loop using Llama 3.3-70B repaired 90.2% of blocked code samples in under 2 retries.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No STELP source code is released. The paper provides no repository URL for the engine itself. Only the InjectedHumanEval dataset is linked (tinyurl.com/24aebhmr)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The InjectedHumanEval dataset is released at https://tinyurl.com/24aebhmr. The Python-Code-Execution-Output dataset is publicly available on HuggingFace (Diwank 2024)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The latency evaluation mentions '16GB of RAM, and a 10-core CPU' but no requirements.txt, Dockerfile, Python version, or library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Without the source code or detailed setup guide, results cannot be replicated."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main safety results (TBR=1.0, TAR=0.981) are reported as point estimates with no confidence intervals. Table 2 reports descriptive statistics (mean, median, std dev, IQR) for latency but not CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims STELP 'significantly outperforms' CodeShield based on comparing raw numbers (TBR 1.0 vs 0.68, TAR 0.981 vs 0.93) without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute metrics with baseline context are provided: STELP TBR=1.0 vs CodeShield TBR=0.68, TAR=0.981 vs 0.93. The magnitude of improvement is clear from these paired comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 634 samples in InjectedHumanEval, 361 for correctness, or 262 for latency. No power analysis or discussion of whether these sizes are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports std dev (46.26ms) and IQR (0.17ms) for latency across 30 executions per sample. The safety evaluation is deterministic (rule-based AST analysis), so variance is not applicable there."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "STELP is compared against Meta's CodeShield on TBR and TAR metrics. Tables 4 and 5 provide feature and CWE coverage comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeShield (Meta PurpleLlama, 2024) is a recent and relevant baseline for static analysis of LLM-generated code."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "STELP has three main components (AST Processor, Safe Code Generator, Feedback Generator) but no ablation study isolates the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses TBR, TAR (safety), correctness rate, and multiple latency statistics (mean, median, per-statement type breakdown). Four distinct evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human annotators validated the InjectedHumanEval dataset creation, but no human evaluation of STELP's outputs (e.g., whether blocking decisions are correct beyond binary labels, or whether transpiled code preserves semantics) was conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The STELP configuration was 'manually developed' as 'the minimum set of permissions that allows the benign InjectedHumanEval samples to run' — meaning the configuration was tuned directly on the test data with no separate dev/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down unsafe samples by CWE type. Table 6 shows STELP exceptions by type. Table 3 provides per-statement latency. Table 5 compares CWE coverage per tool."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 3 safe samples incorrectly blocked (TAR=0.981) and 8% of feedback loop samples exceeding 10 retries, but does not analyze specific failure cases or show examples of incorrect blocking."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: TAR=0.981 (3 safe samples blocked), worst-case 520ms latency, 8% of feedback loop samples exceeded 10 retries, and the paper acknowledges STELP is currently Python-only."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claim that STELP 'outperforms an existing method by a significant margin' is supported by TBR 1.0 vs 0.68 and TAR 0.981 vs 0.93 on InjectedHumanEval. The 'human-validated dataset of 634 samples' is documented in the Evaluation section."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'STELP's dynamic validation of a safe grammar subset is a more robust approach than traditional static pattern matching' — a causal claim about why STELP outperforms CodeShield. No ablation or controlled experiment isolates the design choice (dynamic vs static) as the cause of improvement."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'LLM-Generated Programs' (general) and abstract claims STELP can handle 'production systems,' but evaluation is Python-only on synthetic benchmarks (InjectedHumanEval, Python-Code-Execution-Output). No real production code or other languages tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for STELP's performance are discussed. For example, the possibility that InjectedHumanEval's synthetic injections are formulaic and easy to detect (vs. real adversarial code) is not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures TBR/TAR on a synthetic benchmark but claims STELP 'secures autonomous production AI systems.' The gap between benchmark safety (synthetic CWE injections) and actual production security is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Llama3.3 70B Instruct' for code injection and 'Llama 3-70B' / 'Llama 3.3-70B' for the feedback loop. These are specific model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides the full injection prompt template used for InjectedHumanEval creation. Figures 9 and 10 provide the feedback generation and code repair prompts. These are actual prompt text, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max_tokens, or other sampling parameters are reported for any of the LLM calls (Llama3.3 for injection, Llama 3-70B for feedback)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "STELP's architecture is described in detail: AST Processor (parser + validator), Safe Code Generator and Executor (recursive AST traversal with controls), Feedback Generator (error classification + LLM summarization). Figures 1-4 illustrate the flow with examples."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "InjectedHumanEval creation is documented (LLM injection → human review → 634 samples). However, the correctness subset selection ('361 code samples compatible with our evaluation configuration') and latency subset (262 samples) have no explained filtering criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The 'Conclusion and Future Work' mentions Python-only scope and plans for Java/SQL, but does not substantively discuss limitations of the current evaluation or approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address whether synthetic CWE injections represent real-world threats, whether the configuration was overfit to the benchmark, or whether the results generalize beyond the tested setting."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python-only scope and future language extensions but does not explicitly state what the results do NOT show (e.g., no adversarial robustness testing, no real production deployment, no multi-language support)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The InjectedHumanEval dataset is available at the provided tinyurl link. The Python-Code-Execution-Output dataset is public on HuggingFace."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "InjectedHumanEval creation is described: HumanEval base → LLM injection with Llama3.3 70B using specific prompts (Figure 5) → human annotation review → final dataset of 164 safe + 470 unsafe samples across 12 CWE types."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Human annotators validated the InjectedHumanEval dataset but are not described — no mention of how many annotators, their expertise, inter-annotator agreement, or selection criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The InjectedHumanEval pipeline is partially documented. However, the paper uses 634 samples for safety, 361 for correctness, and 262 for latency with no explanation for the different subset sizes or filtering criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section. All authors are affiliated with Capital One but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list Capital One as their affiliation, with a note that some work was 'performed while at Capital One.' The corporate affiliation is clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Capital One employees developed and evaluated STELP, presumably for Capital One's production use. The employer has a direct interest in the tool being effective, making the funder non-independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The feedback loop evaluation uses Llama 3.3-70B on HumanEval-derived tasks. The training data cutoff for Llama 3.3 is not stated, despite HumanEval being published in 2021 and likely in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "InjectedHumanEval is derived from HumanEval (2021). Llama 3.3 was trained after 2021 and likely saw HumanEval solutions. The 90.2% repair rate may be inflated if the LLM already knows the solutions. This overlap is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval has been publicly available since 2021. Using it as the base for InjectedHumanEval means any model trained after 2021 may have memorized solutions, but contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Human annotators performed dataset quality control but were not research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates a software tool on benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Latency is reported in detail: median 0.19ms overhead, mean 4.93ms, per-statement breakdowns (Table 3), and worst case 520ms. The latency evaluation used 30 runs per sample on specified hardware."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('16GB of RAM, and a 10-core CPU') but total compute budget (e.g., total GPU/API hours for LLM calls in benchmark creation and feedback loop evaluation) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis for the stochastic LLM components (Llama 3.3 for injection, feedback loop). The 30 latency runs are timing runs, not seed variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The latency evaluation explicitly states '30 executions on both STELP and Python's native execution engine.' The feedback loop evaluation reports aggregate results without stating number of runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The STELP configuration was 'manually developed' for the evaluation. No systematic search budget, number of configurations tried, or tuning methodology is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The configuration was tuned on the test data itself: 'a single STELP configuration containing the minimum set of permissions that allows the benign InjectedHumanEval samples to run.' No validation set was used for selection."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created both the tool (STELP) and the benchmark (InjectedHumanEval) and evaluated one against the other. No acknowledgment of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Latency overhead is reported (Table 2) but performance is not plotted as a function of compute budget. No analysis of how changing STELP's configuration complexity affects the safety-latency tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether InjectedHumanEval (synthetic CWE injections into simple coding tasks) actually represents the threat landscape of real LLM-generated code in production. The benchmark's construct validity is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "STELP is the tool being evaluated, not a scaffold confound in model comparison. No model-vs-model comparison is made where scaffolding would confound results."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) was published before Llama 3.3's training. The 90.2% feedback loop repair rate may benefit from Llama 3.3 having seen HumanEval solutions. Temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The feedback loop provides detailed error information from STELP, which could make repair trivially easy compared to real-world scenarios where such feedback is unavailable."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "InjectedHumanEval is derived directly from HumanEval. The safe samples ARE HumanEval samples. Independence between training data and test data is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "STELP achieves a perfect True Block Rate (1.0) for blocking unsafe code on InjectedHumanEval.",
       "evidence": "Evaluation section: 470 unsafe samples all blocked. Table 6 breaks down blocking by exception type (FunctionNotAllowedError: 311, WhileTrueError: 70, etc.).",
       "supported": "moderate"
     },
     {
       "claim": "STELP achieves a True Allow Rate of 0.981 for executing safe code on InjectedHumanEval.",
       "evidence": "Evaluation section: 161/164 safe samples executed correctly. Three safe samples were incorrectly blocked.",
       "supported": "moderate"
     },
     {
       "claim": "STELP significantly outperforms CodeShield on safety metrics (TBR 1.0 vs 0.68, TAR 0.981 vs 0.93).",
       "evidence": "Static Code Analysis Comparison section compares results on InjectedHumanEval. Tables 4 and 5 compare feature coverage.",
       "supported": "moderate"
     },
     {
       "claim": "STELP achieves 100% correctness on the Python-Code-Execution-Output test dataset.",
       "evidence": "Correctness section: tested on 361 samples 'compatible with our evaluation configuration,' all produced correct output matching native Python execution.",
       "supported": "weak"
     },
     {
       "claim": "STELP adds negligible latency overhead (median 0.19ms) compared to native Python execution.",
       "evidence": "Table 2: median execution time increase 0.19ms across 262 samples with 30 runs each. Table 3 provides per-statement breakdowns.",
       "supported": "strong"
     },
     {
       "claim": "90.2% of unsafe code samples were successfully repaired by the feedback loop in under 2 retries.",
       "evidence": "Feedback Loop section: Llama 3.3-70B used for code repair on InjectedHumanEval. 8% exceeded 10 retries, attributed to LLM limitations.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Configuration tuned on test data",
       "detail": "The STELP configuration used for safety evaluation was 'manually developed' as 'the minimum set of permissions that allows the benign InjectedHumanEval samples to run.' This means the configuration was directly fit to the test benchmark, creating circularity: the tool was configured to work on the exact data it was evaluated on, with no held-out validation."
     },
     {
       "flag": "Self-created benchmark evaluating own system",
       "detail": "The authors created both STELP and InjectedHumanEval, then evaluated one against the other. The synthetic CWE injections were designed by the same team that built the defense. No independent evaluation or red-teaming by external parties."
     },
     {
       "flag": "Company evaluating own product",
       "detail": "All authors are Capital One employees evaluating a Capital One tool. No independent replication or third-party evaluation. The corporate interest in STELP being effective is not acknowledged."
     },
     {
       "flag": "Unexplained subset filtering",
       "detail": "Different evaluation dimensions use different unexplained subsets: 634 samples for safety, 361 'compatible' samples for correctness, and 262 for latency. The filtering criteria are not documented, raising questions about whether incompatible samples would expose weaknesses."
     },
     {
       "flag": "No adversarial robustness evaluation",
       "detail": "Unsafe samples were generated by LLM injection of known CWE patterns — a formulaic approach. No testing against adaptive adversaries who might craft code specifically to bypass STELP's AST analysis while remaining malicious."
     },
     {
       "flag": "Contamination risk in feedback evaluation",
       "detail": "The 90.2% feedback repair rate uses Llama 3.3-70B on HumanEval-derived code. Llama 3.3 likely saw HumanEval solutions during training, which would inflate the repair success rate — the model may be recovering memorized solutions rather than genuinely repairing code from feedback."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, which serves as the base for InjectedHumanEval and is central to LLM code generation evaluation."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Early benchmark for LLM code generation capabilities, referenced as containing potentially unsafe code for execution."
     },
     {
       "title": "CodeShield: Shield against LLM generated insecure code (PurpleLlama)",
       "authors": ["Meta"],
       "year": 2024,
       "relevance": "The primary baseline — static analysis tool for LLM-generated code security that STELP claims to significantly outperform."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Xiawu Zheng", "Jonathan Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent code generation framework representing the type of system STELP aims to secure."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent software development system where LLM-generated code needs safety controls like STELP."
     },
     {
       "title": "Executable Code Actions Elicit Better LLM Agents",
       "authors": ["Xingyao Wang", "Yangyi Chen", "Lifan Yuan"],
       "year": 2024,
       "arxiv_id": "2402.01030",
       "relevance": "Proposes executable code as a unified action space for LLM agents — directly motivates STELP's safe execution engine."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Empirical study of AI coding assistant productivity impact, relevant to the agentic code generation ecosystem STELP targets."
     },
     {
       "title": "LLMs Cannot Reliably Identify and Reason About Security Vulnerabilities (Yet?)",
       "authors": ["Saad Ullah", "Mingji Han", "Saurabh Pujar"],
       "year": 2024,
       "arxiv_id": "2312.12575",
       "relevance": "Demonstrates LLM limitations in security vulnerability detection, motivating non-LLM approaches like STELP's AST analysis."
     },
     {
       "title": "Large Language Models and Code Security: A Systematic Literature Review",
       "authors": ["Edin Basic", "Alberto Giaretta"],
       "year": 2025,
       "arxiv_id": "2412.15004",
       "relevance": "Systematic review of LLM code security — directly relevant to survey scope on code generation safety."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["Aaron Grattafiori", "Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "The LLM family used in STELP's evaluation for code injection and feedback generation."
     },
     {
       "title": "Large Language Model Based Multi-Agents: A Survey of Progress and Challenges",
       "authors": ["Taicheng Guo", "Xiuying Chen", "Yaqi Wang"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Survey of multi-agent LLM systems that STELP aims to secure, relevant to agentic AI survey scope."
     },
     {
       "title": "A Survey on Code Generation with LLM-Based Agents",
       "authors": ["Yanzhen Dong", "Xin Jiang", "Jiahui Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based code generation agents, directly relevant to the systems STELP targets."
     }
   ]
 }