ai-research-survey

scan.json (25959B)

---

 {
   "paper": {
     "title": "TFHE-Coder: Evaluating LLM-agentic Fully Homomorphic Encryption Code Generation",
     "authors": ["Mayank Kumar", "Jiaqi Xue", "Mengxin Zheng", "Qian Lou"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.12217",
     "doi": "10.48550/arXiv.2503.12217"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GPT-4o significantly outperforms open-source models (CodeLlama, Qwen2.5-Coder, Deepseek-Coder) in generating compilable and functionally correct TFHE code, with open-source models achieving zero functional correctness in the baseline setting. Few-shot prompting is substantially more effective than RAG alone, and combining both yields the best results. ReLU remains the most challenging task across all models and techniques, while simpler logic gates (NOT, AND, OR) are more tractable. The paper establishes the first benchmark for TFHE code generation, though it is limited to only four tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to the evaluation framework is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No benchmark dataset, task specifications, reference implementations, or generated outputs are released. The 4 tasks (NOT, AND, OR, ReLU) are described but the actual prompts and reference code are not publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section III.B lists hardware (i5-12600K, 64GB RAM, RTX 3090, Ubuntu 24.04, Python 3.8, CUDA 12.2) and the embedding model (jina-embeddings-v2-base-code), but there is no requirements.txt, dependency list, TFHE library version, or sufficient detail to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts to replicate experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Despite averaging over 5 runs, no confidence intervals or error bars are reported. Figures 3, 5, 6, and 7 show only point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparative claims are made (e.g., 'GPT-4o consistently outperforms', 'few-shot prompting significantly improves correctness') without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Specific improvements with baseline context are reported, e.g., 'Deepseek-Coder on AND improves from 0.12 to 0.83 (CrystalBLEU) and from 0.0 to 1.0 (Pass@k).' Absolute scores are given for all models across all conditions."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses nt=5 repeated experiments per condition with no justification for why 5 was chosen or whether this is sufficient for reliable estimates."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'five repeated experiments and averaging the results' but never reports standard deviation, variance, or any spread measure across those runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A clear baseline (compiler-in-the-loop without RAG or few-shot) is established, against which RAG, few-shot, and RAG+few-shot are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o, Claude-3.5-Haiku, Qwen2.5-Coder, and Deepseek-Coder, which are recent and representative of both closed and open-source LLMs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically tests baseline, RAG alone, few-shot alone, and RAG+few-shot, effectively ablating the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: CrystalBLEU (structural similarity), Pass@k (comp) (compilability), Pass@k (func) (functional correctness), Wrong Format Error, and Repetition Error."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality. All evaluation is automated via compilation and test case execution."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The few-shot example uses a correct OR gate implementation, yet OR is also one of the four test tasks. There is no separation between the example used for prompting and the tasks evaluated."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per task (NOT, AND, OR, ReLU) and per model across all metrics in Figures 3-7."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV (RQ2) provides detailed failure analysis: wrong format errors, repetition errors, API misuse, hallucinated function calls, and copy-paste behavior. Specific failure patterns are categorized."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RAG alone is reported as offering 'limited gains' with 'no significant increase in functional correctness.' Open-source models' failures are extensively documented. ReLU remains unsolved for most models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 'significant limitations in off-the-shelf models' and 'agentic optimizations reduce errors and enhance code fidelity' are supported by the experimental results in Section IV."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'few-shot prompting significantly improves correctness' are supported by controlled single-variable comparisons (baseline vs. +RAG, vs. +few-shot, vs. +both). The ablation design provides adequate causal evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies 'focusing on logic gates and ReLU activation' and the paper consistently frames findings around these specific tasks. The title is specific to TFHE code generation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider confounds such as model size differences (7B vs. unknown for GPT-4o/Claude), training data composition, or whether the performance gap is due to TFHE-specific vs. general coding ability."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses CrystalBLEU as a proxy: 'higher similarity often indicates that less effort is required to correct and functionalize the output.' The metrics (compilability, functional correctness, structural similarity) are clearly distinguished from broader claims about TFHE usability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4o', 'Claude-3.5-Haiku', 'CodeLlama-7B-Instruct', 'Qwen2.5-Coder-7B-Instruct', 'deepseek-coder-6.7b-instruct'. GPT-4o and Claude-3.5-Haiku lack snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described only in natural language: 'A user prompt (consisting of task description and reference C code)' and 'a correct implementation of an OR gate using TFHE's bootsOR function.' The actual prompt text is never provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B states: 'we set the temperature to 0.9 and top-p to 0.85' and specifies the iteration limit of 10."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The compiler-in-the-loop workflow (Fig. 1) and agentic-optimized workflow (Fig. 2) are described in detail, including the feedback loop, RAG integration, and few-shot prompting mechanism."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The task space is clearly defined (NOT, AND, OR, ReLU), the evaluation pipeline is documented (generate → compile → feedback → measure), and the reference implementation structure is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section V is titled 'Discussion and Future Work' but contains only a brief paragraph restating results and suggesting future directions. There is no dedicated limitations section with substantive discussion of study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The discussion section mentions that 'additional fine-tuning or domain-specific training may be necessary' but does not address threats to the validity of the current study's conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. It does not discuss limitations of evaluating on only 4 tasks, only gate-level operations, or the non-representativeness of the task set for general TFHE code generation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated code samples, compilation logs, per-run results) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B describes the task design rationale, model selection, experimental setup, evaluation metrics, and the iterative generation/compilation workflow."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates LLMs on constructed coding tasks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from prompt to generation to compilation to feedback to evaluation is documented across Sections III.B and III.C with accompanying workflow diagrams (Fig. 1 and Fig. 2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or funding disclosure is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with the University of Central Florida. They are not evaluating their own commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of funder cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five evaluated models. This is important since TFHE library code and documentation may be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether TFHE code examples, documentation, or the reference implementations appear in the models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TFHE documentation and examples are publicly available and likely in training data for models trained after 2020. The paper does not address this contamination risk despite it being central to interpreting the results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Appendix reports token consumption for each model (e.g., GPT-4o: 65,829 input tokens, 1,784 output tokens) across different experimental conditions."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (RTX 3090, i5-12600K) but total compute budget (wall-clock time, total API spend, GPU hours) is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 5 runs but no variance, standard deviation, or seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B states: 'five repeated experiments and averaging the results' and 'nt = 5 and k = 1.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.9 and top-p 0.85 are used but no justification for these values or search budget is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All four configurations (baseline, RAG, few-shot, RAG+few-shot) are reported across all models and tasks. The 'best' configuration is determined transparently from the full results."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the evaluation framework and evaluate it without acknowledging potential bias in designing the evaluation to favor their approach."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Token consumption is reported in the Appendix but performance is not plotted as a function of compute. Models with vastly different sizes (7B vs. unknown for GPT-4o) are compared without discussing compute parity."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether 4 tasks (NOT, AND, OR, ReLU) are representative of TFHE code generation more broadly, or whether these tasks have adequate construct validity for the claims made."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated within the same compiler-in-the-loop framework, keeping the scaffold constant across model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether TFHE library code, documentation, or example implementations existed in model training data before the evaluation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompt includes reference C code implementations, which provides structural guidance that would not be available in realistic usage. This is acknowledged as part of the design but the implications for generalizability are not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The four tasks are structurally similar gate-level TFHE operations. The non-independence of these tasks (success on one may predict success on others) is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4o consistently outperforms open-source models across all TFHE code generation tasks in the baseline setting.",
       "evidence": "Figure 3 shows GPT-4o achieving the highest Pass@k (comp) and Pass@k (func) scores across NOT, AND, OR, and ReLU, while open-source models achieve zero functional correctness (Section IV, RQ1).",
       "supported": "moderate"
     },
     {
       "claim": "Few-shot prompting is more effective than RAG alone for improving TFHE code generation.",
       "evidence": "Comparing Fig. 5 (RAG) and Fig. 6 (few-shot): few-shot shows substantial improvements across models (e.g., Deepseek-Coder AND: 0.12→0.83 CrystalBLEU, 0.0→1.0 Pass@k), while RAG shows minimal gains (Section IV, RQ3).",
       "supported": "moderate"
     },
     {
       "claim": "Combining RAG and few-shot prompting yields the strongest overall improvements.",
       "evidence": "Fig. 7 and Fig. 4 show RAG+few-shot achieves the highest CrystalBLEU scores, lowest error rates (Wrong Format, Repetition), and near-perfect correctness for GPT-4o and Claude-3.5-Haiku (Section IV, RQ3).",
       "supported": "moderate"
     },
     {
       "claim": "Open-source models fail to generate functionally correct TFHE code in the baseline setting.",
       "evidence": "Fig. 3(c) shows Pass@k (func) scores of zero for CodeLlama, Qwen2.5-Coder, and Deepseek-Coder across all four tasks (Section IV, RQ1).",
       "supported": "strong"
     },
     {
       "claim": "ReLU is the most challenging task for TFHE code generation across all models and techniques.",
       "evidence": "Across all figures (3-7), ReLU consistently shows the lowest CrystalBLEU and Pass@k scores, with only GPT-4o achieving functional correctness. The paper attributes this to 'arithmetic complexity' (Section IV).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny task set",
       "detail": "The entire evaluation is based on only 4 tasks (NOT, AND, OR, ReLU). This is far too few to draw robust conclusions about TFHE code generation capability. The paper claims to establish 'the first benchmark for TFHE code generation' based on these 4 tasks."
     },
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "The paper averages over 5 runs but never reports standard deviation or any spread measure. With only 5 runs on 4 tasks, individual results could vary substantially, and the reader has no way to assess result stability."
     },
     {
       "flag": "Few-shot example overlaps with test set",
       "detail": "The few-shot prompt uses a correct OR gate implementation as the example, and OR is also one of the four evaluated tasks. This conflates the 'training' signal with the test evaluation."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes many comparative claims ('outperforms', 'significantly improves') without any statistical testing, relying solely on visual comparison of point estimates from 5 averaged runs."
     },
     {
       "flag": "Unfair model size comparison",
       "detail": "Open-source models are all ~7B parameters while GPT-4o and Claude-3.5-Haiku are likely much larger. The performance gap may reflect model scale rather than anything specific to TFHE code generation, but this confound is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey on large language models for code generation",
       "authors": ["J. Jiang", "F. Wang", "J. Shen", "S. Kim", "S. Kim"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey of LLM code generation capabilities, directly relevant to understanding the state of the field this paper evaluates."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and the HumanEval benchmark, foundational work for LLM code generation evaluation."
     },
     {
       "title": "CodeGen: An open large language model for code with multi-turn program synthesis",
       "authors": ["E. Nijkamp", "B. Pang", "H. Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Open-source code generation model with multi-turn synthesis, relevant to agentic code generation."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the evaluated open-source code models; foundational LLM for code."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["B. Hui", "J. Yang", "Z. Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for one of the evaluated open-source code models."
     },
     {
       "title": "Deepseek-coder: When the large language model meets programming",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the evaluated open-source code models for domain-specific code generation."
     },
     {
       "title": "GPT-4o system card",
       "authors": ["A. Hurst", "A. Lerer", "A. P. Goucher"],
       "year": 2024,
       "arxiv_id": "2410.21276",
       "relevance": "System card for the top-performing model in this evaluation."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper, relevant to agentic code generation augmentation techniques evaluated in this study."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "relevance": "Introduces few-shot prompting paradigm used as a key technique in this evaluation."
     },
     {
       "title": "A survey on LLM-based code generation for low-resource and domain-specific programming languages",
       "authors": ["S. Joel", "J. J. Wu", "F. H. Fard"],
       "year": 2024,
       "arxiv_id": "2410.03981",
       "relevance": "Directly addresses domain-specific code generation challenges that this paper investigates for TFHE."
     },
     {
       "title": "On the robustness of code generation techniques: An empirical study on GitHub Copilot",
       "authors": ["A. Mastropaolo", "L. Pascarella", "E. Guglielmi"],
       "year": 2023,
       "relevance": "Empirical evaluation of code generation robustness, relevant to understanding LLM code generation reliability."
     },
     {
       "title": "Benchmarking large language models for automated verilog RTL code generation",
       "authors": ["S. Thakur", "B. Ahmad", "Z. Fan"],
       "year": 2023,
       "relevance": "Benchmarks LLMs for domain-specific hardware code generation, closely parallel methodology to TFHE code generation evaluation."
     }
   ]
 }