ai-research-survey

scan.json (27085B)

---

 {
   "paper": {
     "title": "Syntactic Robustness for LLM-based Code Generation",
     "authors": [
       "Laboni Sarker",
       "Mara Downing",
       "Achintya Desai",
       "Tevfik Bultan"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2404.01535",
     "doi": "10.48550/arXiv.2404.01535"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GPT-3.5 and GPT-4 are not syntactically robust when mathematical formulas in code generation prompts are mutated to semantically equivalent forms. Syntactic robustness degree decreases as syntactic distance (number of mutations) increases, from ~85% at distance 1 to ~36% at distance 5 for GPT-4. GPT-4 is 1.55x more robust than GPT-3.5 on average. A pre-processing step that reduces mutated formulas to simplified forms achieves 100% syntactic robustness on all tested equations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section VII mentions 'The working pipeline is available at Link' but the actual URL is a placeholder — no working repository link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 627 equation variations and 3135 generated code responses are not released. The equations are described formally but the actual dataset is not available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section VIII-A mentions hardware (i9-13900K, 192 GB RAM, Ubuntu 22.04.4) and Section VII mentions Python with sympy, but no requirements.txt, dependency versions, or environment specification sufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline is described conceptually in Section VII and Figure 7, but no step-by-step reproduction instructions (commands to run, configuration steps) are provided. The repository link is a placeholder."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '85.05%, 63%, 54%, 51.37%, and 36%' for distances 1-5). No confidence intervals or error bars are provided despite querying each prompt 5 times."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 is more robust than GPT-3.5 and that robustness decreases with distance, but uses no statistical tests. Comparisons are based solely on comparing raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context: '85.05%, 63%, 54%, 51.37%, and 36% respectively' for GPT-4 across distances, and 'GPT-4 has performed 1.55 times better than GPT-3.5' (Section VIII-B, RQ1-RQ2). The magnitude of effects is clear from the percentage breakdowns."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "20 variations per distance, 627 total variations, 5 queries per variation (3135 total), and 1000 differential testing inputs are all used without justification for these specific numbers."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite querying each prompt 5 times to address non-determinism (Section VII), no standard deviation, variance, or spread across the 5 runs is reported. Only aggregate robustness degrees are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5 and GPT-4 are compared against each other, and the pre-processing reduction approach is compared against the no-pre-processing baseline (Figures 8-12)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5-Turbo and GPT-4 were the leading commercial LLMs at the time of writing (early 2024). These are appropriate and contemporary models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies mutation types (5 types, Figure 9), equation types (7 types, Figures 10-12), and syntactic distances (1-5, Figure 8), effectively ablating along multiple dimensions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The only metric used is syntactic robustness degree (Definition 7). While broken down by equation type, distance, and mutation type, it is fundamentally a single metric throughout."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via differential testing with 1000 random inputs per generated code (Section VII). No human evaluation of generated code quality or correctness is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Section VII states they 'tested different setups with temperature, top p, and seed values and chose the one that provides the most deterministic results' — this hyperparameter selection was done on some data, but there is no description of whether this was separated from the final evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by equation type (Figures 10-12), mutation type (Figure 9), and syntactic distance (Figure 8), providing detailed per-category analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The motivating example (Figures 1-2) shows a specific failure. RQ2 discusses a specific quadratic equation failure where 'GPT-4 calculates the discriminant as b²−4×a×(−c) instead of b²−4×a×c.' Rounding issues in logarithmic equations are also discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The main finding is negative — both GPTs fail at syntactic robustness. They also report the surprising case where higher syntactic distance sometimes yields better results for quadratic equations in GPT-4 (Section VIII-B, RQ2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT-3.5 and GPT-4 are not syntactically robust (supported by Figures 8-12) and that pre-processing improves robustness (supported by Figure 8 showing 100% after reduction). All abstract claims are backed by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('reduction improves robustness', 'more mutations create more confusion') are supported by controlled single-variable manipulation — same prompts with/without reduction, same equations at different distances. The experimental design supports causal inference for these specific claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Syntactic Robustness for LLM-based Code Generation' is much broader than the study scope: only GPT-3.5 and GPT-4 tested, only mathematical equation prompts in C code, only 7 equation types. The threats section acknowledges limited equations but not the gap between title/framing and scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats section discusses methodological concerns (API learning, differential testing limitations) but does not discuss alternative explanations for why models fail at syntactic robustness (e.g., tokenization effects, training data distribution of formula representations, attention mechanisms)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines what it measures (syntactic robustness degree via differential testing with 1000 inputs) and acknowledges the proxy limitation: 'we calculate the syntactic robustness degree with respect to a subset of the input domain' (Section VII)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-3.5-turbo' and 'GPT-4' without specifying version snapshots (e.g., gpt-3.5-turbo-0613, gpt-4-0613). Model behavior changes across versions and no API version or snapshot date is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section IV provides the full text of all 5 prompt templates. The equation slot is filled by well-defined mutation rules (Section V, Figure 4) and the base equations are listed explicitly. A reader can reconstruct every prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VII states 'we used temperature and seed values of 0' and mentions testing different setups with temperature, top_p, and seed values."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system makes direct API queries to GPT models."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section VII documents the full pipeline: mutation generation via sympy, GPT response post-processing (extracting code, removing text), GCC compilation, output post-processing for format normalization, and differential testing with epsilon-based comparison."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII-C 'Threats to Validity' contains substantive discussion with both internal and external threat subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: API learning from prior requests with their mitigation (switching APIs), differential testing's inability to prove equivalence, epsilon value limitations for float comparison, and the limited set of equations being non-representative."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "External threats section states: 'We analyze a small set of equations with a limited number of possible mutations, which we recognize may not be representative of all possible equations and mutations given to LLMs for code generation.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 627 equation variations, 3135 generated code responses, and differential testing results are not released. The repository link is a placeholder."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section VII describes data collection in detail: 20 variations per distance for distances 1-5, 627 total variations for 7 equation types, 5 queries per variation via GPT API, 1000 random test inputs per generated code for differential testing."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Test data is deterministically generated from formal grammar rules and mutation operations, not sampled from an external source."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section VII and Figure 7 document the full pipeline: prompt generation → GPT API query → response post-processing → GCC compilation → differential testing → equivalence analysis. Each step is described with implementation details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Footnote on page 1 discloses NSF funding: Award #2124039, Award #2008660, Award #1901098."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from University of California, Santa Barbara. They are not affiliated with OpenAI or any company whose products they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF (National Science Foundation) is a government funding agency with no financial interest in the performance of GPT-3.5 or GPT-4."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-3.5-Turbo or GPT-4 despite evaluating these models' code generation capabilities."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While the prompts are generated programmatically (reducing contamination risk for the mutations), the base equations (e.g., a*x+b=0) and their solutions are certainly in GPT training data. This is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the base mathematical equations and their solutions appeared in GPT training data. The models likely saw canonical forms of these equations many times during training, which could affect the baseline robustness measurement."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "3135 API queries to GPT-3.5 and GPT-4 were made. No API costs, token counts, or per-query latency are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (Section VIII-A) but total compute budget — API spend, total runtime, or GPU/CPU hours — is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "A fixed seed of 0 was used for determinism. The paper mentions testing 'different setups with temperature, top p, and seed values' but does not report sensitivity to seed choice or variation across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly states: 'we have asked GPT to provide code five times for the same prompt' yielding 3135 queries (627 × 5)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section VII mentions testing 'different setups with temperature, top p, and seed values and chose the one that provides the most deterministic results' but does not report how many configurations were tried or the search method."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection criterion is vaguely stated as 'the one that provides the most deterministic results' without specifying what 'most deterministic' means, how it was measured, or on what data the selection was made."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates its own reduction pre-processing method and compares it against the no-reduction baseline. No discussion of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5 and GPT-4 differ substantially in compute cost, but the comparison is made without discussing compute differences. The pre-processing step's computational cost relative to its benefit is also not reported."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether syntactic robustness on mathematical equations generalizes to other forms of code generation robustness. The benchmark is limited to 7 equation types with algebraic mutations, but no discussion of whether this measures robustness more broadly."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used — direct API calls to GPT models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The base mathematical equations (e.g., a*x+b=0, quadratic formula) and their solutions are ubiquitous in training data. The paper does not discuss whether the models' familiarity with canonical forms of these equations affects baseline robustness or the observed robustness decay for mutated forms."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The prompts explicitly state the equation type and ask for a solver, which provides substantial hints about the expected solution approach."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All 627 variations are derived from only 7 base equations via mutations. These are structurally non-independent (mutations of the same equation share most of their structure). This is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-3.5 and GPT-4 are not syntactically robust for code generation with mathematical formulas.",
       "evidence": "Figures 8, 10, 11 show syntactic robustness degree below 100% for all syntactic distances. GPT-4 ranges from 85% (distance 1) to 36% (distance 5); GPT-3.5 from 70% to 16% (Section VIII-B, RQ1).",
       "supported": "moderate"
     },
     {
       "claim": "Syntactic robustness degree decreases as the number of mutations (syntactic distance) increases.",
       "evidence": "Figure 8 shows consistent negative correlation for both models. GPT-4: 85%, 63%, 54%, 51%, 36% for distances 1-5. GPT-3.5: 70%, 51%, 33%, 16%, 16% (Section VIII-B, RQ2). One exception noted for quadratic equations in GPT-4.",
       "supported": "strong"
     },
     {
       "claim": "Pre-processing with formula reduction achieves 100% syntactic robustness.",
       "evidence": "Figure 8 shows reduced form achieves 100% syntactic robustness for all equation types and both GPT models (Section VIII-B, RQ4).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 has 1.55x higher syntactic robustness than GPT-3.5.",
       "evidence": "Averaged across all distances and equations: GPT-4 outperforms GPT-3.5 by 1.55x (Section VIII-B, RQ1). Individual breakdowns shown in Figures 8-12.",
       "supported": "moderate"
     },
     {
       "claim": "Subtraction mutations cause the most confusion, swap mutations the least.",
       "evidence": "Figure 9 shows swap has highest robustness (34%/58% for GPT-3.5/4) and subtraction has lowest (22%/41%) (Section VIII-B, RQ3).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very narrow evaluation domain",
       "detail": "Only 7 equation types with 5 mutation rules in C code prompts, yet the paper title and framing suggest broad conclusions about 'LLM-based Code Generation.' The mathematical equation domain is a small slice of code generation use cases."
     },
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "Each prompt was queried 5 times to address non-determinism, but no standard deviation, variance, or spread across runs is reported. Only aggregate robustness degrees are shown, hiding any run-to-run variation."
     },
     {
       "flag": "100% claim is circular by design",
       "detail": "The reduction rules are the inverse of the mutation rules (explicitly noted in Section VI). Reducing a mutated formula recovers the original, so achieving 100% robustness is expected by construction and does not demonstrate generalizability to arbitrary formula complexity."
     },
     {
       "flag": "Broken repository link",
       "detail": "Section VII states 'The working pipeline is available at Link' — this is a placeholder URL. The code and data cannot be independently verified."
     },
     {
       "flag": "No model version pinning",
       "detail": "GPT-3.5-Turbo and GPT-4 are used without specifying API version or snapshot date. Results may not be reproducible as OpenAI updates these models."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark for LLM code generation evaluation, which is the standard benchmark in this domain."
     },
     {
       "title": "An empirical study of the code generation of safety-critical software using LLMs",
       "authors": ["M. Liu", "J. Wang", "T. Lin"],
       "year": 2024,
       "relevance": "Studies LLM code generation for safety-critical software, arguing for the need for robustness evaluation."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["A. Fan", "B. Gokkaya", "M. Harman"],
       "year": 2023,
       "arxiv_id": "2310.03533",
       "relevance": "Comprehensive survey of LLMs for software engineering that defined robustness for code generation, which this paper extends."
     },
     {
       "title": "LLM is like a box of chocolates: the non-determinism of ChatGPT in code generation",
       "authors": ["S. Ouyang", "J. Zhang", "M. Harman"],
       "year": 2023,
       "relevance": "Empirical study of non-determinism in LLM code generation, directly relevant to this paper's handling of non-deterministic outputs."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. Xia", "Y. Wang"],
       "year": 2023,
       "relevance": "Framework for rigorous evaluation of LLM code generation correctness, relevant methodology for code generation assessment."
     },
     {
       "title": "Coco: Testing code generation systems via concretized instructions",
       "authors": ["M. Yan", "J. Chen", "J. M. Zhang"],
       "year": 2023,
       "arxiv_id": "2308.13319",
       "relevance": "Tests code generation robustness via concretized instructions — a different robustness definition that this paper contrasts against."
     },
     {
       "title": "On the robustness of code generation techniques: An empirical study on GitHub Copilot",
       "authors": ["A. Mastropaolo", "L. Pascarella", "E. Guglielmi"],
       "year": 2023,
       "relevance": "Empirical study of code generation robustness using semantically equivalent natural language descriptors — complementary approach to this paper's formula mutations."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI", "J. Achiam", "S. Adler"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the two models evaluated in this study."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["D. Huang", "Q. Bu", "J. M. Zhang"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent code generation approach achieving 96% on HumanEval, showing the state-of-the-art in LLM code generation."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["F. F. Xu", "U. Alon", "G. Neubig"],
       "year": 2022,
       "relevance": "Systematic evaluation of code LLMs providing broader context for evaluating code generation capabilities."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li", "D. Choi", "J. Chung"],
       "year": 2022,
       "relevance": "Demonstrates competition-level code generation capability, relevant to understanding the state of LLM code generation."
     }
   ]
 }