ai-research-survey

scan.json (29098B)

---

 {
   "paper": {
     "title": "Static Program Analysis Guided LLM Based Unit Test Generation",
     "authors": [
       "Sujoy Roychowdhury",
       "Giriprasad Sridhara",
       "A K Raghavan",
       "Joy Bose",
       "Sourav Mazumdar",
       "Hamender Singh",
       "Srinivasan Bajji Sugumaran",
       "Ricardo Britto"
     ],
     "year": 2024,
     "venue": "CODS-COMAD Dec '24",
     "arxiv_id": "2503.05394",
     "doi": "10.1145/3703323.3703742"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Augmenting LLM prompts with static program analysis information (declaring class, method signatures, field types) significantly increases the number of Java focal methods for which unit tests are generated compared to a whole-file baseline approach. On a commercial project (103 methods), the approach generated tests for 102/103 methods with llama 7b vs 37/103 for the baseline. Prompt token counts dropped from a mean of 5295 to 559. However, only test generation (not quality) was measured, and no statistical tests were applied.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or release link is provided anywhere in the paper. No mention of plans to release code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The commercial project data is proprietary and not released. The specific focal methods selected from Guava are not identified (only DoubleUtils is named in the example). No dataset is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2.1 mentions 'single A100 80 GB GPU' and '8 bit quantization' but provides no requirements.txt, Dockerfile, library versions, or sufficient detail to recreate the environment. The Java Parser Library is mentioned without a version."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, scripts, or step-by-step guides are provided. A reader would have to reverse-engineer the entire experimental setup from the paper's prose."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only raw counts with no confidence intervals or error bars. Table 3 reports mean/std/median for token counts but not for the main evaluation metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims their approach generates tests for more focal methods but provides no statistical tests (no p-values, chi-squared, or any test of significance) to support the comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 show both baseline and proposed approach counts side by side (e.g., 37 vs 102 for llama7b on commercial, 20 vs 30 for llama7b on Guava), providing baseline context for the reader to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 103 commercial focal methods or 34 open-source methods were chosen. No power analysis. Section 5.1 simply states 'We selected a random sub-project' without discussing whether the sample size is adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported for the main evaluation results (test generation counts). Table 3 reports std dev for token counts and Table 4 for time, but the primary metric has no variance measure. Temperature is set to 0 but this is not discussed in terms of result determinism."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against a baseline approach from Siddiq et al. [10] that provides the entire source file as context. Both Tables 1 and 2 show baseline vs proposed approach."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline (Siddiq et al. 2023 [10]) is from the prior year, which is contemporary for a 2024 publication. The related work also discusses recent approaches like RLPG [9] and Monitor Guided Decoding [1], both from 2023."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system augments prompts with three types of information (declaring class, method signatures, field types) but no ablation study tests the contribution of each component individually. Only the full approach vs baseline is compared."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only one metric is used: the number of focal methods for which one or more unit tests were generated. Section 8 acknowledges this limitation: 'One limitation of this study is that we are only looking at generated test cases rather than their quality.'"
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated tests is performed. The only metric is automated counting of whether tests were generated, not their correctness, readability, or usefulness."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of separating development and test data. The motivating example in Listing 1 (getEmail method) appears to be from the evaluation set. No evidence that prompt design was done on separate data from evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (llama7b, llama70b, codellama34b, gpt-4) in Tables 1-2 and by project type (commercial vs open source). Token statistics are also broken down by approach in Tables 3-4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss specific failure cases where their approach failed to generate tests. Section 7 hypothesizes why the baseline fails but does not analyze the 1 (of 103) commercial method or 3-15 open-source methods where their approach did not generate tests."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every comparison shows their approach outperforming the baseline. No configurations or design decisions that were tried and abandoned are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'augmenting prompts with concise and precise context information obtained by program analysis increases the effectiveness of generating unit test code through LLMs,' which is supported by Tables 1-2. The claim of validation on commercial and open-source projects is also supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that program analysis augmentation improves test generation is tested via a controlled comparison: same models, same focal methods, different prompt strategies. This controlled single-variable manipulation (prompt content) is adequate for the causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Java but Section 7 (Portability) claims 'it should be portable across languages' without evidence. The title uses the general 'Unit Test Generation' without bounding to Java. Results are from only 2 projects (1 commercial, 1 open source) but claims are framed broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 offers only one explanation for the baseline's failure (context length/extraneous information). No alternative explanations for their own approach's success are considered — e.g., whether the improvement comes from shorter prompts alone, from the specific information content, or from prompt structure."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly acknowledges the proxy gap: 'One limitation of this study is that we are only looking at generated test cases rather than their quality — this is a scope for future study.' The metric (generation count) is distinguished from the broader outcome (test effectiveness)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Llama models are specified as 'llama 7b/70b 2.0 and CodeLlama 34b based on llama 2.0' which includes version and size. However, GPT-4 is used in Table 2 without any version or snapshot date, violating the criterion."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 provides the full prompt text used in Listing 1 (appendix), including the system instruction, focal method, declaring class, method signatures, and fields. The baseline prompt structure is also described. The actual prompt text is shown, not just a description."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.1 reports: temperature=0, topK=50, topP=0.95, input tokens limited to 1023, context length 4096, 8-bit quantization. 'Other parameters are kept at their default.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct prompting of LLMs with program analysis-augmented prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We selected a random sub-project from our commercial Java project' and 'two random Java classes' from Guava but does not describe how randomization was done, what criteria defined a 'sub-project,' or how focal methods were enumerated from the selected code."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Two limitations are mentioned in Section 8 (Conclusion and Future Work) but they are embedded in the conclusion paragraph, not a substantive standalone discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 identifies two specific threats: (1) 'we are only looking at generated test cases rather than their quality' and (2) 'not all dependencies, especially those at runtime, can be analyzed via static program analysis — such limitations would affect our results too.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not clearly bound its scope. While acknowledging the quality limitation, the Portability section (Section 7) claims cross-language applicability without evidence. No explicit statement of what the results do NOT show beyond the quality caveat."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The commercial project is proprietary, and neither the specific focal methods, generated tests, nor detailed per-method results are released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 says 'We selected a random sub-project from our commercial Java project and this had 103 focal methods' and 'two random Java classes' from Guava. The randomization process, selection criteria, and specific classes/methods are not described in adequate detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The study evaluates LLM-generated unit tests on code from existing projects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from code selection to evaluation results is not documented step-by-step. How Java Parser extracted information, how prompts were assembled, how output was evaluated (manual or automated counting) — these steps are described only at a high level."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present. Seven of eight authors are from Ericsson R&D, suggesting corporate-funded research, but this is not explicitly stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: seven from Ericsson R&D (Bangalore/Stockholm) and one independent researcher. The commercial project is described as 'from a telecom company,' making the Ericsson connection apparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Ericsson employees developed the approach and evaluated it on Ericsson's own commercial codebase. Ericsson has a direct interest in demonstrating that their approach improves developer productivity. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 2 models and GPT-4 without stating their training data cutoff dates. This is relevant because the models may have seen Guava source code and its existing tests during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Guava is one of the most popular Java libraries and its source code is widely available on GitHub. The LLMs almost certainly encountered Guava code during training, yet no discussion of potential overlap between training data and evaluation subjects is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Guava's source code and existing test suites have been publicly available for over a decade. Llama 2 and GPT-4 likely trained on this code. The paper does not address whether the LLMs might already know Guava's test patterns, which would inflate results for the open-source evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports mean, std dev, and median time per inference for both baseline and the proposed approach. Section 7 discusses the cost advantage in token-based pricing systems. Section 5.2.1 specifies the hardware used (single A100 80GB GPU)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-inference time statistics and GPU type are reported, the total computational budget (total GPU hours, total API spend, total experiment wall-clock time) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 (Section 5.2.1), suggesting deterministic outputs, but the paper does not explicitly discuss result determinism or report results across multiple seeds/runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each experiment was run. With temperature=0 the output should be deterministic, but this is not explicitly stated or verified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of how the hyperparameters (temperature=0, topK=50, topP=0.95) or prompt format were selected. No search budget or alternative configurations tried are reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The prompt structure and hyperparameter settings appear to be a single chosen configuration with no justification for why these specific values were selected or how they were determined."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement the baseline from [10] themselves and compare it against their own proposed approach. This self-comparison bias (Lucic et al. 2018) is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 shows their approach uses ~10x fewer tokens than the baseline, and Table 4 shows similar inference times. However, no systematic analysis of performance as a function of compute budget is provided, and the token cost advantage is not quantified in monetary or compute terms."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The sole metric is whether any test was generated, but the paper does not discuss whether this measures the claimed 'effectiveness' of test generation. Generating a test that compiles, passes, or achieves coverage is not addressed. Section 8 acknowledges quality is not measured but does not question the metric's construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding or agentic framework is used. The approach is direct LLM prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Guava has been publicly available since 2009 and Llama 2's training data likely includes it. GPT-4's training data almost certainly includes Guava source and tests. No discussion of temporal leakage is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether providing the focal method body as input constitutes feature leakage or whether the models already know the test patterns for these methods from training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Focal methods are selected from within the same project (same codebase, same coding style, same architecture). No discussion of whether methods within the same project are independent test cases."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, or decontamination analysis is performed."
       }
     }
   },
   "claims": [
     {
       "claim": "Augmenting prompts with static program analysis information increases the number of focal methods for which unit tests are generated compared to the whole-file baseline.",
       "evidence": "Tables 1-2 show consistent improvements across all models: on 103 commercial methods, llama7b generates tests for 102 vs 37 (baseline), llama70b 88 vs 9, codellama34b 63 vs 7. On 34 open-source methods, improvements range from 1→19 (codellama34b) to 32→34 (GPT-4).",
       "supported": "moderate"
     },
     {
       "claim": "The program analysis approach produces significantly more concise prompts than the whole-file baseline.",
       "evidence": "Table 3 shows mean token counts of 559 (std 516) vs 5295 (std 3350), and median of 464 vs 4056. This is a ~10x reduction in prompt length.",
       "supported": "strong"
     },
     {
       "claim": "The approach should be portable across programming languages.",
       "evidence": "Section 7 (Portability) argues this based on the availability of parsers and symbolic resolvers for all languages, but provides no empirical evidence from non-Java languages.",
       "supported": "unsupported"
     },
     {
       "claim": "The baseline fails due to increased context length and extraneous information in the prompt.",
       "evidence": "Section 7 states this as a hypothesis supported by the token length statistics in Table 3. However, no controlled experiment isolates context length from information content as the causal factor.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own approach on its own codebase",
       "detail": "Seven of eight authors are from Ericsson R&D, and the primary evaluation (103 methods) is on Ericsson's own commercial Java project. The company has a direct interest in demonstrating the approach works. No independent evaluation is provided."
     },
     {
       "flag": "Very small sample sizes with no statistical testing",
       "detail": "The evaluation uses only 103 commercial focal methods and 34 open-source methods. No statistical significance tests, confidence intervals, or power analysis support the claims of improvement."
     },
     {
       "flag": "Generation-only metric ignores test quality",
       "detail": "The sole metric is whether any test was generated, ignoring compilation success, correctness, assertion quality, branch coverage, or test smells. A generated test that doesn't compile or test anything meaningful would count as a success. The authors acknowledge this limitation."
     },
     {
       "flag": "Contamination risk for open-source evaluation",
       "detail": "Guava is one of the most widely-used Java libraries. LLMs like Llama 2 and GPT-4 almost certainly trained on Guava source code and its existing test suite, potentially inflating results on the open-source evaluation."
     },
     {
       "flag": "No code or data released",
       "detail": "Neither the implementation, the specific focal methods evaluated, nor the generated tests are released. Results cannot be independently verified or reproduced."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "Section 7 claims the approach 'should be portable across languages' and 'should also work for different complex methods' based solely on Java experiments on 2 projects."
     }
   ],
   "cited_papers": [
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2023,
       "arxiv_id": "2302.06527",
       "relevance": "Directly relevant as an LLM-based unit test generation approach that relies on sample usages of the focal method."
     },
     {
       "title": "An Empirical Study of Using Large Language Models for Unit Test Generation",
       "authors": ["Mohammed Latif Siddiq", "Joanna C. S. Santos", "Ridwanul Hasan Tanvir", "Noshin Ulfat", "Fahmid Al Rifat", "Vinicius Carvalho Lopes"],
       "year": 2023,
       "arxiv_id": "2305.00418",
       "relevance": "Serves as the baseline approach in this paper's evaluation — treats unit test generation as a code completion task using the entire source file."
     },
     {
       "title": "Repository-level prompt generation for large language models of code",
       "authors": ["Disha Shrivastava", "Hugo Larochelle", "Daniel Tarlow"],
       "year": 2023,
       "relevance": "Related approach to augmenting LLM prompts with additional project context for code completion tasks."
     },
     {
       "title": "Monitor-Guided Decoding of Code LMs with Static Analysis of Repository Context",
       "authors": ["Lakshya Agrawal", "Aditya Kanade", "Navin Goyal", "Shuvendu K Lahiri", "Sriram Rajamani"],
       "year": 2023,
       "relevance": "Combines static analysis with LLMs for code generation via a white-box decoding approach, contrasting with this paper's black-box prompting."
     },
     {
       "title": "Unit test case generation with transformers and focal context",
       "authors": ["Michele Tufano", "Dawn Drain", "Alexey Svyatkovskiy", "Shao Kun Deng", "Neel Sundaresan"],
       "year": 2020,
       "relevance": "Encoder-decoder transformer approach to unit test generation using focal method pairs, foundational work in neural test generation."
     },
     {
       "title": "Toga: A neural method for test oracle generation",
       "authors": ["Elizabeth Dinella", "Gabriel Ryan", "Todd Mytkowicz", "Shuvendu K Lahiri"],
       "year": 2022,
       "relevance": "Neural approach to automatically generating test oracles, a key component of unit test generation."
     },
     {
       "title": "EvoSuite: Automatic Test Suite Generation for Object-Oriented Software",
       "authors": ["Gordon Fraser", "Andrea Arcuri"],
       "year": 2011,
       "doi": "10.1145/2025113.2025179",
       "relevance": "Foundational search-based test generation tool, represents the non-AI baseline approach to automated testing."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the LLMs evaluated in this paper (CodeLlama 34b), a code-specialized open-source model."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The approach of augmenting prompts with static analysis info is practically useful for developers, but no tool or code is released for immediate use."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected intuition that providing more precise context to LLMs improves output quality."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released — cannot try it."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Ericsson is a well-known telecom company but not prominent in AI research circles."
     }
   }
 }