ai-research-survey

scan.json (19845B)

---

 {
   "paper": {
     "title": "CHATASSERT: LLM-Based Test Oracle Generation With External Tools Assistance",
     "authors": ["Ishrak Hayet", "Adam Scott", "Marcelo d'Amorim"],
     "year": 2025,
     "venue": "IEEE Transactions on Software Engineering",
     "doi": "10.1109/TSE.2024.3519159"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/ncsu-swat/chatassert (Section I, end of contributions list)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the publicly available TECO dataset. The paper states 'We evaluate CHATASSERT on a part of the evaluation set of the TECO dataset' which is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is described in the paper. Tools like JavaParser and UniXCoder are mentioned but no version details or environment specs are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README instructions or reproduction scripts are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates in tables (e.g., Acc@1 = 0.45). No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Chi-Square tests with p-values are reported for key comparisons (e.g., 'Chi-Square value is 66.76 with a p-value of 3.04 × 10^-16' in RQ2; 'Chi-Square value is 13.02 with a p-value of 0.0003' in RQ3). Mann-Whitney U implied for mutation strength (p-value < 0.05)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's h reported for accuracy comparisons (1.3 for RQ2, 0.58 for RQ3) and Cliff's Delta for mutation strength (0.51 and 0.38)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Sample size of 500 is justified: 'We randomly sample 500 distinct samples from a total of 3,540 examples in the TECO dataset for a 98% confidence level and 5% error margin' (Section V-A3)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across runs. Results appear to be single-run numbers. The stochastic nature of LLM outputs means variance across runs would be relevant."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: ATLAS, TOGA, TECO (prior work), plus four LLMs without CHATASSERT's prompt engineering (CHATGPT, Mistral, Codestral, Magicoder)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "TECO (2023) is the SoTA at time of submission. Codestral and Mistral are contemporary open-source LLMs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table VI shows ablation removing each of the four features (CS, EX, SR, DR) individually, measuring impact on all metrics."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: Acc@1/3/5/10, BLEU, CodeBLEU, ROUGE, EditSim, and mutation scores."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated oracles. Evaluation is entirely automated (exact match, NLP metrics, mutation testing). Human judgment of oracle quality/usefulness would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "They use the TECO evaluation dataset (a separate test set from TECO's training data), with a randomly sampled subset of 500 examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-project or per-category breakdown of results. Only aggregate numbers across all 500 samples are reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A discusses negative examples where CHATASSERT fails (anonymous inner classes, specific test cases). Fig. 9 shows concrete failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that TECO outperforms CHATASSERT on some NLP metrics (EditSim, CodeBLEU in some configs), and that EX feature had less impact. Also notes CHATASSERT-DR has lower NLP metrics than TECO (Table V)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'improves Acc@1 by 15%' is supported by Table IV (CHATASSERT 0.45 vs TECO 0.30). 'Improves Acc@10 by 12%' matches Table IV (0.54 vs 0.42)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about feature contributions are supported by controlled ablation study (Table VI) where each feature is removed individually. The ablation design is adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim generality for 'LLM-based oracle generation' but results are only on Java projects from the TECO dataset. No explicit bounding to Java or to this specific dataset's characteristics."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section is generic. No substantive discussion of alternative explanations for the improvements — e.g., whether ChatGPT may have seen TECO dataset projects in training, or whether the improvement comes from additional compute (more LLM queries) rather than the specific features."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT3.5' / 'CHATGPT' without specifying a version or snapshot date (e.g., gpt-3.5-turbo-0613). Mistral-7B and Magicoder-6.7B and Codestral-22B give sizes but not specific versions/snapshots."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 5 shows the CHATGPT-ONE prompt template. Figs. 2 and 4 show detailed prompt examples with actual text for the full CHATASSERT pipeline including code summarization and repair prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is mentioned conceptually ('a variable that sets the level of (un)predictability') but the actual temperature setting used is not specified. 'Default temperature' is stated for CHATGPT-ONE but the default value is not given. No other hyperparameters (top-p, max tokens) reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The iterative scaffolding is described in detail: Algorithm 1 shows the full pseudocode, Table I describes features, and the generation/repair loop with feedback mechanisms is thoroughly documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A1 describes normalization (semantic-preserving transformations on assertions). Section V-A3 describes the sampling procedure with stratified sampling, exclusion of Hamcrest assertions, and helper method assertions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C 'Threats to Validity' provides a dedicated subsection discussing construct, internal, and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: sampling strategy limitations, possibility of test cases in ChatGPT training data (with Acc@1=0.10 mitigation argument), choice of metrics, and the specific dataset used."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of what the results do NOT show. The paper does not bound its claims to Java, to the TECO dataset characteristics, or state what populations/settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual per-sample results, LLM outputs) is made available. Only aggregate metrics in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: they use the TECO evaluation dataset with a stratified random sampling procedure selecting 500 samples from 3,540, with specific inclusion/exclusion criteria (Section V-A3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from a standard benchmark dataset (TECO)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The sampling pipeline is documented: 51 projects → 10 per project → exclude Hamcrest/helper methods → fill to 500. The evaluation pipeline (compile, run, match) is described in Algorithm 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "NSF grants disclosed: 'This work was supported by the National Science Foundation under Grant 2319472 and Grant 2349961.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with North Carolina State University. No evaluated product is from their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is a government funding agency with no financial interest in whether CHATASSERT outperforms baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date stated for any of the models used (GPT-3.5, Mistral, Codestral, Magicoder)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C discusses this threat: 'Since ChatGPT is a closed model and the training data are not available to end users... it is difficult to determine whether a test case belongs to the training dataset.' They use low Acc@1 (0.10) as evidence against significant contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The TECO dataset uses open-source Java projects that predate GPT-3.5's training. While contamination is mentioned in threats, no concrete analysis is done (e.g., temporal analysis, canary strings). The low Acc@1 argument is weak evidence."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B2 reports '∼10m, on average, for CHATASSERT to generate an assertion' and discusses the cost implications of multiple ChatGPT API calls."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, GPU hours, or overall computational budget is stated. Only average per-assertion time is mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "CHATASSERT improves Acc@1 by 15% over TECO (SoTA)",
       "evidence": "Table IV: CHATASSERT Acc@1 = 0.45 vs TECO Acc@1 = 0.30. Chi-Square = 13.02, p = 0.0003, Cohen's h = 0.58.",
       "supported": "strong"
     },
     {
       "claim": "CHATASSERT's prompt engineering significantly improves all tested LLMs",
       "evidence": "Table III: CHATASSERT-CHATGPT Acc@10 = 0.54 vs CHATGPT Acc@10 = 0.28. Chi-Square = 66.76, p = 3.04e-16, Cohen's h = 1.3. Similar improvements for Mistral, Magicoder, Codestral.",
       "supported": "strong"
     },
     {
       "claim": "All four features (CS, EX, SR, DR) contribute positively to CHATASSERT's performance",
       "evidence": "Table VI ablation: removing any feature reduces Acc@10 by 6-7%. CS has largest impact on Acc@1 (11 ppt drop).",
       "supported": "strong"
     },
     {
       "claim": "CHATASSERT-generated oracles have significantly higher mutation strength than TECO",
       "evidence": "Fig. 6(b): p-value < 0.05, Cliff's Delta = 0.51 (medium effect). CHATASSERT-DR also higher: Cliff's Delta = 0.38.",
       "supported": "strong"
     },
     {
       "claim": "CHATASSERT-DR (without dynamic repair) still outperforms TECO",
       "evidence": "Table V: CHATASSERT-DR Acc@1 = 0.35 vs TECO 0.30, Acc@10 = 0.47 vs 0.42.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CHATASSERT, a prompt engineering framework for LLM-based test oracle generation using code summarization, examples, and static/dynamic repair, improves Acc@1 by 15 percentage points over the state-of-the-art TECO on the TECO dataset (0.45 vs 0.30). The approach generalizes across four LLMs (ChatGPT, Mistral, Codestral, Magicoder) with significant improvements. Ablation shows all four features contribute, with code summarization being most impactful for Acc@1. Generated oracles also demonstrate significantly higher mutation-killing ability than TECO's.",
   "red_flags": [
     {
       "flag": "No variance across runs",
       "detail": "LLM outputs are stochastic, yet all results appear to be single-run. No standard deviation or multiple-run analysis is reported despite using temperature-dependent API calls."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "GPT-3.5 is used without a snapshot version. Model behavior changes across versions, making reproduction uncertain."
     },
     {
       "flag": "Weak contamination argument",
       "detail": "The argument that low Acc@1 (0.10) means ChatGPT hasn't seen the test cases is weak — partial memorization could still inflate results without producing exact matches."
     }
   ],
   "cited_papers": [
     {
       "title": "Learning deep semantics for test completion",
       "authors": ["P. Nie", "R. Banerjee", "J. J. Li", "R. J. Mooney", "M. Gligoric"],
       "year": 2023,
       "relevance": "TECO is the SoTA baseline for test oracle generation that CHATASSERT outperforms."
     },
     {
       "title": "CODAMOSA: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["C. Lemieux", "J. P. Inala", "S. K. Lahiri", "S. Sen"],
       "year": 2023,
       "relevance": "LLM-augmented test generation technique combining search-based testing with LLMs."
     },
     {
       "title": "Automated program repair via conversation: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2024,
       "relevance": "ChatRepair uses conversational LLM interaction for program repair, similar iterative approach to CHATASSERT."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex/HumanEval benchmark paper foundational to LLM code generation evaluation."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei"],
       "year": 2022,
       "relevance": "CHATASSERT's code summarization pipeline is analogous to chain-of-thought prompting."
     },
     {
       "title": "TOGA: A neural method for test oracle generation",
       "authors": ["E. Dinella", "G. Ryan", "T. Mytkowicz", "S. K. Lahiri"],
       "year": 2022,
       "relevance": "Grammar-based neural oracle generation baseline compared against CHATASSERT."
     },
     {
       "title": "On learning meaningful assert statements for unit test cases",
       "authors": ["C. Watson", "M. Tufano", "K. Moran", "G. Bavota", "D. Poshyvanyk"],
       "year": 2020,
       "relevance": "ATLAS: early deep learning approach for assertion generation, baseline in this study."
     },
     {
       "title": "Neural-based test oracle generation: A large-scale evaluation and lessons learned",
       "authors": ["S. B. Hossain", "A. Filieri", "M. B. Dwyer", "S. Elbaum", "W. Visser"],
       "year": 2023,
       "relevance": "Large-scale evaluation of neural oracle generation techniques including TOGA."
     },
     {
       "title": "Less training, more repairing please: Revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "Zero-shot LLM-based program repair, related to CHATASSERT's repair features."
     },
     {
       "title": "Code generation tools (almost) for free? A study of few-shot, pre-trained language models on code",
       "authors": ["P. Bareiß", "B. Souza", "M. d'Amorim", "M. Pradel"],
       "year": 2022,
       "arxiv_id": "2206.01335",
       "relevance": "Study of few-shot LLM code generation by overlapping authors, relevant to LLM code capabilities."
     }
   ]
 }