ai-research-survey

scan.json (28708B)

---

 {
   "paper": {
     "title": "CITYWALK: Enhancing LLM-Based C++ Unit Test Generation via Project-Dependency Awareness and Language-Specific Knowledge",
     "authors": [
       "Yuwei Zhang",
       "Qingyuan Lu",
       "Kai Liu",
       "Wensheng Dou",
       "Jiaxin Zhu",
       "Li Qian",
       "Chunxi Zhang",
       "Zheng Lin",
       "Jun Wei"
     ],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2501.16155",
     "doi": "10.1145/3763791"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We publicly release the artifacts [53] of CITYWALK on Zenodo to facilitate the reproduction' (Section 1). Reference [53] points to https://zenodo.org/records/14022506."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark of 1288 focal methods across ten C++ projects is based on publicly available GitHub repositories whose URLs are provided (footnotes 4-13). The Zenodo artifact package [53] is described as a 'Replicate Package' which facilitates reproduction."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Clang for parsing and llvm-cov for coverage, and Python for implementation, but does not provide a requirements.txt, Dockerfile, or detailed environment setup with library versions. No specific Python version, library versions, or dependency specifications are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a Zenodo artifact is released, the paper itself does not contain step-by-step reproduction instructions. The methodology is described in detail but there is no 'Reproducing Results' section or explicit commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '83.39% CSR', '73.35% EPR') without any confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CITYWALK 'outperforms' all baselines and 'surpasses the best baseline, GPT-4o, by 51.55% in CSR' but provides no statistical significance tests (no p-values, t-tests, or similar). Differences are compared by raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in terms of absolute percentage improvements with baseline context. For example, 'CITYWALK surpasses the best baseline, GPT-4o, by 51.55% in CSR, 43.76% in ERP, 24.78% in CovL, and 21.55% in CovB' (Section 4.1.1). Ablation results show per-component degradation values (Tables 14-15)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark comprises 1288 focal methods from 10 projects, but no justification is given for why this sample size is sufficient for the claims made. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments use greedy decoding (temperature=0) producing deterministic outputs, and results are reported from single runs. No variance, standard deviation, or spread across multiple runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven baselines are compared: two open-source code LLMs (CodeGeeX4, DeepSeek-V3), two closed-source LLMs (GPT-3.5, GPT-4o), and three LLM-based unit test generation approaches (ChatTester, HITS, TestPilot). Results in Tables 4-7."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DeepSeek-V3 (2024), GPT-4o (2023/2024), CodeGeeX4, and recent approaches like HITS (ASE 2024), ChatTester (FSE 2024), and TestPilot (TSE 2024). The paper notes that Coyote C++ and CITRUS are unavailable for reproduction (Section 3.3)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 presents comprehensive ablation studies removing one component at a time (Depc, Depd, Contextintent, PROMPTstep, GuidelineDK, Fixrule+prompt). Results shown in Tables 14-15. Additionally, phase-by-phase error-fixing ablation in Tables 16-17."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used: Compilation Success Rate (CSR), Execution Pass Rate (EPR), Line Coverage (CovL), and Branch Coverage (CovB). Additionally, mutation scores and human evaluation scores are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 describes a human evaluation with five participants (each with >3 years C++ experience) who scored 150 test cases across four aspects: Naming Intuitiveness, Code Layout, Assertion Quality, and Adoption Efforts. Results in Table 20."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Four of the ten projects (json.cpp, glomap, papy, mlx) were created after GPT-4o's training cutoff (October 2023), serving as a held-out test set to address data leakage concerns (Section 3.2, Table 3 'Trained?' column)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per-project across all ten C++ projects in Tables 4-7, Tables 10-13, and Tables 14-15. Error categories are broken down in Figure 8 and Table 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 provides a detailed 'Bad Case Breakdown' analyzing 1567 errors across 1021 failed test cases. Error categories include Access Error, Undefined Symbols, Syntax Error, etc. (Figure 8). Section 5.4 discusses false-positive executable test cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CITYWALK performs poorly on leveldb due to heavy external interactions (disk I/O, synchronization locks) in Section 4.1.2. It also acknowledges that 36.66% of generated test cases still fail (Table 8). Section 5.4 discusses the false positive problem."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CITYWALK 'outperforms current state-of-the-art approaches on a collection of ten popular C++ projects.' Tables 4-7 confirm CITYWALK achieves the best average scores across all four metrics on all ten projects."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., 'removing Depc leads to 29.49% drop in CovL'). The ablation design follows controlled single-variable manipulation: removing one component at a time while keeping others fixed (Section 4.2). This is adequate for causal claims about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly scopes to C++ unit test generation and acknowledges limitations: 'Future work will involve expanding the benchmark and integrating additional LLMs to better assess the generalizability' (Section 5.5). The title and claims are specific to C++."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 (Threats to Validity) discusses alternative explanations including data leakage ('GPT-4o is a closed-source model, the exact composition of its training data are not publicly disclosed'), prompt sensitivity effects, and the possibility that configuration dependency information alone could improve baselines (Section 4.1.1, finding 2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'we employ the gpt-4o version from the GPT family' and 'gpt-3.5-turbo-0125' (Section 4.1). For GPT-4o, no specific snapshot date or API version is given — just 'gpt-4o'. DeepSeek-V3 and CodeGeeX4 are named without version identifiers. Marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides the detailed prompting content for generating initial test cases, including the full prompt template with task definition, step-by-step instructions, and output format. Figure 6 shows the error-fixing prompt. Table 2 lists the domain knowledge guidelines. While some fields are placeholders (e.g., {method_name}), the actual structure and instructions are complete enough to reconstruct prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 states: 'maximum output limit of 4096 tokens', 'greedy decoding', 'top-1 chat completion choice', 'sampling temperature to 0', and 'zero-shot setting'. Maximum prompt length is 14733 tokens (footnote 14)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail across Sections 2.1-2.6: five-stage pipeline with project pre-processing, dependency extraction, intention context retrieval via RAG, empirical observation-based knowledge, and three-step generation with three-phase post-processing. Algorithm 1 provides pseudocode. Figures 2, 5, and 6 show workflow and prompting structure."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes preprocessing: AST parsing with Clang, structured focal context extraction, knowledge base construction with text segmentation and BGE embedding. Section 3.2 documents project selection criteria (>50 GitHub stars, actively maintained, spanning application domains, containing complex methods). The filtering from repository to focal methods is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 'Threats to Validity' provides a dedicated subsection discussing external and internal threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 discusses specific threats: the diversity of only ten C++ projects, GPT-4o's closed training data creating data leakage risk, LLM sensitivity to prompt configuration, single-round iterative fixes versus multi-round in original baselines, and the limitation to specific LLMs. These are specific to this study rather than generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: it focuses on C++ (not other languages), uses ten projects with specific selection criteria, excludes larger projects 'to effectively manage token costs within our limited budget' (Section 3.2), and notes that Coyote C++ and CITRUS are unavailable for comparison. Section 5.5 explicitly notes the need to expand to more LLMs and projects."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replicate package [53] is released. The benchmark projects are all publicly available on GitHub with URLs provided. The raw experimental data (test cases, error messages) can be regenerated from the released artifacts."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the benchmark collection: projects sourced from GitHub, four from prior C++ testing studies, two practical projects (ninja, leveldb), four post-cutoff projects. Selection criteria include >50 stars, active maintenance, diverse application domains, and complex methods (cyclomatic complexity >10)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation (Section 5.3), five participants 'each with over three years of C++ development experience' are mentioned, but recruitment methods are not described — how they were found, whether they are students/colleagues, or potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented across Sections 2-3: project cloning, AST parsing, focal method extraction (1288 methods), knowledge base construction, dependency analysis, test generation, compilation/execution, coverage measurement with llvm-cov. The flow from input to output is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding: 'National Key R&D Program of China (Grant No. 2024YFB4505902), the Major Project of ISCAS (Grant No. ISCAS-ZD-202302), the Basic Research Project of ISCAS (Grant No. ISCAS-JCZD-202403), the Youth Innovation Promotion Association of the Chinese Academy of Sciences (Grant Nos. Y2022044 and 2023121).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Institute of Software, Chinese Academy of Sciences; University of Chinese Academy of Sciences; and Shanghai Stock Exchange Technology Co., Ltd. Some authors are affiliated with the Shanghai Stock Exchange, but the paper evaluates open-source tools rather than a proprietary product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are Chinese government research programs (National Key R&D Program, CAS) and institutional grants. These are general research funding sources with no apparent financial stake in whether CITYWALK outperforms specific baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Some authors are from Shanghai Stock Exchange Technology Co., which could have commercial interest in automated testing tools. Absence of disclosure is noted."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states GPT-4o's 'knowledge cutoff date (October 2023)' in Section 3.2 when justifying the inclusion of four post-cutoff projects."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly addresses this: 'To mitigate potential data leakage concerns, we include four projects — json.cpp, glomap, papy, and mlx — all created after the GPT-4o knowledge cutoff date (October 2023). This ensures that GPT-4o was not trained on these projects.' Table 3 includes a 'Trained?' column."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses benchmark contamination by including post-cutoff projects and comparing results between potentially contaminated (6 projects, 'Y') and clean (4 projects, 'N') subsets. The mutation testing analysis in Section 5.1 shows json.cpp (post-cutoff) achieves higher mutation scores than tinyxml2 (pre-cutoff), providing 'evidence that CITYWALK's ability to generate high-quality test cases is primarily driven by the designed components as guidance, rather than relying on GPT-4o's memorization.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation study (Section 5.3) is not pre-registered. No link to any pre-registration platform is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation involved five participants scoring test cases (Section 5.3), but no IRB or ethics board approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only minimal characterization is provided: 'five participants, each with over three years of C++ development experience.' No further demographics (roles, education level, industry/academia, gender, geographic distribution) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The only stated criterion is '>3 years of C++ development experience.' No formal inclusion/exclusion criteria, screening process, or selection rationale is described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study comparing human groups. All participants evaluated the same set of test cases. Randomization of condition assignment is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 states: 'To avoid bias, participants are not informed about the source of each test case (i.e., whether it is LLM-generated or human-written).'"
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about whether all five participants completed all evaluations or if any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 19 reports average execution time per focal method (34.59s for CITYWALK) and average token usage (5726 tokens). Section 5.2 notes 'the incremental cost per method is approximately $0.03 higher than the best baseline TestPilot.' Section 3.2 mentions 'token costs within our limited budget.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-method costs are reported, the total computational budget (total API spend, total GPU hours, or total cost for all 1288 methods) is not stated. The paper mentions a 'limited budget' but does not quantify it."
       }
     }
   },
   "claims": [
     {
       "claim": "CITYWALK outperforms seven state-of-the-art baselines across all four evaluation metrics (CSR, EPR, CovL, CovB) on C++ unit test generation.",
       "evidence": "Tables 4-7 show CITYWALK achieves 83.39% CSR (vs. 31.84% best baseline GPT-4o), 73.35% EPR (vs. 29.59%), 44.46% CovL (vs. 19.68%), and 37.70% CovB (vs. 16.15%).",
       "supported": "strong"
     },
     {
       "claim": "CITYWALK generates fewer failed test cases than baselines, with only 36.66% failing.",
       "evidence": "Table 8 reports 1021 failed out of 2785 total test cases (36.66%), compared to 81.16% for GPT-4o and 89.01% for DeepSeek-V3.",
       "supported": "strong"
     },
     {
       "claim": "Configuration dependencies (Depc) are the most impactful component, with removal causing 29.49% drop in CovL and 24.58% drop in CovB.",
       "evidence": "Tables 14-15 show ablation results where w/o Depc has the largest average degradation across all components.",
       "supported": "strong"
     },
     {
       "claim": "CITYWALK integrates as a plug-in and consistently improves three additional LLMs (CodeGeeX4, DeepSeek-V3, GPT-3.5).",
       "evidence": "Tables 10-13 show consistent improvements when CITYWALK is applied to each LLM. Performance gains scale with model parameter count.",
       "supported": "strong"
     },
     {
       "claim": "CITYWALK-generated test cases achieve mutation scores of 81.12% (tinyxml2) and 89.34% (json.cpp), demonstrating bug-detection capability.",
       "evidence": "Table 18 reports mutation testing results. Section 5.1 notes these exceed EvoSuite's 34.1% and universalmutator's 35% on their respective benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "CITYWALK-generated test cases are preferred by developers over LLM baselines in readability and usability.",
       "evidence": "Table 20 shows five participants rated CITYWALK higher across Naming Intuitiveness (2.87), Code Layout (2.70), Assertion Quality (2.46), and Adoption Efforts (2.70) compared to all four LLM baselines.",
       "supported": "weak"
     },
     {
       "claim": "CITYWALK's performance is driven by designed components rather than GPT-4o's memorization of training data.",
       "evidence": "Four post-cutoff projects were included; json.cpp (post-cutoff) achieved higher mutation score (89.34%) than tinyxml2 (pre-cutoff, 81.12%). Section 5.1.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CITYWALK, an LLM-based framework for C++ unit test generation, substantially outperforms seven baselines including GPT-4o, achieving 83.39% compilation success rate (vs. 31.84% for GPT-4o) and 44.46% line coverage (vs. 19.68%) across 1288 focal methods from ten C++ projects. The framework's key contributions are project-level dependency extraction (configuration and cross-file data dependencies), RAG-based intention context retrieval, and language-specific domain knowledge derived from empirical error analysis. Ablation studies show configuration dependencies are the most impactful component, and the approach generalizes across multiple LLMs as a plug-in enhancement.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('outperforms', 'surpasses') are based on raw metric comparisons without any statistical tests. With single-run deterministic experiments (temperature=0), there is no way to assess whether observed differences are robust to variations in model behavior or sampling."
     },
     {
       "flag": "Very small human evaluation sample",
       "detail": "The human evaluation uses only 5 participants evaluating 25 focal methods. No inter-rater reliability measures are reported, no recruitment details are given, and no statistical analysis of the human ratings is performed. This sample is too small to make generalizable claims about developer preference."
     },
     {
       "flag": "Mutation score comparison across different benchmarks",
       "detail": "Section 5.1 compares CITYWALK's mutation scores (81-89%) with EvoSuite's score (34.1%) from the SBST 2022 competition and universalmutator's average (35%). These are on entirely different benchmarks, making the comparison misleading."
     },
     {
       "flag": "Single-run deterministic experiments",
       "detail": "All experiments use temperature=0 with a single run, so there is no variance reporting. While deterministic, different model API versions could produce different results, and the robustness of results to prompt variations is unknown."
     },
     {
       "flag": "Baselines limited to single-round fixes for fairness",
       "detail": "Section 4.1.1 notes that baselines (ChatTester, HITS) perform only a single round of iterative fixes 'similar to CITYWALK', which is 'much fewer than the number of iterations used in their original papers.' This may understate baseline performance relative to their published results."
     }
   ],
   "cited_papers": [
     {
       "title": "ChatUniTest: A Framework for LLM-Based Test Generation",
       "authors": ["Yinghao Chen", "Zehao Hu", "Chen Zhi", "Junxiao Han", "Shuiguang Deng", "Jianwei Yin"],
       "year": 2024,
       "doi": "10.1145/3663529.3663801",
       "relevance": "LLM-based unit test generation framework using generate-validate-fix mechanism, direct comparison baseline."
     },
     {
       "title": "Evaluating and Improving ChatGPT for Unit Test Generation",
       "authors": ["Zhiqiang Yuan", "Mingwei Liu", "Shiji Ding", "Kaixin Wang", "Yixuan Chen", "Xin Peng", "Yiling Lou"],
       "year": 2024,
       "doi": "10.1145/3660783",
       "relevance": "ChatTester: evaluation of ChatGPT for Java unit test generation with intent comprehension, direct comparison baseline."
     },
     {
       "title": "HITS: High-Coverage LLM-Based Unit Test Generation via Method Slicing",
       "authors": ["Zejun Wang", "Kaibo Liu", "Ge Li", "Zhi Jin"],
       "year": 2024,
       "doi": "10.1145/3691620.3695501",
       "relevance": "LLM-based high-coverage test generation through program decomposition, direct comparison baseline."
     },
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2024,
       "doi": "10.1109/TSE.2023.3334955",
       "relevance": "TestPilot: empirical evaluation of LLMs for automated unit test generation, direct comparison baseline."
     },
     {
       "title": "Code-Aware Prompting: A Study of Coverage-Guided Test Generation in Regression Setting using LLM",
       "authors": ["Gabriel Ryan", "Siddhartha Jain", "Mingyue Shang", "Shiqi Wang", "Xiaofei Ma", "Murali Krishna Ramanathan", "Baishakhi Ray"],
       "year": 2024,
       "doi": "10.1145/3643769",
       "relevance": "SymPrompt: uses symbolic execution-based path information in prompts for coverage-guided test generation."
     },
     {
       "title": "On the Evaluation of Large Language Models in Unit Test Generation",
       "authors": ["Lin Yang", "Chen Yang", "Shutao Gao"],
       "year": 2024,
       "doi": "10.1145/3691620.3695529",
       "relevance": "Evaluation framework for LLMs in unit test generation, establishing metrics used in this work."
     },
     {
       "title": "Automated Unit Test Improvement using Large Language Models at Meta",
       "authors": ["Nadia Alshahwan", "Jubin Chheda", "Anastasia Finogenova"],
       "year": 2024,
       "doi": "10.1145/3663529.3663839",
       "relevance": "Industrial application of LLMs for unit test improvement at Meta, relevant to practical LLM-based testing."
     },
     {
       "title": "ChatGPT vs SBST: A Comparative Assessment of Unit Test Suite Generation",
       "authors": ["Yutian Tang", "Zhijie Liu", "Zhichao Zhou", "Xiapu Luo"],
       "year": 2024,
       "doi": "10.1109/TSE.2024.3382365",
       "relevance": "Compares ChatGPT with search-based software testing for unit test generation."
     },
     {
       "title": "Software Testing With Large Language Models: Survey, Landscape, and Vision",
       "authors": ["Junjie Wang", "Yuchao Huang", "Chunyang Chen", "Zhe Liu", "Song Wang", "Qing Wang"],
       "year": 2024,
       "doi": "10.1109/TSE.2024.3368208",
       "relevance": "Comprehensive survey of LLM-based software testing covering the landscape and research directions."
     },
     {
       "title": "DeepSeek-V3 Technical Report",
       "authors": ["DeepSeek-AI"],
       "year": 2024,
       "arxiv_id": "2412.19437",
       "relevance": "Technical report for DeepSeek-V3, one of the LLMs evaluated as both baseline and CITYWALK integration target."
     },
     {
       "title": "LLM Hallucinations in Practical Code Generation: Phenomena, Mechanism, and Mitigation",
       "authors": ["Ziyao Zhang", "Yanli Wang", "Chong Wang", "Jiachi Chen", "Zibin Zheng"],
       "year": 2024,
       "arxiv_id": "2409.20550",
       "relevance": "Studies LLM hallucination in code generation, directly relevant to understanding errors in LLM-generated test code."
     }
   ]
 }