ai-research-survey

scan.json (21106B)

---

 {
   "paper": {
     "title": "CODAMOSA: Escaping Coverage Plateaus in Test Generation with Pre-trained Large Language Models",
     "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K. Lahiri", "Siddhartha Sen"],
     "year": 2023,
     "venue": "ICSE 2023",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/microsoft/codamosa. Section VII states 'Our source code is available at' with the URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section VII (Data availability) states the repository includes a replication folder with data from the similarity analysis, scripts to generate plots, and information on accessing raw evaluation data including Codex-generated test cases."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section VII mentions Docker containers for cloning projects, filtering modules, and running CODAMOSA. Built on Pynguin 0.19.0 (Section V-A)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section VII describes the replication folder containing: (1) docker container to clone projects and filter modules, (2) docker container to run CODAMOSA, (3) similarity analysis data, (4) scripts to generate plots."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars reported. Results are presented as counts of benchmarks with significant differences and average coverage values without uncertainty bounds."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B: 'Following Arcuri and Briand's guidelines, we use a Mann-Whitney U-Test to compare the significance of coverage differences between techniques, at p = 0.05.'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as absolute percentage point differences with baseline context, e.g., 'average magnitude of the coverage increase is also higher (10% and 9%) than the average magnitude of decreases (−4% and −3%)' in the abstract and Section V-B."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 486 benchmarks or 16 runs per benchmark were chosen. The 10-minute time budget is justified by reference to prior work (Pynguin evaluation) but the number of runs is not."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results report averages across 16 runs but no standard deviations, IQR, or other spread measures are provided for the coverage results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines: MOSA (search-based only) and CODEXONLY (LLM only). Additionally, a Union baseline combining MOSA and CODEXONLY test suites is evaluated."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MOSA is the state-of-the-art SBST algorithm implemented in Pynguin. The paper explains why DynaMOSA was not used (Pynguin 0.19.0 limitation). Codex was the leading publicly available code LLM at time of evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C evaluates four design decisions individually: uninterpreted statements (CODAMOSA-NOUNINTERP), temperature (CODAMOSA-TEMP-0.2), targeting strategy (CODAMOSA-RANDOM), and prompting (CODAMOSA-TESTCASEPROMPT)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Uses both line coverage and branch coverage (combined as 'line + branch coverage'). Also reports test suite size and number of benchmarks with significant improvements/decreases."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here; the paper evaluates automated test generation where code coverage is the appropriate automated metric."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not a machine learning evaluation paper with train/test splits. The benchmarks are programs to generate tests for, not a test set for a trained model."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-benchmark results shown in scatter plots (Fig. 3). Case studies analyze top-20 improvement and all 10 decrease benchmarks individually. Table I provides per-project characteristics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D2 analyzes all 10 benchmarks where CODAMOSA had lower coverage than MOSA, identifying causes: wrong signature, unparseable constructs, token limitations, and wasted exploration time."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D2 reports coverage decreases on 10 benchmarks. Section V-C reports that test-case prompting was less consistent (49 improvements vs 24 decreases). Uninterpreted statements sometimes cluttered test cases."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '173 and 279' benchmarks with higher coverage vs '10 and 4' with lower coverage are directly supported by Section V-B results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about why CODAMOSA improves coverage are supported by ablation studies (Section V-C) isolating individual components and case studies (Section V-D) analyzing specific mechanisms (special strings, backup callables, uninterpreted statements)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (Threats to Validity, External) explicitly states 'these results may not hold for any arbitrary Python module.' The paper is clear it evaluates Python only and uses Codex specifically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E investigates whether Codex is simply memorizing tests from training data. The Union experiment tests whether CODAMOSA is just combining independent results. Section V-D2 discusses wasted time as an alternative explanation for improvements being offset."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Codex' throughout without specifying a model version (e.g., code-davinci-002). No API version or snapshot date is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B1 describes the exact prompt structure: source code of the module under test, followed by a test function header with specific templates for functions, methods, and constructors. The actual prompt format is given (e.g., '# Unit test for function X\\ndef test_X():')."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A3 reports: maxStallLen=25, max queries to Codex=10, population size=50 (Pynguin default). Temperature=0.8 default, and 200 token completion cutoff (Section V-D2). Search time T=10 minutes, 16 runs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "CODAMOSA does not use agentic scaffolding. It makes single-turn API calls to Codex within a search algorithm — no retry logic, feedback loops, or agent-style scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A2 documents benchmark filtering: 35 projects identified, pipreqs for dependencies, preliminary MOSA runs to remove failures and 100% coverage modules, down-sampling of shared parent modules, resulting in 486 modules from 27 projects."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' is a dedicated section discussing internal, external, and construct validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI provides specific threats: potential deserialization bugs (internal), results may not generalize to arbitrary Python modules (external), and coverage may not correlate with bug-finding ability (construct), citing a specific study [42]."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (External) explicitly states results may not hold for arbitrary Python modules. Section VII discusses that LLMs are 'not the solution to every problem in test case generation.' The evaluation is explicitly scoped to Python with Codex."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section VII states the replication folder contains raw evaluation data and Codex-generated test cases that can replay CODAMOSA runs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A2 describes benchmark collection from Pynguin and BugsInPy sources, with specific project revisions listed in Table I. Section V-A4 describes experimental procedure (16 runs, 10 minutes each)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Benchmarks are software modules from existing projects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A2 documents the full pipeline: 35 projects → dependency identification → module extraction → preliminary MOSA filtering → down-sampling → 486 benchmarks from 27 projects. Filtering criteria are stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Caroline Lemieux at University of British Columbia, three co-authors at Microsoft Research. The footnote notes 'Most implementation/evaluation work conducted at Microsoft Research.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of four authors are from Microsoft Research, and the paper uses Microsoft's Codex (OpenAI, which Microsoft has invested in heavily). Microsoft has a financial interest in demonstrating the utility of LLMs for software engineering. No discussion of this potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state Codex's training data cutoff date. It mentions Codex was trained on '55 million GitHub repositories' but not when this data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E directly investigates whether Codex generations are copied from training data by comparing to out-of-prompt test files. They also test on flutils (hosted on GitLab, unlikely in Codex's GitHub training set) as a contamination control."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-E addresses this with edit distance analysis showing most generated tests have low similarity to existing tests (99.1% of high-similarity cases are just 'pass'). The flutils GitLab experiment provides evidence CODAMOSA works on code unlikely in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D2 reports: 'On average over all benchmarks, CODAMOSA issues 60 queries to Codex and spends 413 seconds waiting for these queries. This rises to 480 seconds on the worst-performing benchmarks.' Time cost per query is implicitly reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. The paper reports 486 benchmarks x 16 runs x multiple configurations but does not state total API cost, GPU hours, or wall-clock time for the full evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "CODAMOSA achieves statistically significantly higher coverage on 173 benchmarks compared to MOSA, while reducing coverage on only 10.",
       "evidence": "Section V-B, Fig. 3a. Mann-Whitney U-test at p=0.05 across 486 benchmarks with 16 runs each.",
       "supported": "strong"
     },
     {
       "claim": "CODAMOSA achieves significantly higher coverage on 279 benchmarks compared to CODEXONLY, while reducing coverage on only 4.",
       "evidence": "Section V-B, Fig. 3b. Same statistical methodology.",
       "supported": "strong"
     },
     {
       "claim": "CODAMOSA outperforms the union of MOSA and CODEXONLY test suites on 17% of benchmarks despite having half the search time.",
       "evidence": "Section V-B. On 429 replayable benchmarks, CODAMOSA was significantly better on 72, worse on 43, same on 314.",
       "supported": "strong"
     },
     {
       "claim": "Higher temperature (0.8) sampling is the most consistently positive design decision.",
       "evidence": "Section V-C2, Fig. 3d. Temperature 0.8 achieves significantly higher coverage on 113 benchmarks vs 9 for temperature 0.2.",
       "supported": "strong"
     },
     {
       "claim": "Codex-generated tests are mostly not copied from out-of-prompt code in the repositories.",
       "evidence": "Section V-E, Fig. 4. Majority of generated tests have similarity ≤0.4. 99.1% of high-similarity cases are the statement 'pass'.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CODAMOSA combines search-based software testing (MOSA) with LLM-generated test case hints from Codex to escape coverage plateaus. On 486 Python benchmarks with 16 runs each, CODAMOSA achieves significantly higher coverage than MOSA on 173 benchmarks and than a Codex-only baseline on 279 benchmarks, with few regressions (10 and 4 respectively). Ablation studies show higher sampling temperature, uninterpreted statements, and low-coverage targeting all contribute, with temperature having the most consistent effect. Coverage decreases are primarily caused by wasted time on Codex queries that produce unparseable or unhelpful output.",
   "red_flags": [
     {
       "flag": "No variance reporting",
       "detail": "Despite 16 runs per benchmark, no standard deviations or other spread measures are reported for coverage results. Only averages and significance test counts are provided."
     },
     {
       "flag": "Codex model version unspecified",
       "detail": "The paper uses 'Codex' without specifying which model version (e.g., code-davinci-001 vs code-davinci-002), making exact replication impossible as the Codex API has since been deprecated."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex, the LLM used in CODAMOSA for test generation, foundational to LLM-based code generation evaluation."
     },
     {
       "title": "Pynguin: Automated Unit Test Generation for Python",
       "authors": ["Stephan Lukasczyk", "G. Fraser"],
       "year": 2022,
       "arxiv_id": "2202.05218",
       "relevance": "The SBST framework CODAMOSA is built on; key tool for automated Python test generation."
     },
     {
       "title": "Can OpenAI Codex and Other Large Language Models Help Us Fix Security Bugs?",
       "authors": ["H. Pearce", "B. Tan", "B. Ahmad", "R. Karri", "B. Dolan-Gavitt"],
       "year": 2021,
       "arxiv_id": "2112.02125",
       "relevance": "Evaluates LLMs for automated program repair of security bugs, directly relevant to LLM code quality assessment."
     },
     {
       "title": "Automatic Program Repair with OpenAI's Codex: Evaluating QuixBugs",
       "authors": ["J. A. Prenner", "R. Robbes"],
       "year": 2021,
       "arxiv_id": "2111.03922",
       "relevance": "Evaluates LLM-based automated program repair, relevant to understanding LLM capabilities in software engineering."
     },
     {
       "title": "Code Generation Tools (Almost) for Free? A Study of Few-Shot, Pre-Trained Language Models on Code",
       "authors": ["P. Bareiß", "B. Souza", "M. d'Amorim", "M. Pradel"],
       "year": 2022,
       "relevance": "Compares Codex to traditional test generation tools (Randoop) on code coverage, directly comparable work."
     },
     {
       "title": "Jigsaw: Large Language Models Meet Program Synthesis",
       "authors": ["N. Jain", "S. Vaidyanath", "A. Iyer"],
       "year": 2022,
       "relevance": "Uses LLMs for program synthesis with test-case-based post-processing, relevant to LLM+testing integration approaches."
     },
     {
       "title": "TOGA: A Neural Method for Test Oracle Generation",
       "authors": ["E. Dinella", "G. Ryan", "T. Mytkowicz", "S. Lahiri"],
       "year": 2022,
       "relevance": "Neural approach to test oracle generation, complementary to CODAMOSA's test case generation."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["A. Ziegler", "E. Kalliamvakou", "X. A. Li"],
       "year": 2022,
       "relevance": "Assesses productivity impact of neural code completion (Copilot), relevant to evaluating LLM tools in software engineering."
     },
     {
       "title": "A Practical Guide for Using Statistical Tests to Assess Randomized Algorithms in Software Engineering",
       "authors": ["A. Arcuri", "L. Briand"],
       "year": 2011,
       "relevance": "Methodological guidelines followed in CODAMOSA's evaluation; important reference for statistical rigor in SE experiments."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li", "D. Choi", "J. Chung"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Major LLM code generation system, relevant to understanding LLM code capabilities."
     }
   ]
 }