ai-research-survey

scan.json (27487B)

---

 {
   "paper": {
     "title": "CodeScore: Evaluating Code Generation by Learning Code Execution",
     "authors": [
       "Yihong Dong",
       "Jiazheng Ding",
       "Xue Jiang",
       "Ge Li",
       "Zhuo Li",
       "Zhi Jin"
     ],
     "year": 2023,
     "venue": "ACM (Conference acronym placeholder in paper)",
     "arxiv_id": "2301.09043"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link for the code (https://github.com/Dingjz/CodeScore) and a HuggingFace link for the model (https://huggingface.co/dz1/CodeScore), both listed in footnotes 2 and 3 on page 3."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link for the constructed code evaluation datasets (https://github.com/YihongDong/CodeGenEvaluation) in footnote 4 on page 3. The datasets (APPS-Eval, MBPP-Eval, HE-Eval) are built from public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training on 'a single GPU of Tesla A100-PCIe-40G' and using 'UniXcoder' as the base model with Adam optimizer, learning rate 0.001, epoch 5, but does not provide a requirements.txt, Dockerfile, or detailed environment/dependency specification listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper itself does not include step-by-step reproduction instructions. The implementation details in Section 4.1.4 describe hyperparameters and architecture but do not provide specific commands or a reproducibility guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results tables (Tables 3, 4, 5) report only point estimates for correlation coefficients (Kendall-Tau, Spearman, Pearson) and MAE without confidence intervals or error bars. The human evaluation in Table 6 reports ± values (e.g., 3.4 ± 0.3), but the main experimental results lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports p-values for the human evaluation: 'All p-values are substantially less than 0.005' (Section 4.2.4, page 15). However, significance tests are not reported for the main correlation comparison results."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context, e.g., 'CodeScore achieved absolute improvements of 40.56%, 55.07%, and 58.87% on τ, rs, and rp, respectively' (Section 4.2.1). Correlation coefficients are reported for all methods allowing magnitude comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the dataset sizes or the choice of 100 samples for human evaluation. The paper notes 'Considering the workload of the evaluators, we choose a moderate sample size of 100' (footnote 10), which is a practical constraint, not a statistical justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we exhibit the average performance of UniCE running five times' (Section 4.1.4) and 'we also run other LLM-based metrics five times with their public source code and provide the average performance' (Section 5). The human evaluation reports standard deviations (e.g., '3.4 ± 0.3'). However, the main tables do not show per-run variance for the 5 runs, so this is borderline. The ± notation in Table 6 does count."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines across three categories: match-based CEMs (BLEU, Accuracy, CodeBLEU, CrystalBLEU), LLM-based EMs (BERTScore, COMET, CodeBERTScore), and execution-based CEMs (AvgPassRatio). These are detailed in Section 4.1.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include CodeBERTScore (2023, concurrent work), CodeBLEU (2020), BERTScore (2020), and CrystalBLEU (2022). Given the paper's 2023 submission date, these are reasonably contemporary. The paper explicitly notes CodeBERTScore as 'a concurrent work.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 evaluates the Exec component independently. RQ3 is an explicit ablation comparing the unified loss L_Uni against individual losses (L_Ref, L_NL, L_Ref+NL) across all three input formats (Tables 3 and 4). Table 5 ablates different base models (UniXcoder, CodeBert, CodeGraphBert)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses four evaluation metrics: Kendall-Tau (τ), Spearman R (rs), Pearson R (rp), and Mean Absolute Error (MAE), all defined in Section 4.1.3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 4.2.4) conducts a human evaluation with 10 computer science PhD students evaluating 100 randomly selected samples from HE-Eval, rating the reasonableness of different metrics' scores on a 0-5 scale."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The datasets have explicit train/dev/test splits (Table 1). Results are reported on test sets. Additionally, HE-Eval is used in a zero-shot setting (trained on APPS-Eval, tested on HE-Eval) providing cross-dataset evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (APPS-Eval, MBPP-Eval, HE-Eval), by input format (Ref-only, NL-only, Ref&NL), by base model (Table 5), and the Exec results are broken down by dataset with Accuracy/F1/Precision (Figure 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ5 (Section 4.2.5) provides case studies showing where previous metrics fail and how CodeScore handles those cases. The Discussion section (Section 6) identifies limitations including restriction to function-level Python code."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that HE-Eval performance is 'the poorest' for Exec due to zero-shot setting (Section 4.2.2). Table 5 shows CodeGraphBert outperforms UniXcoder on some metrics. The paper also notes CodeScore still has 'certain limitations' in the Discussion."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 58.87% correlation improvement' which corresponds to the Pearson R improvement on APPS-Eval in Table 3 (Ref&NL format). The 'state-of-the-art' claim is supported by Tables 3 and 4 showing CodeScore outperforming all non-execution baselines. The three input format claim is demonstrated throughout."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (RQ3): removing L_Uni and using individual losses degrades performance, supporting the claim that the unified training contributes to improvement. The ablation design (controlled single-variable manipulation of loss functions and base models) is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and abstract claim to evaluate 'code generation' broadly, but the Discussion section acknowledges 'CodeScore is more suitable for evaluating function-level code in Python.' The title does not bound this — it says 'Evaluating Code Generation' without qualification. The experiments are exclusively on Python function-level code."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Threats to Validity section discusses hyperparameter sensitivity and dataset-specific generalizability but does not consider alternative explanations for why CodeScore outperforms baselines. For example, it doesn't discuss whether the improvement comes from learning execution patterns vs. better code representation, or whether dataset construction choices might favor CodeScore."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies UniXcoder as the base model and lists the specific code generation models used to produce generated code: 'CodeGen 350M&16B, InCoder 1B&6B, and CodeX 13B&175B' and 'StarCoder 15.5B, CodeLlama 34B, and GPT-4' (Section 4.1.1). Model sizes are given, though specific versions/snapshots of GPT-4 are not stated."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use prompting in its core methodology. CodeScore is a fine-tuned model that takes structured input (code + reference + NL). The generated codes used in evaluation were produced by various LLMs, but the paper is evaluating the metric, not doing prompting. Appendix A shows a ChatGPT prompt for test case generation, which is a supplementary experiment."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.4 reports: learning rate 0.001, training epochs 5, Adam optimizer, input token length limit 1024, feedforward network architecture (3 linear layers with dimensions 3072, 1024, 2, Tanh activation). The hardware (Tesla A100-PCIe-40G) is also specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. CodeScore is a standard fine-tuned LLM with a feedforward head for regression/classification."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 describes the full dataset construction pipeline: starting from public benchmarks, generating codes with multiple LLMs, building test cases (type inference → input enumeration → reference code execution), labeling with PassRatio. Input formatting is described in Section 4.1.4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' and Section 6 'Discussion' both address limitations. The Discussion section identifies three specific limitations: data collection requirements, restriction to function-level Python, and additional computation cost."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: (1) the quality and representativeness of the three specific datasets used, (2) hyperparameter sensitivity with their mitigation (grid search on validation set), and (3) instability of deep learning models mitigated by 5-run averaging. Section 6 adds Python/function-level restriction."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section explicitly states 'CodeScore is more suitable for evaluating function-level code in Python' and identifies that the approach requires 'collecting a certain amount of data, including sufficient test cases, generated codes, reference codes, and NL descriptions.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The datasets are released via GitHub (https://github.com/YihongDong/CodeGenEvaluation, footnote 4). The base benchmarks (APPS, MBPP, HumanEval) are public. This allows independent verification of the evaluation data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 describes the full data collection procedure: which base datasets were used, how generated codes were obtained (from specific LLMs), how test cases were extended (3-step process: infer input types, enumerate inputs, execute reference code), and how PassRatio labels were computed."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation (RQ4), the paper states 'We invite ten computer science PhD students, each with over three years of experience in Python development' but does not describe how they were recruited (e.g., from which institution, how they were selected, whether this introduces selection bias)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: public benchmarks → code generation by LLMs → test case extension (3-step process) → PassRatio labeling → train/dev/test split. Statistics are provided in Tables 1 and 2 showing the counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research is supported by the National Natural Science Foundation of China under Grant No.62192733, 61832009, 62192731, 62192730, 62072007, the Key Program of Hubei under Grant JD2023008.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with 'Key Laboratory of High Confidence Software Technologies (Peking University), Ministry of Education; School of Computer Science, Peking University, Beijing, China' as stated in the header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funding comes from the National Natural Science Foundation of China and Hubei provincial grants, which are government research funding agencies with no commercial interest in CodeScore's performance outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper fine-tunes UniXcoder on the APPS-Eval training set and evaluates on HE-Eval (based on HumanEval). However, the pre-training data cutoff for UniXcoder is not discussed. More critically, the LLMs used to generate code (CodeGen, InCoder, CodeX, GPT-4) may have seen the benchmark problems during pre-training, and no training cutoffs are stated for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the LLMs used to generate the evaluation code (CodeX, GPT-4, etc.) may have seen the benchmark problems (APPS, MBPP, HumanEval) during their pre-training. If a model memorized solutions rather than generating them, the PassRatio labels could be inflated."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021, APPS in 2021, and MBPP in 2021. Several LLMs used (GPT-4 2023, CodeLlama 2023, StarCoder 2023) were trained after these benchmarks were public. The paper does not address whether the generated code quality (and thus the evaluation data) is affected by benchmark contamination of the generating models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation (RQ4) involving 10 PhD students is not pre-registered. No mention of any pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation study involving 10 PhD students."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that evaluators are 'ten computer science PhD students, each with over three years of experience in Python development' (Section 4.2.4). This provides expertise level, though geographic distribution and gender are not reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No explicit inclusion/exclusion criteria are stated for the evaluators beyond being CS PhD students with 3+ years of Python experience. It is unclear how these 10 were selected from a larger pool."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a between-subjects experimental study. Each evaluator rates code-metric score pairs; there are no treatment/control conditions requiring randomization of participants."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper states evaluators rated anonymously: 'We randomly list the generated code with reference code and NL and the corresponding EM score on the questionnaire. Each group is evaluated anonymously by one evaluator' (Section 4.2.4). The metrics are presented without identifying which metric produced each score."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information about whether all 10 evaluators completed their assigned evaluations or if there was any dropout."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports execution time comparisons: CodeScore is 'three orders of magnitude faster than execution-based CEMs' and provides relative execution times for all methods (e.g., '33.7×' vs BLEU baseline in Table 3). Absolute times are given for BLEU as reference (26.0s, 0.87s, 1.96s for the three datasets)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training on 'a single GPU of Tesla A100-PCIe-40G' with 5 epochs, but does not quantify total GPU hours, training time, or total computational budget for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeScore achieves absolute improvements of up to 58.87% in correlation with functional correctness compared to other CEMs on APPS-Eval.",
       "evidence": "Table 3 shows Pearson R of 0.7210 for CodeScore (Ref&NL, L_Uni) vs. the best prior method at 0.1323 (CodeBERTScore), an absolute improvement of 58.87%. Similar large improvements in Kendall-Tau (40.56%) and Spearman (55.07%).",
       "supported": "strong"
     },
     {
       "claim": "CodeScore generalizes to unseen datasets (MBPP-Eval, HE-Eval) without training on them.",
       "evidence": "Table 4 shows CodeScore trained on APPS-Eval achieves the best correlation among non-execution methods on both MBPP-Eval (fine-tuned) and HE-Eval (zero-shot), with Spearman correlations of 0.6027 and 0.6597 respectively.",
       "supported": "strong"
     },
     {
       "claim": "CodeScore operates three orders of magnitude faster than execution-based CEMs.",
       "evidence": "Table 3 shows execution-based CEMs require 1.5k× to 20.7k× the time of BLEU, while CodeScore requires 33.7× to 44.2×. Section 4.2.1 summarizes the speed advantage.",
       "supported": "strong"
     },
     {
       "claim": "The unified loss L_Uni improves performance across all input formats compared to individual losses.",
       "evidence": "Tables 3 and 4 show L_Uni consistently outperforms L_Ref, L_NL, and L_Ref+NL across all datasets and metrics (RQ3, Section 4.2.3).",
       "supported": "strong"
     },
     {
       "claim": "Human evaluation shows CodeScore improves at least 54.6% over previous representative EMs.",
       "evidence": "Table 6 shows CodeScore scores 3.4 vs. CodeBERTScore at 2.2 (the next best), with p-values substantially less than 0.005. However, only 100 samples were evaluated by 10 evaluators with each evaluator seeing only one group.",
       "supported": "moderate"
     },
     {
       "claim": "The Exec component achieves over 90% accuracy, F1, and precision across all datasets.",
       "evidence": "Figure 4 shows Accuracy 0.94/0.953/0.944, F1 0.969/0.973/0.972, and Precision 0.938/0.948/0.943 for APPS-Eval/MBPP-Eval/HE-Eval respectively.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CodeScore, an LLM-based code evaluation metric fine-tuned on code execution data, substantially outperforms match-based metrics (BLEU, CodeBLEU) and embedding-based metrics (BERTScore, CodeBERTScore) in correlation with functional correctness across three datasets (APPS-Eval, MBPP-Eval, HE-Eval). The unified training framework UniCE handles three input formats (reference-only, NL-only, reference+NL) and operates three orders of magnitude faster than execution-based evaluation while achieving comparable correlation. The approach generalizes to unseen datasets in a zero-shot setting, and human evaluation with 10 PhD students confirms the metric's superior alignment with human judgment.",
   "red_flags": [
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The LLMs used to generate evaluation code (GPT-4, CodeLlama, StarCoder) were trained after HumanEval, APPS, and MBPP were published. If these models memorized benchmark solutions, the generated code distribution and PassRatio labels in the evaluation datasets could be systematically biased. The paper does not discuss this."
     },
     {
       "flag": "Small human evaluation with unclear recruitment",
       "detail": "The human evaluation uses only 10 PhD student evaluators, each seeing only 50 code-metric pairs (100 samples / 10 evaluators × 5 metrics). Recruitment methods are not described, no IRB approval is mentioned, and the sample size justification is explicitly about workload rather than statistical adequacy."
     },
     {
       "flag": "Variance across runs not shown in main tables",
       "detail": "Although the paper states results are averaged over 5 runs, the main results tables (3, 4, 5) report only point estimates without standard deviations across runs, making it impossible to assess result stability for the main claims."
     },
     {
       "flag": "Generalization claim broader than evidence",
       "detail": "The title claims 'Evaluating Code Generation' broadly, but experiments are restricted to function-level Python code from three specific benchmarks. The Discussion section acknowledges this limitation, but the title and abstract do not qualify it."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational for code generation evaluation."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "relevance": "Introduces the APPS benchmark used as one of three core evaluation datasets in this paper."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces the MBPP benchmark, one of the three core datasets used in this work."
     },
     {
       "title": "CodeBERTScore: Evaluating Code Generation with Pretrained Models of Code",
       "authors": ["Shuyan Zhou", "Uri Alon", "Sumit Agarwal", "Graham Neubig"],
       "year": 2023,
       "arxiv_id": "2302.05527",
       "relevance": "Concurrent work proposing a BERT-based code evaluation metric, directly compared as a baseline."
     },
     {
       "title": "CodeBLEU: a Method for Automatic Evaluation of Code Synthesis",
       "authors": ["Shuo Ren", "Daya Guo"],
       "year": 2020,
       "arxiv_id": "2009.10297",
       "relevance": "Proposes CodeBLEU metric combining n-gram, AST, and data-flow matching for code evaluation."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang"],
       "year": 2023,
       "relevance": "Open code generation model used to produce generated code for the evaluation datasets."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Large open code model used as one of the code generators for the HE-Eval dataset."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open code model used for generating code in the HE-Eval evaluation dataset."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yihong Dong", "Xue Jiang"],
       "year": 2024,
       "arxiv_id": "2402.15938",
       "relevance": "Addresses data contamination in LLM evaluation, directly relevant to benchmark contamination concerns."
     },
     {
       "title": "UniXcoder: Unified Cross-Modal Pre-training for Code Representation",
       "authors": ["Daya Guo", "Shuai Lu"],
       "year": 2022,
       "relevance": "The base pre-trained model used in CodeScore's UniCE framework."
     },
     {
       "title": "Fault-Aware Neural Code Rankers",
       "authors": ["Jeevana Priya Inala", "Chenglong Wang"],
       "year": 2022,
       "relevance": "Addresses code ranking by functional correctness, related to the code evaluation problem CodeScore solves."
     },
     {
       "title": "BERTScore: Evaluating Text Generation with BERT",
       "authors": ["Tianyi Zhang", "Varsha Kishore"],
       "year": 2020,
       "relevance": "Foundational embedding-based evaluation metric used as a baseline and architectural inspiration."
     }
   ]
 }