ai-research-survey

scan.json (27099B)

---

 {
   "paper": {
     "title": "CODE-DITING: A Reasoning-Based Metric for Functional Alignment in Code Evaluation",
     "authors": [
       "Guang Yang",
       "Yu Zhou",
       "Xiang Chen",
       "Wei Zheng",
       "Xing Hu",
       "Xin Zhou",
       "David Lo",
       "Taolue Chen"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2505.19502"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/Code-DiTing. Section I states 'experimental data and model weights are released at https://github.com/Code-DiTing.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states model weights and experimental data are released at the GitHub repository. The three evaluation benchmarks (HumanEval-Judge, MBPP-Judge, BigCodeBench-Judge) and the training dataset CODEJUDGE-17K are described as released. Additionally, the underlying benchmarks (HumanEval-plus, MBPP-plus, BigCodeBench) are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using a single RTX 4090 GPU with VLLM for inference and states maximum context length of 8k tokens, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released on GitHub, the paper itself does not include step-by-step reproduction instructions, specific commands to run, or a 'Reproducing Results' section. Reproduction would require consulting the repository."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. All main results in Tables III and IV are point estimates (e.g., '0.883 Acc') with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CODE-DITING 'outperforms' and 'surpasses' various models based solely on comparing numerical values in tables. No statistical significance tests (p-values, t-tests, bootstrap tests, etc.) are performed to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While raw performance numbers are reported (e.g., accuracy, F1, MCC), no formal effect sizes (Cohen's d, relative improvement with explicit baseline context) are provided. The paper states percentage differences in parameter counts ('1% of parameters') but does not contextualize performance differences with effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sizes of the evaluation datasets (640, 1512, 800 samples) and the preference leakage experiments (50 problems per dataset) are not justified. No power analysis or reasoning for why these specific sizes are sufficient is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The majority vote strategy uses T=7 independent inferences, but no variance or standard deviation across these runs is reported. All results are single point estimates. For the main comparisons in Tables III and IV, no variance across runs is provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: GPT-3.5-turbo, GPT-4o, DeepSeek-V3, DeepSeek-R1, Llama3, Qwen2.5, and DeepSeek-R1-distill models across multiple scales (1B/1.5B, 7B/8B, 671B). Four prompting methods (Vanilla, CoT, ICE SCORE, CodeJudge) are also compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary models such as GPT-4o, DeepSeek-R1 (2025), DeepSeek-V3 (2024), and Qwen2.5-Coder (2024), which are recent and competitive at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section V-B) presents ablation studies on three components: data filtering (Figure 2), PiSSA initialization vs. standard LoRA (Figure 3), and inference strategy with varying k values (Figure 4). Each component's contribution is isolated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Accuracy (Acc), F1 Score, and Matthews Correlation Coefficient (MCC). All three are reported for all experiments in Tables III and IV."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes CODE-DITING as a metric for evaluating code quality, which is inherently about replacing or supplementing human judgment. While three authors manually verified some labels in the data labeling phase (Section III-A), there is no human evaluation of CODE-DITING's outputs — e.g., no assessment of whether its reasoning explanations are actually helpful or correct."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The training data (CODEJUDGE-17K) is constructed from KodCode, OpenCoder, and CodeHarmony benchmarks, while evaluation is on separately curated HumanEval-Judge, MBPP-Judge, and BigCodeBench-Judge datasets derived from different source benchmarks. The evaluation sets are distinct from the training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (HumanEval-Judge, MBPP-Judge, BigCodeBench-Judge) in Tables III and IV, and per model scale category (1.5B, 7B, 671B). Ablation studies also show per-dataset breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is presented. The paper does not show examples where CODE-DITING makes incorrect judgments, nor does it analyze patterns in its errors."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports that PiSSA initialization shows 'less pronounced' improvement on BigCodeBench-Judge compared to other datasets (Section V-B). Also, the empirical study (Section III-E) reports that reasoning models degrade with complex prompts, which is a negative result for prompt engineering."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that CODE-DITING 1.5B outperforms models of the same parameter magnitude and achieves performance of 5x larger models, and that CODE-DITING 7B surpasses GPT-4o and DeepSeek-V3. These claims are supported by Table IV showing the relevant comparisons. The claim about robustness to preference leakage is supported by Tables V and VI."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies (RQ2) make causal claims about each component's contribution. These are tested via controlled single-variable manipulation: removing data filtering, replacing PiSSA with standard LoRA initialization, and varying inference passes. This constitutes adequate causal design for component-level claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates only on Python code across three benchmarks but makes broad claims about 'code evaluation' generally. The title says 'Functional Alignment in Code Evaluation' without bounding to Python. The threats to validity section mentions 'future work could explore additional programming paradigms and domain-specific languages' but does not bound the current claims to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why CODE-DITING outperforms baselines. For instance, the improvement could stem from the training data being derived from the same distribution as the test data (both using similar benchmarks), or from the majority vote strategy inflating performance at 7x computational cost. These alternatives are not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-3.5-turbo' and 'GPT-4o' without specifying API versions or snapshot dates (e.g., 'gpt-4o-2024-05-13'). DeepSeek models are specified by family and size (e.g., 'DeepSeek-R1-671B') but without version identifiers. Marketing names without snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The four prompting methods (Vanilla, CoT, ICE SCORE, CodeJudge) are described in natural language in Section III-B, but the actual prompt text used is not provided in the paper. The paper states 'The prompt can be found in our GitHub repository' (footnote 2) but does not include them in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D reports temperature settings (0.6 for reasoning models, 0.0 for general models), maximum context length (8k tokens), and the inference strategy (T=7 majority votes). The training uses PiSSA with LoRA, and the base models are specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. CODE-DITING is a fine-tuned model that performs single-pass inference (with majority voting), not an agent with tools, retry logic, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents data sampling (which models generate code, deduplication, comment removal), labeling (automatic via test cases + manual verification by three authors), and Section IV-A documents the training data pipeline (source benchmark collection, code generation, reasoning distillation, multi-stage filtering, class balancing to 1:1 ratio)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides a dedicated discussion of internal, external, and construct validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats to validity section discusses specific issues: potential implementation fidelity concerns for baseline methods, the limitation to three benchmarks that do not cover all programming paradigms, hardware constraints limiting model scale tested (single RTX 4090), and the specific metrics chosen. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the threats to validity section mentions that 'future work could explore additional programming paradigms and domain-specific languages,' it does not explicitly state what the current results do NOT show. There is no explicit bounding of claims to Python, to specific benchmark types, or to the specific code generation models tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that experimental data and model weights are released at https://github.com/Code-DiTing. The evaluation datasets (HumanEval-Judge, MBPP-Judge, BigCodeBench-Judge) and training data (CODEJUDGE-17K) are described as publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes how evaluation data was collected: selection of three benchmark datasets, sampling code from four different models (Qwen2.5-Coder 1.5B/7B, DeepSeekCoder 1.3B/6.7B), data cleaning, deduplication, comment removal, and labeling via test cases with manual verification. Section IV-A describes training data collection similarly."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study uses publicly available benchmarks and LLM-generated code, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in detail: Section IV-A describes source benchmark collection, code generation with specific models, labeling via test cases, static analysis filtering, comment removal, reasoning knowledge distillation using DeepSeek-R1-671B, accuracy filtering, logical coherence filtering using DeepSeek-V3, and class balancing to produce the final 17,000 samples."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Nanjing University of Aeronautics and Astronautics, Nantong University, Northwestern Polytechnical University, Zhejiang University, Singapore Management University, and Birkbeck University of London. No apparent conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of a funding disclosure means this criterion cannot be confirmed as satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple pre-trained models (GPT-4o, DeepSeek-R1, etc.) on benchmarks derived from HumanEval, MBPP, and BigCodeBench but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP are widely known benchmarks that are likely in the training data of the evaluated models (GPT-4o, DeepSeek series, etc.). The paper does not discuss whether the evaluated models may have been trained on these benchmarks. The preference leakage experiment (RQ3) addresses a different kind of contamination (same-family bias) but not benchmark contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and MBPP earlier. All evaluated models were trained after these benchmarks were publicly available. The paper does not discuss this contamination risk. While they use 'plus' versions with additional test cases, the original problems and solutions may still be in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The manual verification by three authors for data labeling does not constitute a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B reports inference latency: 0.15s per pass for the 1.5B model and 0.30s per pass for the 7B model on a single RTX 4090, scaling to approximately 1s and 2s respectively at k=7. This provides practical cost information."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While inference hardware is specified (single RTX 4090), the total training compute budget is not stated. No information is provided about training time, GPU hours for fine-tuning, or total API costs for the distillation process using DeepSeek-R1-671B on 17,000+ samples."
       }
     }
   },
   "claims": [
     {
       "claim": "CODE-DITING 7B surpasses GPT-4o and DeepSeek-V3 (671B) despite using only 1% of their parameter volume.",
       "evidence": "Table IV shows CODE-DITING 7B achieving avg Acc 0.806, F1 0.782, MCC 0.565 vs GPT-4o CoT avg Acc 0.795, F1 0.762, MCC 0.497 and DS-V3 CodeJudge avg Acc 0.781, F1 0.745, MCC 0.498.",
       "supported": "moderate"
     },
     {
       "claim": "CODE-DITING 1.5B outperforms all models of the same parameter magnitude and achieves performance equivalent to models with 5x parameters.",
       "evidence": "Table IV shows CODE-DITING 1.5B avg Acc 0.767, F1 0.736, MCC 0.476 vs DS-R1-distill 1.5B avg Acc 0.652, F1 0.604, MCC 0.241, and comparable to DS-R1-distill 7B avg Acc 0.737, F1 0.710, MCC 0.443.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning models demonstrate superior and more stable performance across different datasets compared to general models at comparable parameter scales.",
       "evidence": "Table III shows DeepSeek-R1 series consistently outperforming general models at each scale (Finding 3, Section III-E). E.g., DS-R1-distill 7B Vanilla avg Acc 0.737 vs Qwen2.5 7B best avg Acc 0.708.",
       "supported": "strong"
     },
     {
       "claim": "For reasoning models, simple prompts (Vanilla) work better than complex prompting strategies.",
       "evidence": "Table III shows DS-R1-distill 7B achieves best performance with Vanilla (avg Acc 0.737) vs CoT (0.687), ICE SCORE (0.712), CodeJudge (0.703). Finding 2, Section III-E.",
       "supported": "strong"
     },
     {
       "claim": "CODE-DITING does not suffer from significant preference leakage.",
       "evidence": "Tables V and VI show agreement rates of 93-98% and Cohen's Kappa 0.86-0.96 across code from GPT-4o and Claude-3.5 generators, and across paraphrased problem descriptions. Section V-C.",
       "supported": "moderate"
     },
     {
       "claim": "Each component of CODE-DITING (data filtering, PiSSA, majority vote) contributes significantly to overall performance.",
       "evidence": "Figures 2, 3, and 4 show ablation results. Data filtering improves F1 consistently; PiSSA improves on HumanEval-Judge and MBPP-Judge but less on BigCodeBench-Judge; majority voting improves with increasing k up to k=7. Section V-B.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CODE-DITING distills reasoning capabilities from DeepSeek-R1-671B into compact 1.5B and 7B models for LLM-as-judge code evaluation, using a curated 17K training dataset with reasoning paths. The empirical study finds that reasoning models outperform general models for code evaluation and prefer simpler prompts. CODE-DITING 7B achieves average accuracy 0.806, F1 0.782, MCC 0.565 across three benchmarks, outperforming GPT-4o and DeepSeek-V3 with only 1% of their parameters. Preference leakage experiments show high consistency (93-98% agreement) across different code generators and paraphrased problem descriptions.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('outperforms', 'surpasses') are based on comparing point estimates without any significance tests, confidence intervals, or variance measures. The margins between CODE-DITING 7B and GPT-4o are relatively small (e.g., 0.806 vs 0.795 avg accuracy) and could be within noise."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "HumanEval and MBPP are widely-used benchmarks published in 2021 and earlier. All evaluated LLMs were trained after these benchmarks were public. The paper does not discuss whether evaluated models (including CODE-DITING's base models) may have seen these problems during pre-training, which would inflate reported performance."
     },
     {
       "flag": "Small preference leakage sample size",
       "detail": "The preference leakage experiments (RQ3) use only 50 problems per dataset, which is a small sample for drawing conclusions about bias. No power analysis or confidence intervals are provided for the agreement rates."
     },
     {
       "flag": "Training data distribution overlap with test data",
       "detail": "Both training data (CODEJUDGE-17K from KodCode, OpenCoder, CodeHarmony) and evaluation benchmarks (HumanEval-Judge, MBPP-Judge, BigCodeBench-Judge) involve code generated by similar models (Qwen2.5-Coder, DeepSeekCoder) for algorithmic programming tasks. The paper does not discuss whether this distributional similarity could inflate results."
     },
     {
       "flag": "Unfair comparison via majority voting",
       "detail": "CODE-DITING uses 7 inference passes with majority voting while baselines use single inference. This 7x computational increase is not accounted for in the main comparison (Table IV). A fairer comparison would show baseline performance with the same majority voting strategy."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced the HumanEval benchmark and Pass@k metric, foundational for code generation evaluation."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["D. Guo", "D. Yang", "H. Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Source of the reasoning model used for knowledge distillation in CODE-DITING and the strongest baseline."
     },
     {
       "title": "Is your code generated by chatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "relevance": "Created HumanEval-plus and MBPP-plus with expanded test suites, used as basis for the evaluation benchmarks."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["T. Y. Zhuo", "M. C. Vu", "J. Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Provides the BigCodeBench benchmark used for evaluation, testing real-world software development scenarios."
     },
     {
       "title": "ICE-Score: Instructing large language models to evaluate code",
       "authors": ["T. Y. Zhuo"],
       "year": 2024,
       "relevance": "Pioneered LLM-as-judge for code evaluation using GPT-3.5, serving as both baseline method and prompting strategy."
     },
     {
       "title": "CodeJudge: Evaluating code generation with large language models",
       "authors": ["W. Tong", "T. Zhang"],
       "year": 2024,
       "relevance": "Two-phase LLM-as-judge method for code evaluation, used as a baseline prompting strategy."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["L. Zheng", "W.-L. Chiang", "Y. Sheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-judge methodology, establishing evaluation frameworks used in this space."
     },
     {
       "title": "Preference leakage: A contamination problem in LLM-as-a-Judge",
       "authors": ["D. Li", "R. Sun", "Y. Huang"],
       "year": 2025,
       "arxiv_id": "2502.01534",
       "relevance": "Identifies the preference leakage problem in LLM-as-judge frameworks that CODE-DITING specifically tests against."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique used as one of the baseline methods in the empirical study."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs in software engineering, providing context for the code generation evaluation landscape."
     },
     {
       "title": "From code to courtroom: LLMs as the new software judges",
       "authors": ["J. He", "J. Shi", "T. Y. Zhuo"],
       "year": 2025,
       "arxiv_id": "2503.02246",
       "relevance": "Recent work on LLM-based code evaluation methods, directly relevant to the LLM-as-judge evaluation paradigm."
     },
     {
       "title": "PiSSA: Principal singular values and singular vectors adaptation of large language models",
       "authors": ["F. Meng", "Z. Wang", "M. Zhang"],
       "year": 2024,
       "relevance": "Parameter-efficient fine-tuning technique used as a core component of CODE-DITING's training pipeline."
     }
   ]
 }