ai-research-survey

scan.json (22352B)

---

 {
   "paper": {
     "title": "From Code to Courtroom: LLMs as the New Software Judges",
     "authors": [
       "Junda He",
       "Jieke Shi",
       "Terry Yue Zhuo",
       "Christoph Treude",
       "Jiamou Sun",
       "Zhenchang Xing",
       "Xiaoning Du",
       "David Lo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2503.02246",
     "doi": "10.48550/arXiv.2503.02246"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code or analysis scripts are released. No repository URL is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The list of 16 reviewed papers is in Table 1, but no structured dataset, extraction forms, or supplementary materials are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tooling specification is provided. A survey could release its analysis environment but this one does not."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions for reproducing the literature review are provided. The search methodology is not documented well enough to replicate."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative literature review with no statistical analysis or experiments."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or statistical comparisons are performed."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative analysis is conducted; the review is entirely qualitative."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or quantitative analysis. The paper reviews literature without statistical aggregation."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with multiple runs. This is a qualitative survey paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare its review against prior surveys or reviews on LLM-as-a-Judge. While it references Li et al. [33] and Wang et al. [65] for definitional contrast, it does not systematically compare its coverage or findings against other reviews."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring baselines. This is a survey/vision paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system components to ablate. This is a literature review."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with metrics. This is a qualitative literature review."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. This is a survey paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 organizes the 16 reviewed studies by SE task (code generation, code summarization, bug report summarization, code translation, question answering, requirements causality extraction, code patches generation). Sections 3.1-3.4 discuss each category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 identifies six specific limitations of current LLM-as-a-Judge approaches: lack of benchmarks, inconsistent findings, unexplored biases, inadequate domain expertise, reliance on internal evaluation only, and insufficient adversarial research."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses conflicting findings in the literature. Limitation 2 (Section 4.1) notes that Wang et al. [65] found traditional metrics outperformed LLM-as-a-Judge for code summarization, while Wu et al. [68] found the opposite."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: the paper does review 16 studies (Section 3, Table 1), analyzes limitations (Section 4 with 6 labeled limitations), and proposes a research roadmap with specific directions."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It describes the state of the field and proposes future research directions without claiming causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title ('LLMs as the New Software Judges') and vision claims are sweeping relative to the evidence base of 16 papers. The abstract hedges with 'While not intended to be a definitive guide' but the roadmap and 2030 vision make broad claims about the entire SE community's trajectory based on a small literature set."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure survey and vision paper that presents no empirical results of its own. Alternative explanations are not applicable."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no measurements of its own. It is a literature review and roadmap with no proxy/outcome gap to address."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used. This is a literature review."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the 16 primary studies were identified. No search queries, databases, date ranges, or inclusion/exclusion criteria are documented. The paper states it reviews '16 primary studies' without explaining the selection methodology."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 discusses six limitations of the LLM-as-a-Judge FIELD, but there is no section discussing limitations of this review itself — its methodology, potential selection bias, or gaps in coverage."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity of the survey methodology are discussed. The paper does not address whether its 16-paper sample is representative, whether its search was comprehensive, or whether the roadmap reflects author bias."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The abstract says 'While not intended to be a definitive guide' but does not state specific scope boundaries — what SE domains are excluded, what paper types were filtered out, or what the review does NOT cover. The formal definition in Section 2 excludes embedding-based methods, which is the closest to a scope boundary."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data or extraction tables are available. The review findings are presented narratively without structured data that could be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the 16 primary studies were found. No search databases, queries, date ranges, or selection criteria are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No pipeline from literature search to final review is documented. The paper jumps from stating it reviews 16 papers to discussing their content, with no description of how the set was assembled."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Singapore Management University, Monash University, CSIRO's Data61, and Australian National University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Absence of disclosure is not evidence of absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark. It is a literature review."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on benchmarks is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted in this paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/vision paper with no computational experiments."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/vision paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, no systematic protocol, no documented search strategy. The paper does not describe how the 16 primary studies were identified or selected. The review appears to be ad-hoc rather than following a structured review protocol."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the methodological quality of its 16 reviewed studies. All studies are treated equally regardless of rigor. No quality scoring rubric or risk-of-bias assessment is applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper does not consider whether its sources are biased toward positive results about LLM-as-a-Judge, nor does it discuss whether negative findings about LLM-as-a-Judge are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-as-a-Judge systems offer a promising solution to address the limitations of both costly human evaluation and traditional automated metrics in SE.",
       "evidence": "Section 1 and Section 2 argue this based on LLMs' coding abilities, human-like reasoning, RLHF alignment, and lack of fatigue. No original empirical evidence is provided; the claim rests on cited studies.",
       "supported": "weak"
     },
     {
       "claim": "The LLM-as-a-Judge field in SE is in its early stages, with only 16 primary studies identified.",
       "evidence": "Table 1 lists 16 studies across 7 SE tasks. Section 3 reviews each. The small number itself is the evidence for the claim.",
       "supported": "moderate"
     },
     {
       "claim": "Existing LLM-as-a-Judge benchmarks rely on small-scale datasets, often only a few hundred samples.",
       "evidence": "Section 4.1 cites Wang et al. [65] using 450 samples across 3 tasks and Ahmed et al. [1] using 420 samples for code summarization.",
       "supported": "moderate"
     },
     {
       "claim": "Empirical findings on LLM-as-a-Judge in SE are inconsistent, with conflicting conclusions across studies.",
       "evidence": "Section 4.1 Limitation 2: Wang et al. [65] found traditional metrics outperformed LLM-as-a-Judge for code summarization, while Wu et al. [68] found the opposite.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-as-a-Judge systems can achieve reliable, robust, and scalable evaluation of software artifacts by 2030.",
       "evidence": "Section 4 presents a roadmap with opportunities but no empirical support for this timeline or feasibility. This is a vision statement, not an evidence-based prediction.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This paper reviews 16 studies on LLM-as-a-Judge for software engineering evaluation and identifies six key limitations: lack of large-scale human-annotated benchmarks, inconsistent empirical findings, unexplored biases, inadequate SE domain expertise in LLMs, reliance on internal evaluation mechanisms only, and insufficient adversarial threat research. The paper proposes a research roadmap toward reliable, robust, and scalable LLM-as-a-Judge systems by 2030, including better benchmarks, domain-specific training, tool integration, human-in-the-loop approaches, and adversarial defenses. The review covers code generation, code summarization, code changes, and other SE tasks but is a qualitative narrative review rather than a systematic review with structured methodology.",
   "red_flags": [
     {
       "flag": "No systematic review methodology",
       "detail": "The paper reviews 16 studies but provides no search methodology — no databases searched, no search queries, no inclusion/exclusion criteria, no PRISMA diagram. It is impossible to determine whether the 16 papers are a representative or comprehensive sample. The review appears ad-hoc."
     },
     {
       "flag": "Sweeping vision claims from thin evidence base",
       "detail": "The paper makes broad claims about what LLM-as-a-Judge can achieve by 2030 based on a review of only 16 papers, without quality assessment of those papers. The roadmap is aspirational rather than evidence-derived."
     },
     {
       "flag": "No quality assessment of reviewed studies",
       "detail": "All 16 reviewed papers are treated equally. No quality scoring, risk-of-bias assessment, or structured evaluation is applied. This is exactly the 'laundering weak results' problem: findings from rigorous and weak papers are given equal weight in shaping the roadmap."
     },
     {
       "flag": "Self-citation pattern",
       "detail": "Multiple authors appear in the reviewed studies (e.g., Terry Yue Zhuo is both a paper author and author of reviewed study [88]; Jieke Shi, Junda He, and David Lo appear in several reviewed papers [55-57, 67, 72-75]). The overlap between reviewers and reviewed is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "ICE-Score: Instructing Large Language Models to Evaluate Code",
       "authors": ["Terry Yue Zhuo"],
       "year": 2023,
       "arxiv_id": "2304.14317",
       "relevance": "Directly uses LLMs (GPT-3.5) to evaluate code generation quality — core to the LLM-as-a-Judge paradigm being surveyed."
     },
     {
       "title": "Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering",
       "authors": ["Ruiqi Wang", "Jiyu Guo", "Cuiyun Gao", "Guodong Fan", "Chun Yong Chong", "Xin Xia"],
       "year": 2025,
       "arxiv_id": "2502.06193",
       "relevance": "Empirical study comparing LLM-as-a-Judge with traditional metrics for SE tasks including code generation, summarization, and translation."
     },
     {
       "title": "Can Large Language Models Serve as Evaluators for Code Summarization?",
       "authors": ["Yang Wu", "Yao Wan", "Zhaoyang Chu", "Wenting Zhao", "Ye Liu", "Hongyu Zhang", "Xuanhua Shi", "Philip S Yu"],
       "year": 2024,
       "arxiv_id": "2412.01333",
       "relevance": "Introduces CODERPE framework using multi-role LLM evaluation for code summaries, directly relevant to LLM-as-a-Judge methodology."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2024,
       "relevance": "Foundational LLM-as-a-Judge paper establishing evaluation methodology with MT-Bench and Chatbot Arena."
     },
     {
       "title": "CodeJudge: Evaluating Code Generation with Large Language Models",
       "authors": ["Weixi Tong", "Tianyi Zhang"],
       "year": 2024,
       "relevance": "Uses LLMs with a taxonomy of programming errors to evaluate generated code, a specialized LLM-as-a-Judge approach."
     },
     {
       "title": "Can LLMs replace manual annotation of software engineering artifacts?",
       "authors": ["Toufique Ahmed", "Premkumar Devanbu", "Christoph Treude", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2408.05534",
       "relevance": "Investigates LLM capability to replace human annotation across multiple SE tasks including code summarization and function similarity."
     },
     {
       "title": "AIME: AI System Optimization via Multiple LLM Evaluators",
       "authors": ["Bhrij Patel", "Souradip Chakraborty", "Wesley A Suttle", "Mengdi Wang", "Amrit Singh Bedi", "Dinesh Manocha"],
       "year": 2024,
       "arxiv_id": "2410.03131",
       "relevance": "Proposes multi-LLM evaluation framework for code quality assessment including correctness, readability, and runtime performance."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Pass@k metric, foundational to code generation evaluation."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Major code generation benchmark with diverse function calls, relevant to the evaluation infrastructure for LLM code capabilities."
     },
     {
       "title": "LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods",
       "authors": ["Haitao Li", "Qian Dong", "Junjie Chen"],
       "year": 2024,
       "arxiv_id": "2412.05579",
       "relevance": "Comprehensive survey on LLM-based evaluation methods that provided the definitional framework adopted in this paper."
     },
     {
       "title": "CodeJudge-Eval: Can Large Language Models be Good Judges in Code Understanding?",
       "authors": ["Yuwei Zhao", "Ziyang Luo", "Yuchen Tian"],
       "year": 2025,
       "relevance": "Empirically evaluates 12 LLMs as code generation judges, providing evidence on which models are effective evaluators."
     },
     {
       "title": "G-Eval: NLG Evaluation Using GPT-4 with Better Human Alignment",
       "authors": ["Yang Liu", "Dan Iter", "Yichong Xu", "Shuohang Wang", "Ruochen Xu", "Chenguang Zhu"],
       "year": 2023,
       "arxiv_id": "2303.16634",
       "relevance": "General-purpose LLM-as-a-Judge method tested in SE contexts, represents cross-pollination from NLP to SE evaluation."
     }
   ]
 }