ai-research-survey

scan.json (29876B)

---

 {
   "paper": {
     "title": "Evaluating Large Language Models for Code Review",
     "authors": [
       "Umut Cihan",
       "Arda İçöz",
       "Vahid Haratian",
       "Eray Tüzün"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2505.20206",
     "doi": "10.48550/arXiv.2505.20206"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GPT-4o and Gemini 2.0 Flash achieved moderate accuracy (68.50% and 63.89%) at classifying code correctness on 492 AI-generated code blocks, with GPT-4o correcting 67.83% of incorrect code. Including problem descriptions in prompts consistently improved all metrics. Results differed significantly across datasets (mixed vs ground truth), with Gemini outperforming GPT-4o on the ground truth set, suggesting performance is code-type-dependent. The authors conclude LLMs are unreliable for fully automated code review and propose a human-in-the-loop process.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a Zenodo replication package at https://doi.org/10.5281/zenodo.14962566 (Section VIII, footnote 1), described as containing 'data obtained from this study, as well as the code used.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The datasets used (HumanEval and AI-generated code blocks from Yetistiren et al.) are publicly available, and the paper's experimental data is shared in the Zenodo replication package (Section VIII)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model versions but does not describe the execution environment, library versions, requirements.txt, or any dependency specifications needed to run the experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a replication package is shared on Zenodo, the paper itself does not include step-by-step reproduction instructions or describe how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results (Figures 3-9) report point estimates without confidence intervals or error bars. Standard deviations are mentioned as ranges across all configurations (e.g., '0.35% to 1.61% for correctness accuracy') but are not attached to individual results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "A chi-square test is used only to confirm within-configuration consistency across 3 runs (Section IV). No significance test is applied to between-model or between-condition comparisons, despite claims like 'GPT4o outperformed Gemini.'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both absolute values for compared conditions (e.g., GPT-4o 68.50% vs Gemini 63.89%) and differences (e.g., 'up to 22.87%'), providing sufficient context to assess the magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes of 492 mixed code blocks and 164 ground truth blocks. No power analysis is discussed. The choice of 3 experimental runs is also not justified beyond citing the stochastic nature of LLMs."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports standard deviation ranges across 3 runs per configuration: '0.35% to 1.61% for correctness accuracy, from 1.02% to 1.93% for false positive rates, from 0.65% to 1.07% for false negative rates, from 0% to 2.88% for regression ratios, and from 0.38% to 1.34% for correction ratios' (Section IV)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares two models (GPT-4o and Gemini 2.0 Flash) against each other across multiple configurations, and uses the ground truth dataset as a control group. However, no comparison against prior code review methods or simple heuristic baselines (e.g., random classifier) is included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Both GPT-4o (November 2024) and Gemini 2.0 Flash (December 2024) are contemporary, state-of-the-art models at the time of the study."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically removes problem descriptions from the prompts and measures the impact on all metrics, functioning as an ablation of the contextual input component. Results show consistent degradation without descriptions (Section IV)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five distinct metrics are reported: correctness accuracy, false positive rate, false negative rate, correction ratio, and regression ratio (Section III-C)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via unit test pass/fail. No human evaluation of the LLM's review quality, suggestion usefulness, or output naturalness is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'We optimized our prompt using a chain-of-thought style' but does not describe using a separate validation set for prompt optimization versus a held-out test set for final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (GPT-4o vs Gemini), prompt type (with/without descriptions), dataset (mixed vs ground truth), and error type (false positives vs false negatives), providing detailed per-category views."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses aggregate error rates (false positives, regressions) and YAML/indentation parsing errors (Section VI-A) but does not show qualitative examples of specific failure cases or analyze why particular code blocks were misclassified."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: regression rates up to 24.80%, performance degradation without descriptions, GPT-4o's poor correctness accuracy (42.07%) on the ground truth dataset, and the overall conclusion that LLMs are unreliable for full automation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's specific claims (68.50%, 63.89% correctness accuracy; 67.83%, 54.26% correction ratios) are all supported by the results in Section IV. The qualitative conclusions about moderate reliability are consistent with the reported numbers."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that including problem descriptions improves performance — is supported by a controlled comparison using the same code blocks with and without descriptions, which is an adequate design for this limited causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Evaluating Large Language Models for Code Review' is broad but the study only tests 2 models on Python code blocks from HumanEval. While the threats-to-validity section acknowledges 'our scope is limited to Python' and 'other LLMs may exhibit different behaviors,' the title and framing substantially over-generalize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats-to-validity section discusses specific methodological concerns (prompt sensitivity, Python-only scope, stochastic LLM behavior) but does not discuss alternative explanations for the observed results, such as why GPT-4o outperforms Gemini on mixed data but not ground truth, or what factors drive the performance patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines code correctness precisely via unit tests (Section III) and explicitly acknowledges the proxy gap in conclusion validity (Section VI-D): 'unit testing is not always conducted,' 'practitioners may apply different criteria for code approval,' and 'unit tests might not have enough coverage.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is versioned as 'gpt-4o-2024-11-20' (Section IV), but Gemini is specified only as 'Gemini-2.0-Flash' without a snapshot date or API version. The schema requires exact versions for all models; the Gemini specification is a marketing/API name without a pinned version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt template is provided in Figure 2, showing the actual text used including instructions, output format requirements, and rules. The placeholder values (#code block, #problem description) come from publicly available datasets, allowing full reconstruction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states only 'default model parameters' (Section IV) without specifying what those defaults are (temperature, top-p, max tokens, etc.). These settings significantly affect LLM output and should be explicitly stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The setup sends single prompts to LLM APIs and parses the YAML response — a simple prompt-response pattern without tools, retry logic, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the data pipeline: datasets sourced from HumanEval and Yetistiren et al., code blocks categorized as correct/incorrect via unit tests (234 correct, 258 incorrect), YAML responses parsed, and error handling documented (94.70% clean execution, 4.08% indentation errors, 1.08% YAML format errors — Section VI-A)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides a detailed discussion with four subsections: internal, external, construct, and conclusion validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: prompt sensitivity and YAML extraction errors (internal), Python-only scope and 3-run averaging (external), HumanEval simplicity and AI-generated dataset limitations (construct), unit test coverage limitations (conclusion)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'our scope is limited to Python. Therefore our findings are only directly generalizable to Python' (Section VI-B), 'other LLMs may exhibit different behaviors' (Section VI-C), and that the HumanEval dataset 'consists of simple questions' not representative of real projects (Section VI-C)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replication package (https://doi.org/10.5281/zenodo.14962566) contains the data and code from the study (Section VIII)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The datasets are clearly described: 164 canonical HumanEval solutions, and 492 AI-generated code blocks from three tools (ChatGPT Jan '23, Amazon CodeWhisperer Jan '23, GitHub Copilot v1.70.8099) as collected by Yetistiren et al. (Section III-A)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (HumanEval) and a prior published dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is described in Figure 1 and Section III-B: code blocks are sent to LLMs via prompt, YAML responses are parsed, code suggestions are extracted, unit tests are run, and results are classified. Error rates at the parsing stage are documented (Section VI-A)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with Bilkent University, Ankara, Turkey. They have no apparent affiliation with OpenAI or Google, the providers of the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding statement makes this unevaluable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosures appear in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for GPT-4o and Gemini 2.0 Flash are not stated anywhere in the paper, despite using the HumanEval benchmark which was published in 2021."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o or Gemini 2.0 Flash may have seen HumanEval solutions or the AI-generated code blocks during training. This is a significant omission given that HumanEval is widely used and likely in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 (Chen et al.) and is one of the most widely reproduced benchmarks. Both GPT-4o and Gemini 2.0 Flash were trained well after 2021 and almost certainly encountered HumanEval solutions. The paper does not discuss this contamination risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using unit tests on code blocks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on code blocks with automated unit tests."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of API costs, tokens consumed, or inference latency for either GPT-4o or Gemini 2.0 Flash, despite running 656 code blocks × 2 models × 2 conditions × 3 runs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware information is provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each configuration was run 3 times with results averaged, and standard deviations are reported (ranging from 0.35% to 2.88% across metrics). The chi-square test confirms these variations are not statistically significant (Section IV)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "'To ensure reliability, we ran each experiment configuration three times and reported the average results' (Section IV)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper states 'default model parameters' were used without documenting what those parameters are or whether alternatives were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 8 experimental configurations (2 models × 2 prompt types × 2 datasets) are reported in full. The paper does not cherry-pick only the best configuration but presents all results transparently."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 8 configurations and 5 metrics without applying any correction for multiple comparisons (Bonferroni, Holm, etc.). The chi-square test used is for within-configuration consistency, not between-condition comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors evaluate third-party LLMs (GPT-4o, Gemini 2.0 Flash), not their own system. Self-comparison bias does not apply."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o and Gemini 2.0 Flash differ substantially in compute cost and model size, but the paper does not discuss compute budget, API cost, or latency differences when comparing their performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges in threats to validity that 'The HumanEval dataset consists of simple questions' (Section VI-C) but does not substantively discuss whether unit-test-based correctness classification on simple coding problems is a valid proxy for real-world code review capability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Both models receive the same prompt through their respective APIs in a simple prompt-response setup."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021, and the AI-generated code blocks from Yetistiren et al. were published in 2023. Both GPT-4o (2024) and Gemini 2.0 Flash (2024) were trained after these datasets were publicly available. The paper does not discuss this temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "If the models have memorized HumanEval solutions from training, their ability to assess code correctness may be inflated — they could be pattern-matching against known solutions rather than truly reviewing code. This is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the training data of GPT-4o or Gemini includes HumanEval problems or their solutions, which would violate independence between training and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection methods (canary strings, membership inference, n-gram overlap analysis, decontamination) are applied despite using a widely-known benchmark that is almost certainly in the training data."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4o correctly classified code correctness 68.50% of the time with problem descriptions on the mixed dataset, compared to Gemini's 63.89%.",
       "evidence": "Figure 3 and Section IV-A report averaged results across 3 runs for both models with problem descriptions.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o corrected 67.83% of incorrect code blocks with problem descriptions, versus Gemini's 54.26%.",
       "evidence": "Figure 6 and Section IV-A show correction ratios across models and conditions.",
       "supported": "moderate"
     },
     {
       "claim": "Including problem descriptions in prompts consistently improved LLM performance across all metrics.",
       "evidence": "All results in Section IV show higher accuracy, correction ratios, and lower regression ratios when descriptions are included. Differences of up to 22.87% are observed.",
       "supported": "moderate"
     },
     {
       "claim": "Performance differs significantly across datasets: Gemini outperformed GPT-4o on the ground truth dataset (66.67% vs 42.07%) despite performing worse on the mixed dataset.",
       "evidence": "Figure 8 (Section IV-B) shows the reversal in correctness accuracy between datasets.",
       "supported": "moderate"
     },
     {
       "claim": "LLM code reviews are unreliable for full automation, with regression rates up to 24.80% and inaccurate approval decisions up to 44.44%.",
       "evidence": "Section V-C cites regression rates of 23.79% and 44.44% inaccurate decisions from Gemini without problem descriptions.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis on a widely-known benchmark",
       "detail": "HumanEval was published in 2021 and is among the most reproduced LLM benchmarks. Both GPT-4o and Gemini 2.0 Flash were trained years later and almost certainly encountered HumanEval solutions during training. Models that have memorized correct solutions would have an unfair advantage at assessing code correctness. The paper does not discuss this at all."
     },
     {
       "flag": "Only 3 runs per configuration",
       "detail": "Three experimental runs per configuration is minimal for statistical inference. With n=3, standard deviation estimates are unreliable, and the chi-square test for consistency has very low power (df=2)."
     },
     {
       "flag": "No statistical significance tests between conditions",
       "detail": "Claims that 'GPT-4o outperformed Gemini' are based solely on comparing averaged percentages without any formal significance test. The observed differences (e.g., 68.50% vs 63.89%) may not be statistically significant."
     },
     {
       "flag": "Construct validity concerns",
       "detail": "Using unit test pass/fail on simple HumanEval-style problems as a proxy for code review capability is questionable. Real code review involves design, readability, security, maintainability, and context understanding — none of which are captured by this evaluation framework."
     },
     {
       "flag": "Only 2 models tested with broad title",
       "detail": "The title 'Evaluating Large Language Models for Code Review' implies a broad evaluation, but only GPT-4o and Gemini 2.0 Flash are tested. No open-source models, no Claude, no specialized code models are included."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the HumanEval benchmark and Codex model, which form the core dataset and motivation for this study's evaluation of LLM code review capabilities."
     },
     {
       "title": "Evaluating the code quality of ai-assisted code generation tools: An empirical study on github copilot, amazon codewhisperer, and chatgpt",
       "authors": ["Burak Yetistiren", "Isık Özsoy", "Miray Ayerdem", "Eray Tüzün"],
       "year": 2023,
       "relevance": "Provides the 492 AI-generated code blocks (from Copilot, CodeWhisperer, ChatGPT) used as the mixed dataset in this study."
     },
     {
       "title": "Code review automation: Strengths and weaknesses of the state of the art",
       "authors": ["Rosalia Tufano", "Ozren Dabić", "Antonio Mastropaolo", "Matteo Ciniselli", "Gabriele Bavota"],
       "year": 2024,
       "relevance": "Qualitatively evaluates prior code review automation work alongside ChatGPT, finding it competitive for code-to-comment tasks."
     },
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["Rosalia Tufano", "Simone Masiero", "Antonio Mastropaolo", "Luca Pascarella", "Denys Poshyvanyk", "Gabriele Bavota"],
       "year": 2022,
       "arxiv_id": "2201.06850",
       "relevance": "Employs T5 model for code review automation, representing the pre-LLM era of neural code review approaches."
     },
     {
       "title": "AI-powered code review with LLMs: Early results",
       "authors": ["Zeeshan Rasheed", "Malik Abdul Sami", "Muhammad Waseem"],
       "year": 2024,
       "relevance": "Develops LLM agents to automate code review, directly related to the question of LLM reliability in code review."
     },
     {
       "title": "CodeAgent: Autonomous communicative agents for code review",
       "authors": ["Xunzhu Tang", "Kisub Kim", "Yewei Song"],
       "year": 2024,
       "relevance": "Presents an agentic approach to code review automation using communicative LLM agents."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, the predecessor to GPT-4o which is one of the two models evaluated in this study."
     },
     {
       "title": "Automating code review activities by large-scale pre-training",
       "authors": ["Zhiyu Li", "Shuai Lu", "Daya Guo"],
       "year": 2022,
       "relevance": "Explores automation of code review through large-scale pre-training on diverse code datasets."
     },
     {
       "title": "A prompt pattern catalog to enhance prompt engineering with chatgpt",
       "authors": ["Jules White", "Quchen Fu", "Sam Hays"],
       "year": 2023,
       "arxiv_id": "2302.11382",
       "relevance": "Provides the prompt engineering methodology foundation that informs the chain-of-thought prompting approach used in this study."
     },
     {
       "title": "Resolving code review comments with machine learning",
       "authors": ["Alexander Froemmgen", "Jacob Austin", "Peter Choy"],
       "year": 2024,
       "relevance": "Deployed ML tool at Google for automatically resolving code review comments, demonstrating practical industry application of automated code review."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Proposes a human-in-the-loop code review process and provides a replication package for practitioners to benchmark LLMs on their own codebases."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that LLMs have only moderate accuracy (~68%) at code review is mildly informative but unlikely to surprise practitioners who have used these tools."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications discussed beyond the general point that faulty code review could introduce bugs."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no strong claims against specific companies or products, and balanced reporting of results."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Zenodo replication package is available but it is an experimental setup, not a ready-to-use tool or demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4o, Gemini 2.0 Flash) but comes from Bilkent University rather than a major AI lab."
     }
   }
 }