ai-research-survey

scan.json (29009B)

---

 {
   "paper": {
     "title": "Retrieval-Augmented Code Review Comment Generation",
     "authors": [
       "Hyunsun Hong",
       "Jongmoon Baik"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2506.11591",
     "doi": "10.48550/arXiv.2506.11591"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "RAG-Reviewer, a retrieval-augmented framework for code review comment generation, outperforms generation-only baselines by up to +1.67% EM and +4.25% BLEU on the Tufano et al. Java benchmark. Pair retrieval (code + comment) consistently outperforms singleton retrieval (comment only) despite fitting fewer exemplars within the token budget. The approach improves generation of low-frequency ground-truth tokens by up to 24.01%, and performance scales with the number of retrieved exemplars, though with diminishing returns due to input length constraints.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'To support reproducibility and encourage future research, we publicly release our implementation on GitHub: https://github.com/RAG-Reviewer/RAG-Reviewer' (Section I, contribution 3)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available Tufano et al. dataset (Section IV.B), and references all baseline replication packages with specific URLs (references [36]–[42])."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV.E mentions 'NVIDIA Tesla V100-SXM2-32GB GPU' and training hyperparameters, but provides no requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released on GitHub, the paper itself contains no step-by-step reproduction instructions, README-level commands, or scripts for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables III, IV, and Figures 6–7 are reported as point estimates (e.g., '2.90% EM', '12.98% BLEU') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., 'RAG-Reviewer outperforms') based solely on comparing raw numbers. No statistical significance tests (t-tests, bootstrap, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides percentage improvements with baseline context throughout: '+1.67% EM' (from 0.87% to 2.54%), '+4.25% BLEU' (from 9.27% to 13.52%), and relative LFGT improvements (e.g., '24.01%' in Table IV)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the dataset size (16,780 test instances from Tufano et al.) or the 100-sample manual analysis. No power analysis or sample adequacy discussion."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported. Results appear to be from single experimental runs with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven baselines are compared: two IR-based (CommentFinder, UniXCoder-IR) and five generation-based (Tufano T5, CodeT5, CodeT5+, CodeReviewer, AUGER), all presented in Table III."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent work: Kartal et al. 2024, CodeT5+ 2023, AUGER 2022, CodeReviewer 2022. These represent the current state of the art for this task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates the retrieval strategy (singleton vs. pair, Table III), the number of retrieved exemplars (0 to 8, Figure 6), and compares with/without retrieval augmentation (vanilla vs. RAG)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used throughout: BLEU-4 and Exact Match (EM), as stated in Section IV.C."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.B reports manual analysis of 100 review comments classified into four categories (Exact Match, Semantically Equivalent, Alternative Solution, Incorrect) in Table V, following the methodology of Tufano et al. [9]."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "They use the standard train/valid/test split from Tufano et al.: 134,239/16,780/16,780 (Table II). 'The best model checkpoint was selected based on validation performance' (Section IV.E)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by retrieval strategy (Table III), token frequency thresholds (Table IV: ≤20 through ≤100), code/review length intervals (Figure 7), and number of retrieved exemplars (Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 illustrates where generation-based and IR-based methods fail. The manual analysis (Table V) reports 50/100 incorrect outputs for RAG-Reviewer. Section VI.B discusses concrete success and failure examples (Figures 8, 9)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that gains over IR baselines are small (e.g., Pair CodeT5 2.90% vs. CommentFinder 2.80% EM), that RAG-Reviewer had fewer exact matches than CommentFinder in manual analysis (2 vs. 4, Table V), and that CodeT5 showed only modest LFGT improvements (3.67%–6.09%, Table IV)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '+1.67% higher exact match' (Tufano T5: 0.87%→2.54%, Table III), '+4.25% higher BLEU' (CodeReviewer: 9.27%→13.52%, Table III), and '24.01%' LFGT improvement (CodeReviewer ≤100, Table IV) are all supported by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('RAG improves generation') are supported by controlled comparisons: vanilla vs. RAG (same PLM backbone), singleton vs. pair retrieval, and varying k from 0 to 8. These ablation-style experiments isolate the retrieval augmentation component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Retrieval-Augmented Code Review Comment Generation' implies general applicability, but all experiments use a single Java dataset (Tufano et al.). While Section VIII acknowledges the Java-only limitation, the title and framing do not bound to Java or this specific benchmark."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why RAG helps. It does not consider whether improvements stem from simply having longer input context, memorization of training exemplars, or other confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures EM and BLEU and frames these as 'review comment generation quality.' They acknowledge the proxy gap by conducting manual analysis (Section VI.B) noting that 'these metrics does not fully reflect the quality of generated review comments.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model variants are identified: 'CodeT5-base' (12 encoder/12 decoder layers), 'CodeT5p-220m', CodeReviewer, UniXcoder. Exact HuggingFace model URLs are provided in references [37]–[42]."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. All models are fine-tuned encoder-decoder PLMs with structured input sequences (Equations 6–8), not prompt-based."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.E provides detailed hyperparameters: AdamW optimizer (lr=3e-5, weight decay=0.01), gradient accumulation (3 steps), batch size 12, gradient clipping (max norm=1.0), beam size 10, 20 epochs, early stopping after 3 epochs, input/output token limits (512/128)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RAG-Reviewer is a standard retrieval + generation pipeline without agents, tools, or iterative reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.B documents the dataset source (Java open-source projects from GitHub/Gerrit), token length distributions (Figure 5, Table II), tokenization (CodeT5 tokenizer), and token truncation at 512/128. The retrieval database construction is described in Section III.B."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII 'Threats to Validity' provides substantive discussion of both internal validity (reimplementation, hyperparameter choices) and external validity (Java-only evaluation)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: reimplementation of Tufano T5 in PyTorch from TensorFlow 2.6.0 with converted pretrained weights, hyperparameters constrained by GPU memory limits, and evaluation limited to a single Java dataset."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII explicitly states: 'Our evaluation is based solely on the Java dataset from Tufano et al. To improve generalizability, future work should evaluate across more languages and diverse software projects.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Tufano et al. dataset is publicly available, and all model replication packages are referenced with URLs (refs [36]–[42]). The authors also release their implementation on GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.B describes the dataset: 'constructed from large-scale Java open-source projects on GitHub and Gerrit. Each code is function-level granularity written in Java and paired with its corresponding review comment.' Statistics in Table II."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is a standard public benchmark (Tufano et al.)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: code encoding with UniXcoder (Eq. 1), retrieval database construction (Eq. 2), similarity computation (Eq. 4), top-k selection (Eq. 5), input augmentation (Eqs. 6–8), and training loss (Eq. 10). Token frequency distribution analysis provided in Table I."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant disclosures appear anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors clearly list their affiliation as School of Computing, KAIST, Daejeon, Republic of Korea. They are not evaluating a product from their own organization."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder from the outcome cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the pre-trained models used (CodeT5, CodeT5+, CodeReviewer, UniXcoder). The pre-training corpora included GitHub code which may overlap with the benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Tufano et al. test examples could appear in the pre-training data of models like CodeT5 (pre-trained on CodeSearchNet and GitHub repositories)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Tufano et al. benchmark was published in 2022, and models like CodeT5 were pre-trained on overlapping sources (GitHub, CodeSearchNet). No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The manual analysis of 100 samples was performed by the authors, not external human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example computation time is reported for RAG-Reviewer. The paper mentions CommentFinder's 49× speedup over generation baselines (from prior work) but does not measure RAG-Reviewer's own cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The GPU type is mentioned (V100-SXM2-32GB) and training hyperparameters are given, but total GPU hours, training wall-clock time, and total compute budget are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing the reported results is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section VIII notes hyperparameters were 'selected based on prior work and GPU memory limits' but no search budget, number of configurations tried, or search method is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.E states: 'The best model checkpoint was selected based on validation performance' with early stopping applied after 3 epochs of no improvement on the validation set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is moot. The absence of statistical testing is captured by significance_tests."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors reimplemented all baselines themselves ('We reproduced all the baselines,' Section IV.D) and do not acknowledge the well-documented bias of authors' reimplementations systematically underperforming."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RAG-Reviewer adds retrieval encoding and database lookup on top of generation, using more compute than vanilla generation baselines. This compute overhead is not discussed or compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses EM and BLEU without discussing whether these metrics actually capture review comment quality. The manual analysis (Section VI.B) partially addresses this gap but the paper does not explicitly question construct validity of the automated metrics."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. RAG-Reviewer is a standard retrieval + generation pipeline, not an agentic scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained models' training data includes code-review pairs from after the Tufano et al. dataset's collection period, or whether benchmark solutions were available during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A states: 'To prevent data leakage, the top-1 retrieved exemplar—which is identical to the input training instance—is excluded during training.' This addresses the primary feature leakage vector in the RAG evaluation setup."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples share structural similarities such as originating from the same projects, same authors, or containing near-duplicate code patterns."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "The top-1 exclusion during training is a design choice to avoid trivial copying, not a leakage detection method in the sense of canary strings, membership inference, or decontamination pipelines. No systematic leakage detection is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "RAG-Reviewer (Pair) outperforms generation-based baselines by up to +1.67% EM and +4.25% BLEU.",
       "evidence": "Table III shows Tufano T5 improves from 0.87% to 2.54% EM (+1.67%), and CodeReviewer improves from 9.27% to 13.52% BLEU (+4.25%). Consistent gains across all five PLMs.",
       "supported": "strong"
     },
     {
       "claim": "Pair retrieval consistently outperforms singleton retrieval across all PLMs.",
       "evidence": "Table III shows pair retrieval achieves higher EM (+0.14% to +0.53%) and BLEU (+0.53% to +1.76%) than singleton retrieval for all five models tested.",
       "supported": "strong"
     },
     {
       "claim": "RAG-Reviewer improves generation of low-frequency ground-truth tokens by up to 24.01%.",
       "evidence": "Table IV shows relative improvements across all models, with CodeReviewer achieving 24.01% at the ≤100 threshold and Tufano T5 consistently showing 20–23% gains.",
       "supported": "moderate"
     },
     {
       "claim": "Performance improves as the number of retrieved exemplars increases.",
       "evidence": "Figure 6 shows EM improving from 1.39% (k=0) to 2.90% (k=8) for Pair CodeT5, with diminishing marginal gains at higher k values.",
       "supported": "moderate"
     },
     {
       "claim": "RAG-Reviewer produces more semantically equivalent comments and fewer incorrect outputs than baselines in manual analysis.",
       "evidence": "Table V shows RAG-Reviewer achieves 39 semantically equivalent vs. 30 for CommentFinder and 36 for Tufano T5, with 50 incorrect vs. 57 and 54 respectively. Sample size is only 100.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical testing on any comparison",
       "detail": "All claims of improvement are based on comparing single-run point estimates with no significance tests, confidence intervals, or variance measures. With margins as small as 0.10–0.21% EM between methods, these differences may not be statistically significant."
     },
     {
       "flag": "Single benchmark evaluation",
       "detail": "All experiments use only the Tufano et al. Java dataset. Generalizability to other languages, review styles, or project types is untested despite general framing in the title."
     },
     {
       "flag": "Marginal improvement over IR baselines",
       "detail": "RAG-Reviewer's best EM (3.01%) is only marginally better than CommentFinder (2.80%) and UniXCoder-IR (2.79%). Without statistical tests, it is unclear whether these gains are meaningful or within noise."
     },
     {
       "flag": "Self-reimplementation of all baselines",
       "detail": "The authors reimplemented all baselines ('We reproduced all the baselines') without acknowledging or mitigating the well-documented bias that authors' reimplementations of competitors tend to systematically underperform."
     },
     {
       "flag": "Small manual analysis without inter-annotator agreement",
       "detail": "The manual analysis uses only 100 samples with no mention of who performed the evaluation, how many annotators were involved, or inter-annotator agreement. Author self-evaluation of their own system's output is prone to bias."
     }
   ],
   "cited_papers": [
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["R. Tufano", "S. Masiero", "A. Mastropaolo", "L. Pascarella", "D. Poshyvanyk", "G. Bavota"],
       "year": 2022,
       "relevance": "Provides the primary benchmark dataset and baseline (Tufano T5) for code review comment generation evaluation."
     },
     {
       "title": "CodeReviewer: Pre-training for automating code review activities",
       "authors": ["Z. Li", "S. Lu", "D. Guo", "N. Duan"],
       "year": 2022,
       "arxiv_id": "2203.09095",
       "relevance": "Proposes code review-specific pre-training and a benchmark dataset that is foundational for evaluating RCG methods."
     },
     {
       "title": "CommentFinder: a simpler, faster, more accurate code review comments recommendation",
       "authors": ["Y. Hong", "C. Tantithamthavorn", "P. Thongtanunam", "A. Aleti"],
       "year": 2022,
       "relevance": "First IR-based code review comment generation approach, reporting 32% improvement over generation-based methods with 49× speedup."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Y. Wang", "W. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "Pre-trained code model used as a backbone generator in RAG-Reviewer and found to be a strong baseline for code review tasks."
     },
     {
       "title": "CodeT5+: Open code large language models for code understanding and generation",
       "authors": ["Y. Wang", "H. Le", "A. D. Gotmare", "N. D. Bui", "J. Li", "S. C. Hoi"],
       "year": 2023,
       "arxiv_id": "2305.07922",
       "relevance": "Extended code PLM with mixture of pre-training objectives, used as a generator backbone achieving the best overall RAG-Reviewer performance."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper that this work builds upon to integrate retrieval with generation for code review."
     },
     {
       "title": "Retrieval augmented code generation and summarization",
       "authors": ["M. R. Parvez", "W. U. Ahmad", "S. Chakraborty", "B. Ray", "K.-W. Chang"],
       "year": 2021,
       "arxiv_id": "2108.11601",
       "relevance": "Applies RAG to code summarization and generation with PLMs; provides the pair/singleton retrieval strategy methodology adopted by this work."
     },
     {
       "title": "UniXcoder: Unified cross-modal pre-training for code representation",
       "authors": ["D. Guo", "S. Lu", "N. Duan", "Y. Wang", "M. Zhou", "J. Yin"],
       "year": 2022,
       "arxiv_id": "2203.03850",
       "relevance": "Code encoder used for the retrieval module in RAG-Reviewer; shown to achieve state-of-the-art IR-based RCG performance."
     },
     {
       "title": "Generation-based code review automation: How far are we?",
       "authors": ["X. Zhou", "K. Kim", "B. Xu", "D. Han", "J. He", "D. Lo"],
       "year": 2023,
       "relevance": "Comparative study of PLMs for code review that found CodeT5 consistently outperformed other code-specific models."
     },
     {
       "title": "Llama-Reviewer: Advancing code review automation with large language models through parameter-efficient fine-tuning",
       "authors": ["J. Lu", "L. Yu", "X. Li", "L. Yang", "C. Zuo"],
       "year": 2023,
       "relevance": "Applies LLM fine-tuning (LLaMA with PEFT) to code review automation, providing evidence on LLM vs PLM effectiveness for RCG."
     },
     {
       "title": "AUGER: Automatically generating review comments with pre-training models",
       "authors": ["L. Li", "L. Yang", "H. Jiang"],
       "year": 2022,
       "relevance": "T5-based review comment generator with review-line tagging, used as a baseline in this work."
     },
     {
       "title": "InferFix: End-to-end program repair with LLMs",
       "authors": ["M. Jin", "S. Shahriar", "M. Tufano"],
       "year": 2023,
       "relevance": "Applies RAG to LLM-based program repair by augmenting prompts with retrieved bug-fix pairs, demonstrating RAG effectiveness in SE tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Code review automation is practically relevant, but this is a research framework evaluated on academic benchmarks, not a deployable tool."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "RAG improving generation quality is an expected and well-established finding; the paper confirms rather than challenges conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications are raised by automated code review comment generation."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward application of RAG to a new task."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is publicly released on GitHub (https://github.com/RAG-Reviewer/RAG-Reviewer), enabling reproduction though not a simple demo."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "KAIST is a respected research university but not a brand that drives mainstream attention."
     }
   }
 }