ai-research-survey

scan.json (30022B)

---

 {
   "paper": {
     "title": "RePair: Automated Program Repair with Process-based Feedback",
     "authors": [
       "Yuze Zhao",
       "Zhenya Huang",
       "Yixiao Ma",
       "Rui Li",
       "Kai Zhang",
       "Hao Jiang",
       "Qi Liu",
       "Linbo Zhu",
       "Yu Su"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics (Findings)",
     "arxiv_id": "2408.11296",
     "doi": "10.18653/v1/2024.findings-acl.973"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RePair demonstrates that process-based feedback (SFT + PPO with a reward model critic) enables a 15B parameter model to achieve competitive performance with commercial LLMs like GPT-3.5 and Claude2 on automated program repair. The ablation study shows both process supervision and reward model feedback are necessary — process supervision without feedback actually degrades performance significantly (pass@1 drops from 36.32 to 20.43). Pairwise ranking outperforms point-wise and list-wise alternatives for reward modeling. Small-scale models (< 20B parameters) struggle to utilize explicit prompt-based feedback from compilers and test cases, motivating the RL approach.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data are publicly available at https://github.com/TnTWoW/RePair' in the abstract footnote."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same GitHub URL claims data availability. The CodeNet4Repair dataset is described as Apache-2.0 licensed (Section 2.3) and the underlying CodeNet dataset is also public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python 3.11.3 for testing, ZeRO++ for distributed training, and AdamW optimizer, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Training details are described but there is no 'Reproducing Results' section or instructions for running the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 and Table 4 report only point estimates for pass@k. No confidence intervals, error bars, or ± notation are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims their model 'outperforms all other open-source models' and achieves 'competitive results' with commercial LLMs based solely on raw number comparisons. No statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw pass@k numbers in tables. The paper does not explicitly report effect sizes, percentage improvements, or Cohen's d. Readers must compute differences themselves from the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set contains 61 problems and 10,144 repair records (Table 2). No justification is given for why this size is adequate, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single training runs with the unbiased pass@k estimator applied to generated samples, but no cross-run variance is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 includes extensive baselines: 4 closed-source models (PaLM, GPT-3.5, Claude2, ChatGLM-Pro) and 6 open-source models (StarCoderBase/Chat, CodeGen2, CodeGeeX2, LLaMA2, LLaMA2-Chat)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-3.5 (gpt-3.5-turbo-0613), Claude2, LLaMA2, and CodeGeeX2, which were all contemporary models at the time of the paper (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents a thorough ablation study examining process supervision, feedback, and reward function design (pairwise, pointwise, listwise ranking) across six different configurations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports pass@1, pass@3, and pass@5, each at three difficulty levels (Easy, Medium, Hard) as well as overall. Per the schema, these count as multiple metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs is performed. Evaluation is entirely automated via pass@k using test case execution."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'To prevent data leakage, we divided CodeNet4Repair based on the problem ID in a ratio of 9:1.' The test set has 61 separate problems from the training set's 563."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down all results by difficulty level (Easy, Medium, Hard) in addition to overall scores."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The qualitative examples in Appendix A.3 and Figure 6 show only successful repairs. No failure cases or error analysis of where the model breaks down are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows that process supervision without feedback severely degrades performance (pass@1 drops from 36.32 to 20.43). Section 4.8 shows small models struggle with explicit prompt feedback. These are genuine negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims process-based feedback 'outperforms larger outcome-based generation methods' (Table 3: 44.34 pass@1 vs LLaMA2-70B's 24.63) and 'nearly matches...closed-source commercial large-scale LMs' (44.34 vs GPT-3.5's 46.39 and Claude2's 43.98). Both claims are supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims process-based feedback improves repair performance. The ablation study in Table 4 uses controlled single-variable manipulation (adding/removing process supervision and feedback independently), which is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitation) explicitly acknowledges scope limits: 'Due to time and resource constraints, we were unable to collect all repair processes across different languages and software engineering domains.' They bound to Python competition-level programs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed performance gains beyond the ablation study. For example, it does not consider whether the improvement comes from additional training data exposure, the specific properties of CodeNet problems, or other confounding factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k (programs passing test cases) and frames results as program repair capability. The measurement closely matches the claim. Section 7 acknowledges the gap between competition and engineering scenarios."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "StarCoderBase 15.5B is specified. Baselines include specific versions: 'gpt-3.5-turbo-0613' (Section 4.3), 'chat-bison-001' for PaLM, and specific model sizes for all open-source models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 (Appendix A.1) provides the full prompt templates used for both standard SFT and explicit feedback experiments, including system prompt, user prompt, and assistant response format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports detailed hyperparameters: learning rate 2e-5 for SFT, 9.6e-6 for RM, 9e-6 for RL, cosine schedule, K=9 programs for RM, batch size 512, KL penalty β=0.02, top_p=0.95, top_k=50, temperature=0.2."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a trained RL policy with iterative generation guided by a reward model — this is a training/inference procedure, not agentic scaffolding with tools, memory, or agent frameworks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 2.1–2.3 document the full pipeline: HTML extraction of problem descriptions, preliminary filtering (removing duplicates, malicious submissions, privacy breaches), fine filtering with Python 3.11.3 re-execution (1,227,259 → 278,408 programs), and organization into procedural format."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitation' and provides substantive discussion of the work's scope constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific threats: limited to Python language, limited to competition scenarios, unable to collect repair processes across different languages and engineering domains. These are specific to this study rather than generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states: 'Due to time and resource constraints, we were unable to collect all repair processes across different languages and software engineering domains.' It explicitly identifies what was not tested (other languages, engineering scenarios)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims code and data are publicly available at https://github.com/TnTWoW/RePair, and the underlying CodeNet dataset is also publicly available (Puri et al., 2021)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 2.1–2.3 describe the data collection procedure in detail: sourcing from CodeNet, HTML extraction of problem descriptions, filtering criteria, Python 3.11.3 re-execution for status verification, and test case collection."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from CodeNet, a public programming competition dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with specific numbers at each stage: 1,227,259 programs after preliminary filtering → 278,408 after fine filtering with Python 3.11.3 re-execution → 94,062 training records and 10,144 test records. Filtering criteria are explained at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Acknowledgement) lists funding: NSFC (No. 62106244), Anhui Provincial Natural Science Foundation (No. 2308085QF229), Fundamental Research Funds for Central Universities (No. WK2150110034), and CIPSC-SMP-Zhipu.AI Large Model Cross-Disciplinary Fund."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: USTC (State Key Laboratory of Cognitive Intelligence), Institute of AI Comprehensive National Science Center, and Hefei Normal University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are government research agencies (NSFC, Anhui Provincial Foundation) and university funds with no financial stake in whether process-based APR outperforms alternatives."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "StarCoderBase is described as trained on The Stack (1 trillion tokens) but no training data cutoff date is stated. The training cutoffs for baseline models (GPT-3.5, Claude2, PaLM) are also not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper splits its own dataset by problem ID to prevent internal leakage (Section 4.1), but does not discuss whether the pre-trained baseline models (GPT-3.5, Claude2, StarCoderBase) may have seen CodeNet solutions in their training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeNet was published in 2021 (Puri et al., 2021) and its solutions are publicly available. Any model trained after 2021 may have encountered these solutions, but this risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported. The paper mentions 1,364 Core·hours for data preparation but does not report inference costs for the model."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training details include 32k episodes with batch size 512 and use of ZeRO++ across accelerators, but the total GPU hours, number of GPUs, or wall-clock training time are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from a single training run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of independent training runs is not stated. The pass@k estimator uses n generated samples per problem, but n is not specified and no multi-run variance is reported."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported (Section 4.2) but no search budget, search method, or number of configurations tried is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation study in Table 4 shows different configurations but does not explain how the final configuration was selected or whether selection was based on validation data rather than test data."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 10+ baselines at 3 difficulty levels, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system against baselines without acknowledging author-evaluation bias. No discussion of whether their baseline implementations are representative."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares a 15B fine-tuned+RL model against 70B and 540B few-shot models without controlling for or discussing compute budget differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether pass@k on competition-level Python problems measures real-world program repair capability. The limitations section actually overclaims: 'there are significant similarities between competition and engineering scenarios' without evidence."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The main comparison (Table 3) compares their model with multi-step iterative inference + reward model feedback against baselines using single-pass generation. This inference-time advantage is a confound. Section 4.5 acknowledges 'some performance gains come from supervised fine-tuning' but does not isolate the multi-step inference advantage."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeNet was published in 2021 with publicly available solutions. Models like GPT-3.5 and StarCoderBase (trained on The Stack) may have encountered these solutions. This temporal leakage risk is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The prompt includes problem descriptions that may provide implicit hints about expected solutions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'To prevent data leakage, we divided CodeNet4Repair based on the problem ID in a ratio of 9:1.' This ensures train and test problems are completely separate."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Problem-ID-based splitting is a concrete prevention method ensuring no overlap between train and test problems within their dataset."
       }
     }
   },
   "claims": [
     {
       "claim": "Process-based feedback with a 15B model outperforms larger outcome-based open-source models including LLaMA2-70B on program repair.",
       "evidence": "Table 3: RePair achieves 44.34 pass@1 vs LLaMA2-70B's 7.59 and LLaMA2-Chat-70B's 24.63 on CodeNet4Repair.",
       "supported": "strong"
     },
     {
       "claim": "RePair nearly matches the performance of closed-source commercial LLMs like GPT-3.5 and Claude2.",
       "evidence": "Table 3: RePair achieves 44.34 pass@1 vs GPT-3.5's 46.39 and Claude2's 43.98. At pass@5, RePair gets 65.66 vs GPT-3.5's 66.67 and Claude2's 69.44.",
       "supported": "strong"
     },
     {
       "claim": "Both process supervision and feedback are necessary — process supervision without feedback significantly degrades performance.",
       "evidence": "Table 4 ablation: process+no feedback gets 20.43 pass@1 vs no process+no feedback (SFT only) at 36.32 pass@1. Process+feedback achieves 44.34.",
       "supported": "strong"
     },
     {
       "claim": "Pairwise ranking is the most effective reward function design compared to pointwise and listwise alternatives.",
       "evidence": "Table 4: Pairwise achieves 44.34 pass@1 vs pointwise 40.11 and listwise 39.45.",
       "supported": "moderate"
     },
     {
       "claim": "Small-scale models (< 20B parameters) struggle to receive effective explicit feedback through prompts.",
       "evidence": "Figure 5 shows StarCoder and CodeGen2 performance does not consistently improve with additional explicit feedback steps, peaking at step 3 then declining, while ChatGPT can effectively use explicit feedback.",
       "supported": "moderate"
     },
     {
       "claim": "SFT alone is not the primary factor for performance improvement — process-based feedback significantly amplifies the model's potential beyond SFT.",
       "evidence": "Figure 4 shows that while SFT improves several open-source models, the full RePair system (SFT+RL) substantially outperforms SFT-only variants across all metrics.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Unfair main comparison",
       "detail": "Table 3 compares their fine-tuned + RL model (with multi-step iterative inference and reward model feedback) against baseline models using zero/few-shot prompting. The comparison conflates model capability with training and inference procedure advantages. While Section 4.7 partially addresses this with SFT comparisons, the main results table presents a fundamentally unequal comparison."
     },
     {
       "flag": "No error bars or variance",
       "detail": "All results are reported as point estimates without confidence intervals, standard deviations, or any indication of result stability across runs. For RL-trained models, which are known to have high variance across seeds (Henderson et al., 2018), this is a significant omission."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "CodeNet solutions are publicly available since 2021. StarCoderBase was trained on The Stack (public code), and commercial models like GPT-3.5 could have encountered these solutions. The baseline comparisons may be unfairly disadvantaging or advantaging certain models depending on their training data."
     },
     {
       "flag": "Single dataset evaluation",
       "detail": "All experiments are conducted solely on CodeNet4Repair. No evaluation on other program repair benchmarks (e.g., Defects4J, QuixBugs, or other APR datasets) is performed, limiting evidence for generalization."
     },
     {
       "flag": "Compute costs unreported",
       "detail": "Training a 15B parameter model with SFT and PPO-based RL for 32k episodes requires substantial compute. No GPU hours, number of accelerators, or wall-clock time is reported, making it impossible to assess practical feasibility."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and the pass@k metric used in this paper for evaluating code generation capabilities."
     },
     {
       "title": "CodeRL: Mastering code generation through pretrained models and deep reinforcement learning",
       "authors": ["Hung Le", "Yue Wang", "Akhilesh Deepak Gotmare", "Silvio Savarese", "Steven Hoi"],
       "year": 2022,
       "relevance": "Pioneered using reinforcement learning for code generation, directly relevant to process-based feedback in code tasks."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu"],
       "year": 2022,
       "relevance": "The RLHF methodology that this paper's approach is built upon — aligning language models using reward models and PPO."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben"],
       "year": 2023,
       "relevance": "The foundation model (StarCoderBase 15.5B) used as the base for RePair's fine-tuning pipeline."
     },
     {
       "title": "Let's verify step by step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yura Burda"],
       "year": 2023,
       "relevance": "Key prior work on process-based reward models for mathematical reasoning, directly motivating this paper's approach to process supervision."
     },
     {
       "title": "Solving math word problems with process- and outcome-based feedback",
       "authors": ["Jonathan Uesato", "Nate Kushman", "Ramana Kumar"],
       "year": 2022,
       "relevance": "Compared process-based and outcome-based supervision methods on math problems, providing theoretical motivation for this paper's process-based APR approach."
     },
     {
       "title": "CRITIC: Large language models can self-correct with tool-interactive critiquing",
       "authors": ["Zhibin Gou", "Zhihong Shao", "Yeyun Gong"],
       "year": 2024,
       "relevance": "Related work on LLM self-correction with tool feedback, relevant to the feedback-based repair paradigm."
     },
     {
       "title": "Project CodeNet: A large-scale AI for code dataset for learning a diversity of coding tasks",
       "authors": ["Ruchir Puri", "David S Kung"],
       "year": 2021,
       "relevance": "Source dataset from which CodeNet4Repair was derived; a major code benchmark for evaluating program analysis tasks."
     },
     {
       "title": "DeepFix: Fixing common C language errors by deep learning",
       "authors": ["Rahul Gupta", "Soham Pal", "Aditya Kanade", "Shirish Shevade"],
       "year": 2017,
       "relevance": "Early deep learning approach to automated program repair; one of the baseline datasets compared in Table 1."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": ["Yuxiang Wei", "Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Contemporary LLM-based automated program repair approach relevant to the survey scope of AI-assisted programming."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "relevance": "Major work on competition-level code generation using large-scale sampling and filtering, relevant to AI coding capabilities."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "relevance": "Contemporary code LLM relevant to the survey scope of LLM capabilities for programming tasks."
     },
     {
       "title": "Code LLaMA: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2024,
       "relevance": "Major open-source code LLM that establishes baselines for code generation and repair capabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Requires training a 15B model with RL — not immediately usable by practitioners, though the dataset and approach could inform APR tool development."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "A 15B model matching GPT-3.5 on program repair is mildly surprising, but the paradigm of process-based feedback improving results is expected from prior work on math reasoning."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security implications — focused on improving program repair, a benign application."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; standard research contribution."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code and data released on GitHub, but running the system requires training a 15B model with RL, making it impractical to demo casually."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From USTC (University of Science and Technology of China), not a widely recognized AI lab in the broader tech community."
     }
   }
 }