ai-research-survey

scan.json (29448B)

---

 {
   "paper": {
     "title": "ReMind: Understanding Deductive Code Reasoning in LLMs",
     "authors": ["Jun Gao", "Yun Peng", "Xiaoxue Ren"],
     "year": 2025,
     "venue": "arXiv (under review)",
     "arxiv_id": "2511.00488",
     "doi": "10.48550/arXiv.2511.00488"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMs exhibit a fundamental gap (up to 44.7%) between code generation and deductive reasoning abilities, even for code they generate themselves. A self-reasoning bias causes up to 43% relative performance drop when reasoning about code from other LLMs. The proposed ReMind multi-agent framework (Mutator, Executor, Inspector) achieves up to 23.2 absolute accuracy improvement over baselines and substantially reduces cross-source performance variance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymous repository link is provided in Section 8: 'The code and data are available at https://anonymous.4open.science/r/remind-71F0/'. This is an anonymized URL for review, but it is a working link rather than a promise of future release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same anonymous repository is stated to contain both code and data. Additionally, the benchmarks used (HumanEval, LiveCodeBench) are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided in the paper. The only tool mentioned is Python's AST module (footnote 1)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The anonymous link may contain a README, but the paper itself does not include reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-7 are point estimates with no confidence intervals or error bars. Table 5 reports standard deviation across code sources (not across experimental runs), which measures cross-source stability rather than experimental uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used anywhere. Claims like 'ReMind consistently and significantly outperforms all baselines' (Section 4.2.1) are based solely on comparing raw accuracy numbers without any formal test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute accuracy differences with baseline context throughout, e.g., 'maximum average improvement of 23.2 in absolute accuracy over the best performing baselines' (abstract), and provides full accuracy tables (Tabs 2-7) showing both proposed and baseline numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the 152 HumanEval and 328 LiveCodeBench problems. These sizes are artifacts of the filtering protocol (problems where all 5 LLMs produce correct code) rather than a deliberate sample size choice."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports standard deviation of accuracy across different code sources, not across experimental runs. There is no mention of running experiments multiple times or reporting variance across seeds/runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: CoT (Chain-of-Thought), CoC (Chain-of-Code), and RHDA (Reflective Hypothesis Decomposition and Amendment). Described in Section 4.1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CoC (Li et al., ICML) and RHDA (Zhao et al., ICLR 2025) are recent and represent current approaches to code reasoning. The baselines are appropriate and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 presents an ablation study removing the Mutator (M) and Inspector (I) individually and together, showing each component's contribution across 5 LLMs on both benchmarks."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only one metric is used: accuracy as defined in Equation 1 (product of per-test-case correctness indicators). Table 5 adds standard deviation across sources as a stability measure, but this is derived from the same accuracy metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of reasoning traces or outputs. All evaluation is automated via the accuracy metric comparing predicted outputs to ground truth execution results."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench is explicitly used as a zero-shot evaluation set with problems 'later than the knowledge cutoff of OpenAI o1' (Section 2.2). HumanEval and LiveCodeBench serve as separate evaluation benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by executor LLM (rows), code source/generator LLM (columns), benchmark (HumanEval vs LiveCodeBench), and method (CoT, CoC, RHDA, ReMind) in Tables 2-7."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 presents two detailed motivating examples (Fig. 3) showing specific reasoning errors. Section 5.1 provides a case study (Fig. 5) demonstrating how ReMind detects and corrects errors that other methods miss."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CoC and RHDA sometimes hurt performance compared to native CoT for reasoning models like DeepSeek-R1 and o1 (Section 4.2.1: 'Both CoC and RHDA fluctuate around 75% accuracy, significantly underperforming native CoT prompting'). The empirical study itself documents LLM failures."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'intrinsic gap,' 'consistent bias towards code sources,' and '23.2 in absolute accuracy' improvement are all supported by the experimental results in Tables 2-7. The empirical findings in RQ1-RQ3 back the three stated challenges."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 7) provides controlled single-variable removal of Mutator and Inspector to support claims about component contributions. The mutation experiment (Table 2, 'Mutation' rows) isolates the effect of style bias by having the executor rewrite code before reasoning."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to address 'Deductive Code Reasoning in LLMs' broadly, but experiments are conducted only on Python code (HumanEval and LiveCodeBench are Python-only), with 5 specific LLMs, and a single accuracy metric. The abstract says 'five LLMs' and 'two benchmarks' without bounding claims to Python or these specific models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.3 (Threats to Validity) lists limitations (weaker models, sequential programs, computational overhead) but does not discuss alternative explanations for the observed improvements. No consideration of whether ReMind's gains stem from test-time compute scaling alone rather than the specific architectural design."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy of output prediction (Eq. 1) and frames it as 'deductive code reasoning' ability. The metric directly measures output prediction correctness, which is the claimed capability. No proxy gap exists between the measurement and the claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4o-mini,' 'DeepSeek-V3,' 'DeepSeek-R1,' and 'o1' without specific version dates, snapshot IDs, or API versions. No model version identifiers like 'gpt-4o-mini-2024-07-18' are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states models are 'prompted with simple while widely used Chain-of-Thought Prompting (CoT)' (Section 2.2) but does not provide the actual prompt text used for CoT, CoC, RHDA, or any ReMind agent prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the 5 LLMs used. Test cases are capped at 15 (Section 2.2), but LLM inference settings are not stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The ReMind multi-agent framework is described in detail in Section 3 with a workflow diagram (Fig. 4). The roles of Mutator (Section 3.1.1), Executor (Section 3.1.2), and Inspector (Section 3.1.3) are explained, including the CFG-based validation mechanism."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the validation protocol in detail: LLMs generate code, solutions are validated against official test cases, only instances where all LLMs produce correct code are retained. Resulting sizes (152 HumanEval, 328 LiveCodeBench) and test case cap (15) are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 'Threats to Validity' provides a dedicated subsection discussing limitations with substantive content across three paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 identifies specific threats: (1) weaker models may not benefit from mutation/inspection, (2) the approach is limited to 'sequential, deterministic programs' and struggles with 'concurrent, asynchronous, or side-effect-heavy code,' (3) computational overhead of the multi-agent workflow."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 explicitly states the approach is 'primarily designed for sequential, deterministic programs' and 'may face challenges when applied to concurrent, asynchronous, or side-effect-heavy code (e.g., I/O operations, network calls).'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states 'The code and data are available at https://anonymous.4open.science/r/remind-71F0/'. The underlying benchmarks (HumanEval, LiveCodeBench) are also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the data collection: LLMs generate code for generation benchmarks, code is validated against test cases, filtering retains only problems where all models produce correct solutions. Source benchmarks and their properties are described in Table 1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (HumanEval, LiveCodeBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The filtering pipeline is documented: start with full benchmarks (164 HumanEval, 880 LiveCodeBench) → select time period (Oct 2023-Jan 2025 for LiveCodeBench) → generate code from 5 LLMs → validate → retain mutual correct solutions → cap test cases at 15 → final sets (152, 328)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper. No acknowledgments section discusses grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Jun Gao and Xiaoxue Ren at Zhejiang University, Yun Peng at The Chinese University of Hong Kong. Authors are from academic institutions and do not evaluate products from their employers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funding cannot be assessed. The absence of any funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions LiveCodeBench problems are 'later than the knowledge cutoff of OpenAI o1' (Section 2.2) but does not state specific training cutoff dates for any of the 5 models (GPT-4o-mini, DeepSeek-V3, DeepSeek-R1, o1-Low, o1-High)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 (RQ3) explicitly discusses contamination: 'HumanEval has been publicly available for several years, raising concerns that potential data leakage may compromise the authenticity of reported findings.' LiveCodeBench is used specifically to address this concern."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges HumanEval contamination risk and uses LiveCodeBench (problems released Oct 2023-Jan 2025, after model knowledge cutoffs) as a contamination-free evaluation. This temporal split is a concrete mitigation strategy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments evaluate LLM performance on code benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 reports 'ReMind requires 5.0 and 5.2 API calls for HumanEval and LiveCodeBench on average' and compares this to baselines (CoT/CoC: fewer, RHDA: 5.3/5.5). While not dollar costs, the API call count provides a concrete cost comparison."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated — no GPU hours, total API spend, or wall-clock time for the full experimental evaluation across 5 LLMs × 2 benchmarks × 4 methods."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds, seed sensitivity, or results across multiple seeds anywhere in the paper. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be single-run without explicit confirmation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The mutation strategies and ReMind configuration appear fixed with no description of how design choices were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the ReMind configuration (number of mutations, iteration count, etc.) was selected. The paper does not explain whether configurations were tuned on a validation set or chosen a priori."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable. The absence of statistical tests is captured in the significance_tests item."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (CoT, CoC, RHDA) and evaluate their own system (ReMind) without acknowledging the bias of evaluating their own implementation of baselines vs. their own proposed system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 acknowledges ReMind uses more API calls than CoT/CoC but does not report performance as a function of compute budget. No performance-compute curves or matched-compute comparisons are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper repurposes code generation benchmarks (HumanEval, LiveCodeBench) for evaluating deductive reasoning but does not discuss whether these benchmarks actually measure the claimed 'deductive code reasoning' capability or whether the construct is valid."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "ReMind IS the scaffold being tested. The comparison is between different methods (ReMind vs CoT/CoC/RHDA), not between different models using the same scaffold. The scaffold is the independent variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly uses LiveCodeBench with problems from 'Oct 2023 to Jan 2025 (later than the knowledge cutoff of OpenAI o1)' to mitigate temporal leakage (Section 2.2)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The code and test inputs are provided to LLMs, but there is no analysis of whether context formatting could provide unintended hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential non-independence between HumanEval and LiveCodeBench problems, or between training data and test problems beyond temporal separation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "A temporal split is used as a concrete prevention method: LiveCodeBench problems are selected from after the models' knowledge cutoffs. This is a deliberate leakage prevention strategy, not just conceptual discussion."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs demonstrate up to a 44.7% gap between code generation and deductive code reasoning abilities",
       "evidence": "Table 2 shows GPT-4o-mini achieves only 55.3% accuracy on self-execution in HumanEval despite producing functionally correct code (100% pass rate), yielding a 44.7% gap. All models show non-zero gaps (Section 2.3, RQ1).",
       "supported": "strong"
     },
     {
       "claim": "LLMs exhibit a consistent self-reasoning bias, with up to 43% relative performance drop in cross-execution settings",
       "evidence": "Table 2 and Figure 1(b) show DeepSeek-V3 drops from 85.5% (self-execution) to 48.7% when reasoning about o1-Low's code, a 43% relative decline. All models show diagonal dominance in the performance matrix (Section 2.3, RQ2).",
       "supported": "strong"
     },
     {
       "claim": "ReMind achieves a maximum average improvement of 23.2 in absolute accuracy over best baselines",
       "evidence": "Comparing Tables 4 and 6: on LiveCodeBench with DeepSeek-R1, ReMind averages 77.7% vs RHDA at 53.7% (~24pp) and CoC at 54.5% (~23.2pp). Multiple LLM×benchmark combinations show large improvements.",
       "supported": "strong"
     },
     {
       "claim": "ReMind eliminates self-reasoning bias and sustains high robustness regardless of code source",
       "evidence": "Table 5 shows ReMind achieves the lowest standard deviation across code sources (2.9 on HumanEval, 1.3 on LiveCodeBench) compared to CoT (8.7, 3.9). Tab 4 color coding also shows uniform performance.",
       "supported": "moderate"
     },
     {
       "claim": "ReMind enables robust zero-shot generalization on complex benchmarks",
       "evidence": "Table 6 shows ReMind with o1-High achieves 78.4% average on LiveCodeBench vs CoT (59.1%), CoC (57.6%), RHDA (61.1%). Performance gap between HumanEval and LiveCodeBench is smaller for ReMind than baselines.",
       "supported": "moderate"
     },
     {
       "claim": "Both Mutator and Inspector are indispensable components, and their collaboration yields the best performance",
       "evidence": "Table 7 ablation study: removing Inspector causes larger drops than removing Mutator (e.g., DS-V3 LiveCodeBench: 70.1 → 54.9 w/o I vs 70.1 → 58.0 w/o M). Removing both degrades to CoT performance.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All results across Tables 2-7 are point estimates without confidence intervals, error bars, or significance tests. Claims of superiority ('significantly outperforms') are made by comparing raw accuracy numbers. With no indication of how many runs were conducted, results could be due to sampling noise."
     },
     {
       "flag": "Single-run experiments without seed sensitivity",
       "detail": "The paper does not state how many runs produced the results or whether seed sensitivity was tested. LLM outputs are stochastic, and single-run results on 152-328 problems may not be stable. This is especially concerning given the paper's own finding about reasoning inconsistency."
     },
     {
       "flag": "Model versions not specified",
       "detail": "No exact API versions or snapshot dates are given for any of the 5 LLMs. Since LLM behavior changes across versions, results may not be reproducible with current model versions."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only accuracy (Eq. 1) is reported. The strict all-or-nothing metric (product across test cases) could obscure partial improvements. No complementary metrics like per-test-case accuracy or reasoning trace quality are provided."
     },
     {
       "flag": "Prompts not provided",
       "detail": "None of the actual prompts used for CoT, CoC, RHDA, or the three ReMind agents (Mutator, Executor, Inspector) are provided in the paper. This makes the comparison between methods unreproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the primary benchmark used in this paper's evaluation of LLM code reasoning."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "relevance": "Contamination-free code evaluation benchmark used as the zero-shot test set in this paper."
     },
     {
       "title": "CruxEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Rozière", "Hugh Leather"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Code reasoning benchmark discussed as insufficient for studying deductive reasoning challenges."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of the five evaluated LLMs, a reasoning-focused model tested for deductive code reasoning."
     },
     {
       "title": "DeepSeek-V3 technical report",
       "authors": ["Aixin Liu", "Bei Feng"],
       "year": 2024,
       "arxiv_id": "2412.19437",
       "relevance": "One of the five evaluated LLMs, showing the strongest self-reasoning bias in cross-execution settings."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting method used as a baseline and the default reasoning strategy in the empirical study."
     },
     {
       "title": "Chain of Code: Reasoning with a Language Model-Augmented Code Emulator",
       "authors": ["Chengshu Li", "Jacky Liang", "Andy Zeng"],
       "relevance": "Code-specific reasoning baseline that prompts LLMs to simulate execution, compared against ReMind."
     },
     {
       "title": "Unveiling the Magic of Code Reasoning through Hypothesis Decomposition and Amendment",
       "authors": ["Yuze Zhao", "Tianyun Ji", "Wenjun Feng"],
       "year": 2025,
       "relevance": "RHDA framework baseline that decomposes execution into sub-procedures, the strongest non-ReMind baseline."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez"],
       "year": 2024,
       "relevance": "Multi-agent code reasoning system relevant to understanding LLM-based software engineering agents."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "LLM-driven multi-agent framework using structured interaction and role-based constraints for collaborative problem solving."
     },
     {
       "title": "CodeHalu: Investigating code hallucinations in LLMs via execution-based verification",
       "authors": ["Yuchen Tian", "Weixiang Yan"],
       "year": 2025,
       "relevance": "Studies code hallucination in LLMs through execution-based verification, directly related to code reasoning reliability."
     },
     {
       "title": "How Accurately Do Large Language Models Understand Code?",
       "authors": ["Sabaat Haroon", "Ahmad Faraz Khan"],
       "year": 2025,
       "arxiv_id": "2504.04372",
       "relevance": "Empirical study of LLM code understanding accuracy, using mutation-based evaluation related to ReMind's Mutator."
     },
     {
       "title": "Mutation-guided llm-based test generation at Meta",
       "authors": ["Christopher Foster", "Abhishek Gulati", "Mark Harman"],
       "year": 2025,
       "arxiv_id": "2501.12862",
       "relevance": "Industry-scale LLM mutation testing at Meta, relevant to understanding LLM-driven code mutation approaches."
     }
   ]
 }