ai-research-survey

scan.json (26728B)

---

 {
   "paper": {
     "title": "Neuro-Symbolic Verification on Instruction Following of LLMs",
     "authors": ["Yiming Su", "Kunzhao Xu", "Yanjie Gao", "Fan Yang", "Cheng Li", "Mao Yang", "Tianyin Xu"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.17789",
     "doi": "10.48550/arXiv.2601.17789"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The footnote states 'This paper is a result of an open-source project' but no repository URL or archive link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "VIFBENCH is described in detail but no download link or repository URL for the benchmark data is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper only mentions models are queried through API interfaces."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README are provided. Prompts are included in Appendix B but no instructions for running the full pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 report only point estimates (e.g., '94.8% F1 score') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims NSVIF 'significantly outperforms' baselines but reports no statistical significance tests — comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports percentage improvements with baseline context (e.g., '25.7%↑' from 69.1% to 94.8% F1), providing enough to gauge effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "VIFBENCH contains 820 data points. No justification is given for why this size was chosen or whether it provides sufficient statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures across runs are reported. Results appear to be from single runs at temperature 0.2."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: standard LLM-as-a-judge (§5), GEPA-COT, CONV-COT, and NSVIF-NEU ablation (Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use contemporary models (GPT-4.1, DeepSeek-R1/V3.1, Qwen3-Max) and recent prompt optimization techniques (GEPA from DSPy, conversation-based CoT)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents an explicit ablation study (§5.1.1) comparing NSVIF-NEU (without logic reasoning) and the full NSVIF, along with intermediate variants."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: F1 score, Precision, Recall, and Pass@1 (§5, Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated against VIFBENCH labels. No human evaluation of NSVIF's verification outputs is performed. The semantic constraints in VIFBENCH were spot-checked by authors during data generation (§A.2) but this is dataset construction, not system evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "VIFBENCH is a new benchmark not used for tuning NSVIF. NSVIF's in-context examples (Appendix B) are hand-crafted examples unrelated to VIFBENCH. GEPA-COT was optimized with a small minibatch (size=3) but the full 820-point benchmark is the evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 breaks down F1 by number of constraints (2-10). Table 4 breaks down false positives vs false negatives. Table 2 breaks down by model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.3 provides detailed analysis of 23 false positives and 18 false negatives, categorizing root causes (hallucination, checker failures, misclassification). §5.2 discusses GPT-4o-mini failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o-mini makes NSVIF worse than the baseline (37.3% vs 66.6% F1, Table 2), which is explicitly discussed as an unexpected negative finding in §5.2."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims NSVIF 'significantly outperforms LLM-based approaches' — supported by Table 2 showing 7-25.7% F1 improvements across models. Claim about interpretable feedback is supported by the constraint-level explanations described in §3.2. Claim about improving instruction following without post-training is supported by §5.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 3) makes causal claims about component contributions through controlled single-variable manipulation: NSVIF vs NSVIF-NEU isolates the logic reasoning component. The design is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims NSVIF is 'universal, general-purpose' (abstract, §1, §3) but evaluation is limited to VIFBENCH which covers only English writing tasks with 8 logic and 2 semantic constraint types. The limitations (§8) acknowledge this but the title and abstract framing far exceeds the evaluation scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for NSVIF's improvements beyond the intended neuro-symbolic mechanism. For instance, the additional compute budget (100x more tokens per Table 7) is not discussed as a potential confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures F1/precision/recall on VIFBENCH for instruction-following verification, and claims are about verification effectiveness. The measurements match the claims at the granularity presented."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "§5 specifies model versions with dates: GPT-4.1 (2025-04-14), GPT-4o (2024-11-20), DeepSeek-R1 (0528), Qwen3-Max (2025-09-23), GPT-4o-mini (2024-07-18)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts for all three NSVIF agents (formulation, checking, solver), NSVIF-NEU, baseline, GEPA-COT, CONV-COT, and the travel agent are provided in Appendix B."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0.2 is stated for all models (§A.5). GEPA parameters are reported: max_full_evals=1, num_threads=1, reflection_minibatch_size=3. Self-reflection budget of 3 retries for checker code."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffold is described in detail in §3.2: formulation agent, checking agent, and solver agent, with workflow diagram (Figure 1), retry logic (3 self-reflections), and fallback mechanism."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§4 and Appendix A document VIFBENCH construction: constraint selection from existing datasets, instruction synthesis via Cartesian products, output generation with sat/unsat balancing, mutation strategy for unsat examples, and re-generation budget of 5."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§8 'Limitations' provides substantive discussion covering CSP formulation limits, LLM hallucination paradox, benchmark scope, and model coverage."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§8 discusses specific threats: CSP cannot handle soft vs hard constraints, unsatisfiable instructions could cause infinite retries, VIFBENCH limited to English writing tasks, LLM hallucination during verification is irreducible, and evaluation limited to available models."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§8 explicitly states what was not tested: multilingual instructions, tasks beyond writing (information retrieval, QA, planning), and LLMs beyond those evaluated. §2.1 also clarifies NSVIF checks behavior correctness, not linguistic quality."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "VIFBENCH data (820 instruction-output-result tuples) is not made available for download. No data archive or repository link is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§4 describes in detail how VIFBENCH was constructed: constraint types sourced from instruction-tuning datasets and benchmarks (Table 5), constraint values generated by GPT-4.1, instructions synthesized via Cartesian product permutation, outputs generated by GPT-4.1 with sat/unsat balancing."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. VIFBENCH is entirely synthetic."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "§4.1-4.2 and Appendix A document the full pipeline: constraint definition → Cartesian product combinations (with 100-cap) → instruction synthesis via templates → GPT-4.1 output generation → sat verification with re-prompting (budget=5) → mutation for unsat examples. Counts given: 820 total, 391 sat, 429 unsat."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments are provided. The paper mentions Kunzhao Xu's internship at Microsoft Research but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UIUC, USTC, and Microsoft Research. The MSR internship is noted in the footnote."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three authors are from Microsoft Research. The paper evaluates Azure AI Foundry models (GPT-4.1, GPT-4o, GPT-4o-mini) and the models are 'provided by Azure AI Foundry.' Microsoft has a financial interest in demonstrating the need for better LLM verification tools. This dependency is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (GPT-4.1, GPT-4o, DeepSeek-R1/V3.1, Qwen3-Max)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether VIFBENCH data or similar instruction-following examples could have appeared in model training data. While VIFBENCH is newly created, the constraint types and writing topics may overlap with training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VIFBENCH is new and presumably not in training data, but this is not explicitly discussed or verified. The paper does not address contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 reports input and output token usage for every model and verifier combination on VIFBENCH, showing e.g., NSVIF with GPT-4.1 uses 130M input + 6.7M output tokens."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 provides total token consumption for all experiments. Models are accessed via API (Azure AI Foundry, Alibaba Cloud) so token counts serve as the compute budget metric."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results are reported at temperature 0.2 but no multiple-seed experiments are conducted."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for NSVIF. The self-reflection count (3) and temperature (0.2) appear chosen without documented search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Default configurations are used without justification. The choice of 3 self-reflections, temperature 0.2, and specific prompt designs are not justified through systematic comparison."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 6 models × 4 verifiers with no correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own system (NSVIF) and benchmark (VIFBENCH) without acknowledging this bias. The benchmark was designed by the same team that built NSVIF."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "NSVIF uses ~130M input tokens vs ~812K for the baseline (Table 7) — roughly 160x more compute. This massive cost difference is not discussed as a factor in the performance comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§4 discusses why LLMBar is inadequate (preference ≠ instruction following) and designs VIFBENCH with fine-grained constraint labels to address this validity gap. The paper explicitly discusses what the benchmark measures vs what is claimed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "When comparing across models (Table 2), the same NSVIF scaffold is used for all models, and the same baseline prompt is used for all models, controlling the scaffold variable. The scaffold IS the independent variable when comparing NSVIF vs baseline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. While VIFBENCH is newly created, the paper does not explicitly state when it was created relative to model training cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup could leak answer information. For instance, NSVIF provides the full instruction and output to the LLM multiple times across agents."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VIFBENCH instructions are generated from Cartesian products of constraint combinations, creating structural similarities across test examples. This non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "NSVIF significantly outperforms LLM-as-a-judge baselines, achieving up to 25.7% F1 improvement on VIFBENCH.",
       "evidence": "Table 2 shows NSVIF with GPT-4.1 achieves 94.8% F1 vs 69.1% for baseline (25.7%↑). Similar improvements across DeepSeek-V3.1 (22.0%↑) and Qwen3-Max (23.2%↑).",
       "supported": "moderate"
     },
     {
       "claim": "The neuro-symbolic approach with logic reasoning substantially improves verification over using LLMs alone.",
       "evidence": "Table 3 ablation: NSVIF (94.8% F1) vs NSVIF-NEU without logic reasoning (84.0% F1) vs best LLM-only approach CONV-COT (78.9% F1), all using GPT-4.1.",
       "supported": "moderate"
     },
     {
       "claim": "NSVIF provides interpretable feedback that helps LLMs improve instruction following without post-training.",
       "evidence": "§5.4: On 9 selected VIFBENCH instructions, GPT-4.1 produces satisfying outputs in <5 iterations with NSVIF feedback, vs 3/9 failing within 15 iterations with boolean-only feedback.",
       "supported": "weak"
     },
     {
       "claim": "NSVIF is a universal, general-purpose verifier making no assumption on the instruction or LLM.",
       "evidence": "The framework description (§3) makes no structural assumptions, but evaluation (§5) is limited to VIFBENCH's English writing tasks with 10 constraint types.",
       "supported": "weak"
     },
     {
       "claim": "Small models (GPT-4o-mini) are ineffective for NSVIF due to constraint misclassification.",
       "evidence": "§5.2: GPT-4o-mini misclassifies constraints in 508/820 instructions, causing NSVIF (37.3% F1) to underperform the baseline (66.6% F1).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "NSVIF, a neuro-symbolic framework for verifying LLM instruction following, formulates verification as a constraint-satisfaction problem combining logic reasoning (via Z3 SMT solver) with semantic analysis. On VIFBENCH (820 examples), NSVIF achieves 94.8% F1 with GPT-4.1, outperforming standard LLM-as-a-judge by 25.7%. Ablation shows logic reasoning for symbolic constraints accounts for ~11% F1 improvement over neural-only verification. However, NSVIF requires ~160x more tokens than the baseline and fails with smaller models (GPT-4o-mini) due to constraint misclassification.",
   "red_flags": [
     {
       "flag": "Massive compute cost difference ignored",
       "detail": "NSVIF uses ~130M input tokens vs ~812K for the baseline (Table 7), approximately 160x more compute. This enormous cost difference is not discussed when comparing effectiveness, making the comparison misleading — the baseline could potentially improve with equivalent compute budget (e.g., multiple LLM calls, ensemble voting)."
     },
     {
       "flag": "Authors evaluate on their own benchmark",
       "detail": "Both NSVIF and VIFBENCH are created by the same team. The benchmark's constraint types (8 logic + 2 semantic, all in English writing) may favor NSVIF's architecture. No external benchmark evaluation is included."
     },
     {
       "flag": "Universality claims far exceed evaluation scope",
       "detail": "The paper repeatedly claims NSVIF is 'universal, general-purpose' but evaluates only on English writing tasks with 10 constraint types. Real-world instruction following involves code generation, multi-modal outputs, tool use, and complex reasoning — none tested."
     },
     {
       "flag": "No variance or significance testing",
       "detail": "All results are point estimates from apparent single runs at temperature 0.2. LLM outputs are stochastic; without multiple runs and significance tests, the claimed improvements may not be reliable."
     },
     {
       "flag": "Multi-turn feedback evaluation on 9 cherry-picked examples",
       "detail": "§5.4 evaluates NSVIF's feedback loop on only 9 instructions 'which the LLM failed to follow,' sampled by the authors. This tiny, non-random sample cannot support generalizable claims about NSVIF improving instruction following."
     }
   ],
   "cited_papers": [
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang"],
       "year": 2025,
       "relevance": "Studies failure modes in multi-agent LLM systems, directly relevant to agentic AI reliability."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced Codex/HumanEval benchmark and Pass@k metric used in this paper's evaluation."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM code generation agents on real software engineering tasks."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
       "authors": ["Omar Khattab", "Arnav Singhvi"],
       "year": 2024,
       "relevance": "Framework for structured LLM programming, used for GEPA-COT baseline optimization in this paper."
     },
     {
       "title": "LLMs Get Lost In Multi-Turn Conversation",
       "authors": ["Philippe Laban", "Hiroaki Hayashi"],
       "year": 2025,
       "arxiv_id": "2505.06120",
       "relevance": "Studies LLM instruction following degradation in multi-turn settings, relevant to agentic workflow reliability."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-judge methodology, the baseline approach this paper improves upon."
     },
     {
       "title": "Benchmarking Complex Instruction-Following with Multiple Constraints Composition",
       "authors": ["Bosi Wen", "Pei Ke"],
       "year": 2024,
       "relevance": "ComplexBench — instruction following benchmark that models instructions as constraint compositions, directly inspiring VIFBENCH."
     },
     {
       "title": "Instruction-Following Evaluation for Large Language Models",
       "authors": ["Jeffrey Zhou", "Tianjian Lu"],
       "year": 2023,
       "arxiv_id": "2311.07911",
       "relevance": "IFEval benchmark for LLM instruction following, part of the evaluation landscape this paper builds on."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework; the CONV-COT baseline uses a conversation-based approach inspired by this work."
     },
     {
       "title": "LINC: A Neurosymbolic Approach for Logical Reasoning by Combining Language Models with First-Order Logic Provers",
       "authors": ["Theo Olausson", "Alex Gu"],
       "year": 2023,
       "relevance": "Prior neurosymbolic approach combining LLMs with formal logic, directly inspiring NSVIF's architecture."
     },
     {
       "title": "SWE-smith: Scaling Data for Software Engineering Agents",
       "authors": ["John Yang", "Kilian Lieret"],
       "year": 2025,
       "arxiv_id": "2504.21798",
       "relevance": "Scaling training data for SE agents; cited for Python being the best LLM-supported language."
     },
     {
       "title": "AGENTIF: Benchmarking Instruction Following of Large Language Models in Agentic Scenarios",
       "authors": ["Yunjia Qi", "Hao Peng"],
       "year": 2025,
       "arxiv_id": "2505.16944",
       "relevance": "Benchmark specifically for instruction following in agentic contexts, directly related to NSVIF's target use case."
     }
   ]
 }