ai-research-survey

scan.json (25049B)

---

 {
   "paper": {
     "title": "TextResNet: Decoupling and Routing Optimization Signals in Compound AI Systems via Deep Residual Tuning",
     "authors": ["Suizhi Huang", "Mei Li", "Han Yu", "Xiaoxiao Li"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.08306"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TextResNet introduces a framework for decoupling and routing textual gradients in Compound AI Systems via four innovations: Additive Semantic Deltas, Semantic Projector, Causal Routing, and Density-Aware Scheduling. On HotpotQA, it surpasses TextGrad by +21.37 F1 (46.23 vs 24.86). The framework achieves ~3x token efficiency (21k vs 63k tokens) while maintaining stability in deep chains where baselines collapse. Ablation studies confirm each component contributes monotonically to performance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub URL is provided in the abstract: https://github.com/JeanDiable/TextResNet."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HotpotQA, BigCodeBench (HuggingFace), PubMedQA, and STaRK-PRIME (HuggingFace). Data sources and splits are documented in Appendix B.1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper lists models used but not library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While hyperparameters and configurations are detailed in Appendix B, there are no specific commands or scripts referenced for reproducing results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports ± notation for all methods across all benchmarks (e.g., '46.23 ± 1.15' for TextResNet on HotpotQA)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests (p-values, t-tests, etc.) are reported despite claims that TextResNet 'outperforms' baselines. Comparisons rely solely on comparing means with standard deviations."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context (e.g., '+21.37 F1' over TextGrad on HotpotQA, from 24.86 to 46.23). This provides sufficient context for understanding magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of train/dev/test split sizes (e.g., 1000/250/100 for HotpotQA, 500/25/70 for BigCodeBench). No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports standard deviations across runs (e.g., ± 1.15). Appendix B.4 states 'Test repeats: 3' and 'Repeated dev trials: 3'."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baselines are compared: CoT, HBC, DSPy (MIPRO), TextGrad, and TextGrad+Sum (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "TextGrad (2025), DSPy/MIPRO (2024), and TextGrad+Sum (2025a) are recent. OPTIMAS (2025) is discussed but excluded from comparison with justification."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides a progressive ablation study incrementally adding each of the four components (Res, Proj, Route, Sched) on HotpotQA and BigCodeBench."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four different metrics across benchmarks: F1 (HotpotQA), Pass% (BigCodeBench), Accuracy (PubMedQA), MRR (STARK-PRIME)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated (F1, pass rate, accuracy, MRR)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 describes explicit train/dev/test splits for all four benchmarks. The dev set is used for validation during optimization, and separate test sets are used for final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (Table 1). Additional per-component analysis in Figures 4-6 and per-scheduling-strategy breakdowns in Figure 9."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 (Figure 1) provides detailed failure case analysis showing three failure modes (Signal Blockage, Downstream Over-correction, Upstream Pollution). Section 6.3 tests robustness under batch shuffling intervention."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "TextGrad+Sum is shown to often underperform vanilla TextGrad (Section 5.2). Figure 7 shows TextGrad+Sum performs worst in deep chains. Ablation variants show degraded performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'superior performance' and 'remarkable stability' are supported by Table 1 (performance) and Figure 7 (stability across chain depths)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about each component's contribution are supported by controlled ablation studies (Table 2) with single-variable manipulation. The batch shuffling intervention (Section 6.3) provides a counterfactual test of causal attribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims optimization for 'Compound AI Systems' broadly, but results are on four specific benchmarks with specific pipeline configurations. The paper does not bound its generalization claims to these tested settings. Section G discusses limitations of window control but not generalization boundaries."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. For instance, the gains could partly come from the additional LLM calls for the Semantic Projector, or from the structured prompt format rather than the routing mechanism. Section G discusses limitations but not alternative explanations for the observed results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific metrics (F1, pass rate, accuracy, MRR) on specific benchmarks and frames claims at the level of these metrics. It does not overclaim broader capabilities beyond the measured proxies."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 specifies exact model names: 'openai/gpt-4o-mini', 'anthropic/claude-3-haiku', 'anthropic/claude-3-haiku-20240307'. These include version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides full prompt templates for the Backward Semantic Projector (Listing 1), Prompt Optimizer (Listing 2), and auxiliary formatting constraints including the Causal Routing and Context Construction templates."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3-6 provide comprehensive hyperparameter details: temperatures, max tokens, batch sizes, training steps, evaluation frequency, random seed, and resource limits for code execution."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline architecture is described in detail: Appendix B.2 documents all four system pipelines with component names and dataflow. Figure 2 provides an architectural overview. The three-stage optimization process is thoroughly described in Section 4."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 documents data splits, sampling procedures, and preprocessing for each benchmark. Appendix B.4 describes trace/stderr truncation limits and retrieval top-k settings."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G 'Limitations and Future Directions' provides a dedicated limitations discussion covering context window control limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Appendix G discusses only one limitation (window control) and frames it as a future research direction. It does not discuss threats specific to the experimental validity of the current results (e.g., limited benchmark diversity, reliance on specific LLMs, sensitivity to prompt formatting)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to the tested pipeline configurations, models, or benchmark types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual predictions, per-example scores, optimization trajectories) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 describes how data is sourced for each benchmark (HuggingFace datasets, standard splits, random seed-based split generation)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.1 documents the pipeline from data source to final splits including split ratios, random seeds, and subset sizes. Appendix B.4-B.6 document the forward execution pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Nanyang Technological University, Shanghai Jiao Tong University, University of British Columbia, Vector Institute."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o-mini and Claude 3 Haiku but does not state their training data cutoff dates. These models may have been trained on data including some benchmark solutions."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between model training data and benchmark test sets. HotpotQA (2018) and PubMedQA (2019) are old benchmarks that likely appear in LLM training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of benchmark contamination risk despite using well-known benchmarks (HotpotQA, PubMedQA) that predate the models' training cutoffs."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 10 reports token consumption: TextResNet uses ~21k feedback tokens vs TextGrad's ~63k tokens over 100 steps. Token efficiency is analyzed in Appendix F.4."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API cost, GPU hours, or wall-clock time is reported. Only feedback token counts are shown, which is a partial cost measure (does not include forward pass tokens or API pricing)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 states 'Random seed: 42' (single seed). While ± values are reported in Table 1, these appear to be from 3 test repeats (Table 3), not multiple seeds. No seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 states 'Test repeats: 3' and 'Repeated dev trials: 3'."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The Boltzmann temperature τ and other hyperparameters appear tuned but no search process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 describes validation-based selection: 'evaluate on a fixed held-out validation set every two optimization steps, and select the best-performing prompt.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No significance tests are performed at all, let alone corrections for multiple comparisons across four benchmarks and multiple baselines."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own framework and compare against their own implementations of baselines. No acknowledgment of self-comparison bias per Lucic et al. (2018). OPTIMAS is excluded from comparison with a justification that could be seen as avoiding a harder baseline."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 10(b) directly compares total feedback token consumption vs final F1 performance for TextResNet vs TextGrad."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HotpotQA, BigCodeBench, PubMedQA, or STARK-PRIME actually measure the claimed capability of 'optimization in Compound AI Systems.' The benchmarks are used because OPTIMAS used them, without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'we adopt the SCG configurations of OPTIMAS to ensure topological consistency.' All methods use the same pipeline architecture, controlling for scaffold differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage despite using benchmarks (HotpotQA 2018, PubMedQA 2019) created well before the models' training data was collected."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context that wouldn't be available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train/dev/test splits share structural similarities or whether examples are truly independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "TextResNet surpasses TextGrad by +21.37 F1 on HotpotQA",
       "evidence": "Table 1: TextResNet 46.23 ± 1.15 vs TextGrad 24.86 ± 1.19 on HotpotQA F1",
       "supported": "moderate"
     },
     {
       "claim": "TextResNet achieves 37.86% pass rate on BigCodeBench, improving over DSPy (33.81%) and TextGrad (35.71%)",
       "evidence": "Table 1: TextResNet 37.86 ± 0.45 vs DSPy 33.81 ± 2.75 and TextGrad 35.71 ± 0.10",
       "supported": "moderate"
     },
     {
       "claim": "TextResNet maintains performance stability as chain depth increases while TextGrad collapses",
       "evidence": "Figure 7 shows TextResNet stable from L=5 to L=20 while TextGrad degrades. However, the experiment uses synthetic 'Identity Node' padding.",
       "supported": "moderate"
     },
     {
       "claim": "TextResNet consumes approximately 3x fewer tokens than TextGrad while achieving nearly double the performance",
       "evidence": "Figure 10(b): 21,909 vs 63,073 total feedback tokens; F1 46.23 vs 24.86",
       "supported": "moderate"
     },
     {
       "claim": "Each of the four components contributes monotonically to performance",
       "evidence": "Table 2 ablation: 24.86 → 32.15 → 36.69 → 37.71 → 46.23 F1 on HotpotQA as components are added",
       "supported": "strong"
     },
     {
       "claim": "TextResNet correctly identifies batch-shuffled upstream errors in 96% of cases",
       "evidence": "Figure 5 and Section 6.3: counterfactual batch shuffling intervention on HotpotQA",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "OPTIMAS excluded from comparison",
       "detail": "The most relevant baseline (OPTIMAS, which uses the same benchmark suite and pipeline configurations) is excluded from Table 1 with justification about 'fundamental divergence in supervision.' However, OPTIMAS results would provide important context. The stated reason (RL-based training vs training-free) is a valid methodological difference, but readers cannot assess relative performance."
     },
     {
       "flag": "No significance tests",
       "detail": "Claims of 'outperformance' across all benchmarks rely on comparing means ± std without any statistical tests. Some improvements are within overlapping error ranges (e.g., PubMedQA: 60.31 ± 1.51 vs DSPy 60.26 ± 0.40)."
     },
     {
       "flag": "Single random seed",
       "detail": "All experiments use seed=42 with only 3 test repeats. No seed sensitivity analysis despite the stochastic nature of LLM-based optimization."
     },
     {
       "flag": "Synthetic depth scalability test",
       "detail": "The depth scalability experiment (Figure 7) adds artificial 'Identity Nodes' to extend chains. This may not represent real-world deep chains where each node has meaningful computation."
     },
     {
       "flag": "No contamination analysis",
       "detail": "HotpotQA (2018) and PubMedQA (2019) are old benchmarks. GPT-4o-mini and Claude 3 Haiku were trained after these benchmarks were published. Contamination could differentially affect methods that better exploit memorized information."
     },
     {
       "flag": "Marginal improvements on some benchmarks",
       "detail": "On PubMedQA (60.31 vs 60.26) and STARK-PRIME (41.75 vs 41.40), improvements over DSPy are within error bars. The strong narrative is driven primarily by HotpotQA results."
     }
   ],
   "cited_papers": [
     {
       "title": "Optimizing generative AI by backpropagating language model feedback",
       "authors": ["Mert Yuksekgonul", "Federico Bianchi", "Joseph Boen"],
       "year": 2025,
       "relevance": "TextGrad — the primary baseline; pioneered textual differentiation for compound AI systems."
     },
     {
       "title": "The shift from models to compound AI systems",
       "authors": ["Matei Zaharia", "Omar Khattab"],
       "year": 2024,
       "relevance": "Defines the Compound AI Systems paradigm that TextResNet optimizes."
     },
     {
       "title": "Demonstrate-search-predict: Composing retrieval and language models for knowledge-intensive NLP",
       "authors": ["Omar Khattab"],
       "year": 2022,
       "arxiv_id": "2212.14024",
       "relevance": "DSPy framework for programmatic prompt compilation, used as a baseline."
     },
     {
       "title": "OPTIMAS: Optimizing compound AI systems with globally aligned local rewards",
       "authors": ["Shuai Wu"],
       "year": 2025,
       "arxiv_id": "2507.03041",
       "relevance": "RL-based compound AI optimization; provides the pipeline configurations used in this paper."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes in multi-agent LLM systems, directly relevant to attribution ambiguity."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn"],
       "year": 2023,
       "relevance": "Verbal self-reflection approach for LLM agents; related optimization paradigm."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "relevance": "Self-refinement via LLM feedback; baseline approach for iterative improvement."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao"],
       "year": 2023,
       "relevance": "Structured reasoning for LLMs; related compound reasoning approach."
     },
     {
       "title": "Automated design of agentic systems",
       "authors": ["Shengran Hu"],
       "year": 2024,
       "arxiv_id": "2408.08435",
       "relevance": "Automated agentic system design, related to compound AI system optimization."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "relevance": "One of the four evaluation benchmarks used in this paper."
     },
     {
       "title": "Are more LLM calls all you need? Towards the scaling properties of compound AI systems",
       "authors": ["Lingjiao Chen"],
       "year": 2024,
       "relevance": "Studies scaling properties of compound AI systems; directly relevant to optimization challenges."
     }
   ]
 }