ai-research-survey

scan.json (18945B)

---

 {
   "paper": {
     "title": "Conformal Constrained Policy Optimization for Cost-Effective LLM Agents",
     "authors": ["Wenwen Si", "Sooyong Jang", "Insup Lee", "Osbert Bastani"],
     "year": 2025,
     "venue": "arXiv preprint (AAAI 2026 submission)",
     "arxiv_id": "2511.11828"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HotpotQA (Yang et al. 2018) and MMLU (Hendrycks et al. 2021)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions LLaMA-2-7B (8-bit) and GPT-4o but not software dependencies or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Box plots over 100 random evaluation splits are provided in Figures 2-5, showing distribution of coverage and cost."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CCPO outperforms baselines but provides no statistical significance tests (no p-values, t-tests, etc.)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage cost reductions with context, e.g., 'CCPO reduces cost by 12% to 27% compared to this baseline' with absolute cost figures in tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Training set sizes (1000 for HotpotQA, 560 for MMLU) and test set size (200) are stated but not justified. No power analysis or discussion of whether these sizes are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Box plots over 100 random evaluation splits (Figures 2-5) show variance in coverage and cost across splits."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: Random policy, LLM policies (GPT-4o, LLaMA variants), LLM-EXIT, UALA, CPO, CPO batch, CPO online (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: UALA (2024), LLM-EXIT (Lu et al. 2025), SWEET-RL (Zhou et al. 2025), CPO variants. These are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "CPO vs CPO batch vs CPO online vs CCPO effectively ablates the contribution of conformal prediction and online threshold calibration. CCPO is tested with different lambda values (0, 2e-4, 1e-4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are reported: Cost, Coverage, Avg. Len., and Set Size (Section 5.1 Metrics)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates automated QA performance on benchmarks with ground-truth answers. Human evaluation is not relevant to these claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1: 'we use 200 examples for batch calibration and 200 for testing.' Explicit train/calibration/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (HotpotQA, MMLU), by base model (LLaMA-2-7B, LLaMA-3.2-3B), and by alpha level (0.1, 0.2, 0.05) across Tables 2-9."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses CPO failures: 'CPO often fails to find an optimal policy, especially at higher coverage levels' and sensitivity to noisy/limited data causing 'severe divergence during training.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that smaller LLMs as policies 'exhibit limited ability to assess the correctness of responses... often performing comparably to random.' CPO failures at higher coverage are also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 30% cost reduction compared to other cost-aware baselines.' Tables show CCPO achieving 12-27% cost reduction vs CPO batch/online baselines, and larger savings vs LLM methods. The 30% claim is supported by the data."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are mainly ablation-style: comparing CCPO to CPO variants isolates the contribution of conformal prediction integration. The controlled single-variable manipulation (adding/removing conformal components) is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim 'Cost-Effective LLM Agents' and 'a principled and practical framework for deploying LLM agents' but results are only on two QA benchmarks with specific model pairs. The generalization to other agent tasks is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. Section 6 discusses technical design choices (clipping, stochastic policy) but not confounds or alternative explanations for observed improvements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4o' without a snapshot date or API version. LLaMA-2-7B and LLaMA-3.2-3B are specified by size but GPT-4o lacks version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix 8: guide model template (8.1), LLM EXIT template (8.2), and LLM-as-a-Policy template (8.3) with actual text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 Training: learning rate 10^-3, batch size 10, KL constraint delta=0.01, epsilon=0.01, conformal decay xi=0.1, temperature 0.0/1.0 for MMLU, 3-layer NN with 64 hidden units, 1500-2000 training steps."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The orchestration framework is described in detail in Section 3 (LLM Agent Orchestration): base agent generates reasoning trace, guide agent evaluates, policy selects among three actions. The POMDP formulation describes the full workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how HotpotQA and MMLU examples were selected or preprocessed. The paper states training/calibration/test split sizes but not selection criteria or any filtering applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The conclusion is brief with no discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. The broad claims about 'LLM agents' are not bounded to the tested QA setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-example results, training logs) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data sources are described: HotpotQA and MMLU are standard benchmarks. Training/calibration/test split sizes are specified (Section 5.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; uses standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw benchmark examples to final results is not documented. How the 1000/560 training examples were selected from the full datasets is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section: 'This work was supported in part by NIH R01EY037101 and NSF Award CCF-2338777.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from University of Pennsylvania, Department of Computer and Information Science. No product being evaluated is affiliated with their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NIH and NSF, which are independent government agencies with no stake in the outcome of LLM cost optimization research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is used as the guide model and its training data cutoff is not stated. HotpotQA and MMLU are well-known benchmarks that could be in GPT-4o's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o may have seen HotpotQA or MMLU examples during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HotpotQA (2018) and MMLU (2021) were both published well before GPT-4o's training cutoff. No contamination discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost is a primary metric. Tables 2-9 report cumulative API cost in cents for all methods across all experimental configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total GPU hours, training time, or total API spend for the experiments. Training details (1500-2000 steps) are given but not the computational cost."
       }
     }
   },
   "claims": [
     {
       "claim": "CCPO achieves up to 30% cost reduction compared to cost-aware baselines while maintaining target coverage.",
       "evidence": "Tables 2-5 show CCPO reducing cost by 12-27% vs CPO batch/online across HotpotQA and MMLU with different base models, while meeting coverage targets.",
       "supported": "strong"
     },
     {
       "claim": "CCPO provides coverage guarantees via conformal prediction that pointwise methods cannot match.",
       "evidence": "Tables 2-5 show CPO (pointwise) consistently fails to meet coverage targets (e.g., 0.832 vs 0.9 target in Table 2), while CCPO achieves >=0.9 coverage. Theoretical guarantees in Section 4.",
       "supported": "strong"
     },
     {
       "claim": "LLM-based policies using small models perform comparably to random policies.",
       "evidence": "Tables 2-5: LLaMA-2-7B as policy achieves 0.675 coverage vs Random's 0.578 (Table 2), and LLaMA-3.2-3B achieves 0.615 vs Random's 0.61 (Table 3). Section 5.2 discusses this.",
       "supported": "strong"
     },
     {
       "claim": "CCPO delivers more stable training compared to CPO under data challenges.",
       "evidence": "Section 5.2 states this with reference to Appendix 10 training curves, but the training curves are not shown in the main text.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "CCPO combines constrained policy optimization with online conformal prediction to orchestrate cheap and expensive LLM agents for cost-effective question answering. On HotpotQA and MMLU, CCPO reduces API costs by 12-27% compared to CPO-based baselines while maintaining user-specified coverage guarantees. The method formalizes the orchestration problem as a POMDP and provides theoretical coverage guarantees via conformal prediction. Small LLMs used as policy controllers perform poorly, suggesting learned RL policies are more effective than LLM-based decision-making for this orchestration task.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no discussion of limitations, threats to validity, or scope boundaries despite making broad claims about 'cost-effective LLM agents' based on two QA benchmarks."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "GPT-4o is used as the guide model on HotpotQA (2018) and MMLU (2021), both likely in its training data. The coverage guarantees may be inflated if GPT-4o already knows the answers."
     },
     {
       "flag": "No significance tests",
       "detail": "Claims of superiority over baselines rely on point comparisons in tables without statistical tests, despite having 100-split box plot data that could support such tests."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2024,
       "relevance": "Directly addresses LLM cost reduction through cascading and caching, a key baseline for cost-effective LLM deployment."
     },
     {
       "title": "Towards Uncertainty-Aware Language Agent",
       "authors": ["J. Han", "W. Buntine", "E. Shareghi"],
       "year": 2024,
       "relevance": "Proposes uncertainty-aware LLM agents (UALA), a key baseline in this paper for uncertainty-guided model selection."
     },
     {
       "title": "SWEET-RL: Training Multi-Turn LLM Agents on Collaborative Reasoning Tasks",
       "authors": ["Y. Zhou", "S. Jiang", "Y. Tian", "J. Weston", "S. Levine", "S. Sukhbaatar", "X. Li"],
       "year": 2025,
       "arxiv_id": "2503.15478",
       "relevance": "RL-based multi-turn LLM agent training, related approach to training agentic orchestration policies."
     },
     {
       "title": "Reflect, Retry, Reward: Self-Improving LLMs via Reinforcement Learning",
       "authors": ["S. Bensal", "U. Jamil", "C. Bryant", "M. Russak", "K. Kamble", "D. Mozolevskyi", "M. Ali", "W. AlShikh"],
       "year": 2025,
       "arxiv_id": "2505.24726",
       "relevance": "Self-improvement via RL for LLM agents, related work on RL-based agent training."
     },
     {
       "title": "Runaway is Ashamed, But Helpful: On the Early-Exit Behavior of Large Language Model-based Agents in Embodied Environments",
       "authors": ["Q. Lu", "L. Ding", "S. Cao", "X. Liu", "K. Zhang", "J. Zhang", "D. Tao"],
       "year": 2025,
       "relevance": "Early-exit strategies for LLM agents to reduce cost, directly relevant to cost-effective agent deployment."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans", "M. Bosma", "F. Xia", "E. Chi", "Q. V. Le", "D. Zhou"],
       "year": 2022,
       "relevance": "Foundational prompting technique used as part of the base agent reasoning strategy in this work."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu", "N. Du", "I. Shafran", "K. Narasimhan", "Y. Cao"],
       "year": 2023,
       "relevance": "Key agentic reasoning framework combining reasoning with tool use, foundational to LLM agent design."
     },
     {
       "title": "Self-Regulation and Requesting Interventions",
       "authors": ["S. Y. Min", "Y. Wu", "J. Sun", "M. Kaufmann", "F. Tajwar", "Y. Bisk", "R. Salakhutdinov"],
       "year": 2025,
       "arxiv_id": "2502.04576",
       "relevance": "Process reward models for training LLM agent help-seeking policies, related approach to learned orchestration."
     }
   ]
 }