ai-research-survey

scan.json (24339B)

---

 {
   "paper": {
     "title": "Assessing and Verifying Task Utility in LLM-Powered Applications",
     "authors": [
       "Negar Arabzadeh",
       "Siqing Huo",
       "Nikhil Mehta",
       "Qingyun Wu",
       "Chi Wang",
       "Ahmed Awadallah",
       "Charles L. A. Clarke",
       "Julia Kiseleva"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2405.02178"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we make the data, code and all the logs publicly available at https://bit.ly/3w3yKcS' in the abstract, and references a Git repository in Section 5.1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (MATH and ALFWorld) and states all data and logs are publicly available at the provided URL."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned in the paper. The implementation details (Section 5.1) only mention GPT-4 version and temperature, not the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references a Git repository but does not include step-by-step reproduction instructions in the paper itself. The implementation section (5.1) describes the general setup but not specific commands or scripts to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% confidence intervals on multiple figures (Fig. 3, Fig. 4, Fig. 10, Fig. 11), and explicitly discusses them: 'even with 95% interval confidence on all the success and failed cases' (Section 5.2)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Despite making comparative claims about which solutions perform better (e.g., 'Autogen outperforms ReAct and Vanilla GPT-4'), no formal significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares solutions primarily through bar charts and distributions but does not report formal effect sizes (Cohen's d, percentage improvements with baselines, etc.). Comparisons are qualitative or visual."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 120 math problems and 134 ALFWorld tasks without justifying why these sample sizes are sufficient. No power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports variance through coefficient of variation analysis across 50 runs with different seeds (Section 6.2.1, Fig. 6), and box plots showing distribution spread (Fig. 5). The coefficient of variation is explicitly computed and discussed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates AgentEval on three different solutions: AutoGen, Langchain ReAct, and Vanilla GPT-4 solver (Section 4.1). These serve as comparative baselines for assessing AgentEval's ability to distinguish performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "AutoGen (Wu et al., 2023) and ReAct (Yao et al., 2022) are contemporary and widely-used agent frameworks at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper studies the contribution of different components: task-based vs. solution-based criteria (Section 6.1), criteria stability analysis removing unstable criteria (Section 6.2.1), and the VerifierAgent's role in filtering criteria (Section 6.2.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation dimensions: criteria diversity (Section 6.1), criteria stability via coefficient of variation (Section 6.2.1), discriminative power via adversarial testing (Section 6.2.2), and per-criterion quantified performance (Section 5.2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly acknowledges in Limitations (Section 8.1): 'the absence of human evaluation in our validation process could be viewed as a drawback.' No human evaluation of AgentEval's outputs is performed, despite the framework being about assessing utility that would benefit from human validation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no separation of development and test data. The same 120 math problems and 134 ALFWorld tasks are used throughout all experiments. No held-out set is mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by individual criteria (Clarity, Efficiency, Error Analysis, Completeness) in Fig. 3, by success/failure categories, and by solution type. ALFWorld results are similarly broken down in Fig. 10."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses the 'error analysis' criterion that fails to distinguish success/failure cases (Section 6.2.1, Appendix A.4.2), and discusses criteria that lack discriminative power (Section 6.2.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the 'error analysis' criterion shows 'variable performance' and 'does not consistently predict one group (success or failed) to perform better than the other' (Appendix A.4.2). It recommends 'either modify or eliminate this criterion.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'comprehensive analysis of the effectiveness and robustness of AgentEval' for two datasets, which is supported by the experiments in Sections 5-6. The claims are appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are mostly about the framework's design choices (e.g., solution-based criteria produce more diversity). These are supported by controlled comparisons — e.g., task-based vs. solution-based criteria generation (Section 6.1) and adversarial perturbation experiments (Section 6.2.2) that use controlled manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to address 'LLM-Powered Applications' broadly, but the framework is only tested on math problem solving and ALFWorld household tasks. The Limitations section acknowledges testing was 'limited to specific scenarios' but the title and abstract still overgeneralize. The paper says AgentEval can 'gauge the utility of arbitrary LLM-powered agentic applications' based on only two domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, it does not consider whether GPT-4's known biases as an evaluator might systematically favor certain solution types, or whether the criteria stability results could be due to the model memorizing criteria rather than genuinely generating relevant ones."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states: 'we use GPT-4 version 0613, accessed through Azure OpenAI services' — this is a specific model version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'we further provide all the prompts used in our experiments in our Git repository.' The prompts are made available via the code release."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports 'the temperature of 0' for all experiments. The number of seeds (50) and similarity threshold (τ) values (0.7, 0.85, 1) are also reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-agent scaffolding (CriticAgent, QuantifierAgent, VerifierAgent) is described in detail in Section 3, with the workflow illustrated in Fig. 1. The roles, inputs, and outputs of each agent are specified. The VerifierAgent algorithm is provided in Algorithm 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the data selection process: 120 level-5 problems from MATH dataset (20 from each of 6 categories) following Wu et al. (2023), and 134 test cases from ALFWorld (Section 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 is a dedicated Limitations section that discusses four specific limitations of the work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 discusses specific threats: dependence on output log quality, exclusive use of closed-source GPT-4 limiting generalizability, limited test scenarios, and absence of human evaluation in validation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 explicitly states: 'AgentEval is currently focused on tasks where success is clearly defined and multiple successful solutions may exist.' Section 8.1 notes experiments were 'limited to specific scenarios within math problem solving and household tasks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'we make the data, code and all the logs publicly available' and provides a URL. The underlying datasets (MATH, ALFWorld) are also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the MATH dataset (12,500 problems, level-5 subset of 120 problems from 6 categories) and ALFWorld dataset (134-test set) with specific references to their origins and selection criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks (MATH, ALFWorld)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: CriticAgent generates criteria → QuantifierAgent quantifies each criterion → VerifierAgent validates criteria. The criteria refinement pipeline (50 runs, consolidation, coefficient of variation filtering) is described in Section 6 and Algorithm 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The Acknowledgement section thanks individuals for 'inspiring discussions' but does not mention any grants or funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Waterloo, Microsoft Research, Pennsylvania State University, Purdue University. The footnote notes 'Work done during an internship at Microsoft Research.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Multiple authors are from Microsoft Research, and the paper evaluates AutoGen (a Microsoft product) as one of the solutions, finding it outperforms alternatives. The funder/employer has a stake in the outcome, and this potential conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 (version 0613) to evaluate solutions on MATH and ALFWorld benchmarks, but does not state GPT-4's training data cutoff. Since GPT-4 is used both as the solution LLM and as the evaluator, contamination is a concern."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4 may have seen MATH or ALFWorld problems during training. The MATH dataset was published in 2021, well before GPT-4's training cutoff."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The MATH dataset (Hendrycks et al., 2021) was published before GPT-4's training, meaning GPT-4 may have been trained on these problems. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. The evaluation is entirely automated using LLM agents."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions trying to 'keep the balance between computational costs and analyzing the robustness' (Section 6.1) but does not report actual API costs, token consumption, or wall-clock time for the evaluation process."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs 50 seeds of CriticAgent and QuantifierAgent across 120+ problems using GPT-4, which likely involves substantial API costs. No total computational budget is stated."
       }
     }
   },
   "claims": [
     {
       "claim": "AgentEval can distinguish between successful and failed cases across multiple criteria, even with 95% confidence intervals.",
       "evidence": "Section 5.2 and Fig. 3 show that for most criteria, successful and failed cases are distinguished with 95% confidence intervals on math problems. Similar results for ALFWorld in Fig. 10.",
       "supported": "moderate"
     },
     {
       "claim": "AutoGen outperforms ReAct and Vanilla GPT-4 in terms of accuracy, completeness, and efficiency on math problems.",
       "evidence": "Section 5.2 and Fig. 3 show quantified criteria values where AutoGen bars are higher. However, no significance tests are provided to confirm these differences.",
       "supported": "weak"
     },
     {
       "claim": "Solution-based criteria generation produces more diverse criteria than task-based generation.",
       "evidence": "Section 6.1 and Fig. 4 show that solution-based approaches (especially AutoGen) yield more unique criteria, though this varies by model creativity. The analysis uses semantic similarity filtering at multiple thresholds.",
       "supported": "moderate"
     },
     {
       "claim": "The QuantifierAgent is robust, with mean coefficient of variation around or below 0.5 across criteria.",
       "evidence": "Section 6.2.1 and Fig. 6 show the coefficient of variation stabilizes after approximately 18 seeds. However, whether 0.5 represents acceptable robustness is not formally justified.",
       "supported": "moderate"
     },
     {
       "claim": "AgentEval provides a scalable and cost-effective alternative to human evaluations for assessing LLM-powered applications.",
       "evidence": "The framework is demonstrated on two datasets, but no comparison with actual human evaluation is performed. The Limitations section (8.1) explicitly acknowledges this gap.",
       "supported": "weak"
     },
     {
       "claim": "The VerifierAgent can identify and filter out unreliable criteria, improving the overall assessment quality.",
       "evidence": "Section 6.2.3 describes the VerifierAgent's role, and Algorithm 1 provides the procedure. The 'error analysis' criterion is identified as unstable (Appendix A.4.2). However, no before/after comparison demonstrates quantitative improvement.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AgentEval introduces a three-agent framework (CriticAgent, QuantifierAgent, VerifierAgent) for automatically assessing the utility of LLM-powered applications beyond binary success/failure metrics. The framework demonstrates the ability to distinguish successful from failed cases across multiple criteria with 95% confidence intervals on MATH and ALFWorld benchmarks. The QuantifierAgent shows reasonable robustness (coefficient of variation below 0.5) after approximately 18 seeds, and solution-based criteria generation produces more diverse evaluation criteria than task-based generation. However, the framework lacks human evaluation validation, and the claim of being a 'scalable alternative to human evaluations' is not directly substantiated.",
   "red_flags": [
     {
       "flag": "Conflict of interest: Microsoft authors evaluating Microsoft product",
       "detail": "Multiple authors are from Microsoft Research, and the paper evaluates AutoGen (a Microsoft product) favorably against competitors. AutoGen is found to outperform ReAct and Vanilla GPT-4 across criteria, but this conflict is not acknowledged."
     },
     {
       "flag": "No human evaluation to validate LLM-as-evaluator claims",
       "detail": "The paper proposes LLMs as alternatives to human evaluation but never validates against actual human judgments. The authors acknowledge this limitation but it fundamentally undermines the central claim."
     },
     {
       "flag": "Circular evaluation: GPT-4 evaluates GPT-4-based solutions",
       "detail": "GPT-4 is used both as the underlying model for the solutions being evaluated (AutoGen, Vanilla solver) and as the evaluator (CriticAgent, QuantifierAgent). This circularity could bias results, as GPT-4 may systematically favor its own outputs."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MATH dataset (2021) was published before GPT-4's training cutoff, meaning the model may have memorized these problems. This is particularly problematic since GPT-4 serves as both solver and evaluator."
     },
     {
       "flag": "No cost reporting despite heavy API usage",
       "detail": "The experiments involve 50 runs of CriticAgent, 50 runs of QuantifierAgent on 120+ problems, all using GPT-4. The total API cost is never reported, undermining the 'cost-effective' claim."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Core multi-agent framework used for implementing AgentEval and as one of the evaluated solutions."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Benchmark for evaluating LLMs as agents in interactive environments, directly relevant to agentic AI evaluation methodology."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Key agent framework used as a baseline solution in the paper's evaluation."
     },
     {
       "title": "ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate",
       "authors": ["Chi-Min Chan"],
       "year": 2023,
       "relevance": "Multi-agent debate framework for evaluation, directly relevant to LLM-as-judge methodology."
     },
     {
       "title": "Can Large Language Models Be Trusted for Evaluation? Scalable Meta-Evaluation of LLMs as Evaluators via Agent Debate",
       "authors": ["Steffi Chern"],
       "year": 2024,
       "arxiv_id": "2401.16788",
       "relevance": "Studies trustworthiness of LLMs as evaluators through agent debate, directly relevant to evaluator reliability."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent collaboration framework relevant to agentic AI architecture."
     },
     {
       "title": "Large Language Models Cannot Self-Correct Reasoning Yet",
       "authors": ["Jie Huang"],
       "year": 2023,
       "relevance": "Identifies weaknesses in single LLMs as evaluators, motivating multi-agent evaluation approaches."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Scale Language Model Society",
       "authors": ["Guohao Li"],
       "year": 2023,
       "arxiv_id": "2303.17760",
       "relevance": "Multi-agent framework for exploring LLM collaboration, relevant to agentic AI architectures."
     },
     {
       "title": "Holistic Evaluation of Language Models",
       "authors": ["Percy Liang"],
       "year": 2023,
       "relevance": "Comprehensive LLM evaluation framework directly relevant to evaluation methodology quality."
     },
     {
       "title": "A Survey on Evaluation of Large Language Models",
       "authors": ["Yupeng Chang"],
       "year": 2023,
       "relevance": "Survey of LLM evaluation methods, relevant to understanding the evaluation landscape."
     },
     {
       "title": "Autonomous Evaluation and Refinement of Digital Agents",
       "authors": ["Jiayi Pan"],
       "year": 2024,
       "arxiv_id": "2404.06474",
       "relevance": "Autonomous agent evaluation and refinement, closely related to the AgentEval framework's goals."
     },
     {
       "title": "Aligning Offline Metrics and Human Judgments of Value for Code Generation Models",
       "authors": ["Victor Dibia"],
       "year": 2023,
       "relevance": "Studies alignment between automated metrics and human judgments for LLM code generation, relevant to evaluation methodology."
     }
   ]
 }