ai-research-survey

scan.json (23616B)

---

 {
   "paper": {
     "title": "AIME: AI System Optimization via Multiple LLM Evaluators",
     "authors": [
       "Bhrij Patel",
       "Souradip Chakraborty",
       "Wesley A. Suttle",
       "Mengdi Wang",
       "Amrit Singh Bedi",
       "Dinesh Manocha"
     ],
     "year": 2024,
     "venue": "Preprint (Under Review)",
     "arxiv_id": "2410.03131"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/Bridge00/aime (footnote on page 3, repeated on page 6)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: LeetCodeHard (from Shinn et al., 2024) and HumanEval (Chen et al., 2021). These are standard public benchmarks not modified by the authors."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states experiments were run on 'an Apple M1 Pro and macOS 14.5' (Section 4) but does not provide a requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A code repository is linked but the paper itself does not contain a 'Reproducing Results' section or detailed instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in tables (e.g., Table 1: '89.26 ± 2.10', Table 3) and error bars appear in the figures (Figures 2, 4, 5). Results are reported as mean ± std dev over 3 trials."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AIME outperforms Single-Eval but does not report any statistical significance tests (no p-values, t-tests, or similar). Comparisons are based solely on comparing mean values with overlapping standard deviations."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported in context: 'up to 62% higher error detection rate', 'up to 16% higher success rate', and tables provide baseline and AIME values (e.g., SR from 83.70% to 89.26%), allowing readers to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only 39 LeetCodeHard problems and the first 20 HumanEval problems with only 3 trials. No justification is given for why these sample sizes are adequate, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across 3 trials in all tables (Tables 1, 2, 3, 4) and figures show error bars."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are included: Zero-Shot (no optimization), Single-Eval (single evaluator), and Implicit Eval (Reflexion by Shinn et al., 2024). See Section 4.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "TextGrad (Yuksekgonul et al., 2024) and Reflexion (Shinn et al., 2024) are both recent and represent state-of-the-art approaches for iterative text-based optimization."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides ablation studies: varying number of evaluators (1→3→6), role diversity (same role vs. distinct roles), role combinations (Table 1), and evaluation temperature (Tables 2-3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Success Rate (SR), Completion Rate (CR), Error Detection Rate (EDR), and Robustness to Adversarial Evaluator (RAE). See Section 4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. Evaluation of code quality is entirely automated via test case pass/fail and heuristic error detection phrases. Given claims about evaluation thoroughness and explainability (Figure 3), human evaluation of the generated evaluations would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe a held-out test set. The same datasets (LeetCodeHard, HumanEval) appear to be used for all experiments and ablations with no clear dev/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (LeetCodeHard vs. HumanEval), per role combination (Table 1), per temperature setting (Tables 2, 3), and per number of evaluators (Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.2 and Table 1 discuss when AIME fails: using only readability/runtime/redundancy roles degrades performance, and is the only setting where Single-Eval outperforms AIME."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that AIME with only readability/runtime/redundancy roles performs worse than Single-Eval (Table 1, Section 4.3.2), and acknowledges the gap between EDR improvement and downstream SR/CR improvement (Section 4.2 Remark)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 62% higher error detection rate' and 'up to 16% higher success rate' are supported by Tables 2 and 3. The claim about 12% impact from evaluator selection is supported by Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims ('multiple evaluators reduce the suboptimality gap', 'increasing evaluators helps') are supported by: (1) theoretical proof (Theorem 1) under stated assumptions, and (2) controlled ablation studies varying the number of evaluators while holding other factors constant (Section 4.3.1)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'AI System Optimization via Multiple LLM Evaluators' implies general applicability, but results are only on code generation with GPT-4o. The limitations section acknowledges this ('We only empirically study our approach in code generation') but the title and abstract do not bound the claims to code generation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 Remark discusses alternative explanations: the feedback LLM may also detect errors independently of the evaluation, and the EDR measurement may undercount Single-Eval's detection due to needing more detection phrases. This acknowledges that the SR/CR gap may not be solely attributable to evaluation quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We use GPT-4o for all LLM calls' (Section 4) but does not specify a version snapshot (e.g., gpt-4o-2024-05-13). GPT-4o behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation system prompt is provided in Figure 6 (Appendix A.1), including the template and role insertion mechanism. The adversarial prompts are also shown in Figure 6."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports: max output tokens (3600 for Single-Eval, 3600/K for AIME), top_p=0.99, temperature ablated across 0/0.25/0.5/0.75/1, 10 iterations per problem, and other LLM calls use 2000 max tokens with temperature 0."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The scaffolding is described: TextGrad framework with separate evaluation and feedback LLM calls, initial prompt vs. update prompt mechanism, Algorithm 1 specifying the loop structure. See Sections 2 and 3.2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper states it uses 'the entire LeetCodeHard dataset of 39 problems' and 'the first 20 problems of HumanEval' with no modification. Unit tests are withheld from evaluators. The selection criterion for HumanEval (first 20) is stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Conclusion, Limitations, and Further Works' and contains a dedicated 'Limitations and Further Work' subsection discussing multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: single task domain (code generation only), single-agent system (not tested with compound/agentic systems), uniform weighting of evaluators. The remark in Section 4.2 also identifies a specific validity concern about the EDR heuristic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states what was NOT tested: other tasks like molecule optimization or text generation, compound AI systems with multiple agents, non-uniform weighting methods."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (individual run results, generated evaluations, generated code) is not made available. Only aggregate statistics (mean ± std over 3 trials) are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The experimental procedure is described: for each coding problem, generate initial code (same for both methods), run 10 iterations of optimization, record test case pass/fail results. See Section 4."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The data sources are standard public benchmarks (LeetCodeHard, HumanEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: initial code generation → iterative evaluation (single or multiple) → feedback generation → code update → test case evaluation. Algorithm 1 formalizes the process. The EDR heuristic and phrase list are provided in Appendix A.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Co-author Wesley Suttle is affiliated with the U.S. Army Research Laboratory, but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Maryland, U.S. Army Research Laboratory, Princeton University, University of Central Florida."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of any funding disclosure is itself a concern given that one co-author is at a government research lab."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o to generate and evaluate code on HumanEval and LeetCodeHard benchmarks but does not state GPT-4o's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021) and LeetCode problems are widely available online and likely in GPT-4o's training data. The paper does not discuss potential train/test overlap at all."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and LeetCode problems are publicly available. GPT-4o was almost certainly trained after these were published. The paper does not address contamination risk, which is especially relevant since they compare iterative refinement methods — contamination could differentially affect baseline vs. AIME if the model already 'knows' solutions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "AIME uses K separate LLM evaluation calls per iteration (K=6 in the main experiments) vs. 1 for Single-Eval, multiplying API costs. No inference cost, total API spend, or latency comparison is reported despite this being a significant practical consideration."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware ('Apple M1 Pro and macOS 14.5') but does not quantify total computational budget, API costs, or wall-clock time for experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "AIME achieves up to 62% higher error detection rate than single-evaluator approaches on LeetCodeHard.",
       "evidence": "Table 2 shows EDR at temperature 0: Single-Eval 38.06% vs AIME 91.20% on LeetCodeHard, a difference of ~53 percentage points. The 62% figure appears in the abstract and is supported by the range across temperatures (Section 4.1).",
       "supported": "moderate"
     },
     {
       "claim": "AIME-based optimization achieves up to 16% higher success rate than single-evaluator optimization.",
       "evidence": "Table 3 shows SR improvements across temperatures. The largest gap on LeetCodeHard is ~6.3 percentage points (82.96% vs 89.30% at temp 0.5). On HumanEval the gaps are smaller (~1-3 pp). The '16%' figure in the abstract likely refers to a relative improvement, but even so appears to be an optimistic reading of the data.",
       "supported": "moderate"
     },
     {
       "claim": "A linear combination of multiple evaluators can approximate the optimal evaluation policy (Theorem 1).",
       "evidence": "Theorem 1 in Section 3.1 provides a formal proof under the linear additivity assumption on the aggregation function g. The proof shows the suboptimality gap is bounded by the total variation distance between the optimal evaluator and the mixture.",
       "supported": "strong"
     },
     {
       "claim": "The choice of number of evaluators and role combinations impacts success rate by up to 12%.",
       "evidence": "Table 1 shows that AIME with readability/runtime/redundancy roles achieves SR of 77.41% vs 89.26% with all 6 roles, a difference of ~12 percentage points on LeetCodeHard.",
       "supported": "strong"
     },
     {
       "claim": "Increasing evaluator diversity (distinct roles vs. same role) improves performance.",
       "evidence": "Figure 5 (right) shows that 6 distinct-role evaluators outperform 6 same-role (correctness) evaluators in SR, CR, and EDR on LeetCodeHard.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "AIME proposes using multiple role-specific LLM evaluators instead of a single evaluator for iterative AI system optimization, with theoretical justification that mixture-of-evaluators can reduce the suboptimality gap. On LeetCodeHard and HumanEval code generation benchmarks using GPT-4o, AIME with 6 evaluators achieves substantially higher error detection rates (53-62 pp improvement) and moderately higher success rates (3-7 pp improvement) compared to single-evaluator baselines. Ablation studies reveal that both the number and diversity of evaluator roles matter, though the downstream success rate improvement is much smaller than the error detection improvement.",
   "red_flags": [
     {
       "flag": "Very small sample size",
       "detail": "Only 39 LeetCodeHard problems and 20 HumanEval problems with 3 trials. The standard deviations often overlap between AIME and Single-Eval (e.g., SR 83.70±2.28 vs 89.26±2.10), and no significance tests are performed to determine whether differences are statistically meaningful."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "HumanEval (2021) and LeetCode problems are very likely in GPT-4o's training data. The paper does not discuss this risk. If the model already 'knows' solutions, the relative benefit of better evaluation may be confounded."
     },
     {
       "flag": "No cost analysis despite multiplicative API overhead",
       "detail": "AIME with K=6 evaluators makes 6x more evaluation API calls per iteration than Single-Eval. The paper does not report any cost comparison, making practical deployment assessment impossible."
     },
     {
       "flag": "Unspecified model version",
       "detail": "Only 'GPT-4o' is stated without a version/snapshot date. GPT-4o behavior changes across versions, making reproduction uncertain."
     },
     {
       "flag": "EDR heuristic is fragile",
       "detail": "The Error Detection Rate metric relies on keyword matching of specific phrases (Appendix A.2). The authors acknowledge this may undercount Single-Eval's detections (Section 4.2 Remark), potentially inflating AIME's advantage. The 62% gap in EDR translates to only a ~6 pp gap in actual success rate."
     },
     {
       "flag": "Overstated headline numbers",
       "detail": "The abstract's '16% higher success rate' is presented as if it is a percentage-point improvement, but the actual SR improvements in Tables 3 and 4 range from ~1-7 percentage points depending on dataset and temperature. The framing is misleading."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used in this paper and foundational to LLM code generation evaluation."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2024,
       "relevance": "Baseline method for iterative LLM self-improvement; provides the LeetCodeHard dataset used in experiments."
     },
     {
       "title": "TextGrad: Automatic 'differentiation' via text",
       "authors": ["Mert Yuksekgonul", "Federico Bianchi", "Joseph Boen"],
       "year": 2024,
       "arxiv_id": "2406.07496",
       "relevance": "Core framework used to implement AIME; state-of-the-art AI system optimization via text-based gradients."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Foundational work on LLM-as-a-Judge paradigm, directly relevant to evaluation methodology in LLM systems."
     },
     {
       "title": "Replacing judges with juries: Evaluating LLM generations with a panel of diverse models",
       "authors": ["Pat Verga", "Sebastian Hofstatter", "Sophia Althammer"],
       "year": 2024,
       "arxiv_id": "2404.18796",
       "relevance": "Closely related work showing panels of smaller LLM judges can outperform single larger judges, directly relevant to multi-evaluator approaches."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2024,
       "relevance": "Pioneering work on iterative text-based feedback loops for LLM refinement, precursor to AI system optimization."
     },
     {
       "title": "DSPy: Compiling declarative language model calls into state-of-the-art pipelines",
       "authors": ["Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari"],
       "year": 2024,
       "relevance": "Major framework for LLM pipeline optimization, directly relevant to compound AI system design and evaluation."
     },
     {
       "title": "Critique-out-loud reward models",
       "authors": ["Zachary Ankner", "Mansheej Paul", "Brandon Cui"],
       "year": 2024,
       "arxiv_id": "2408.11791",
       "relevance": "Uses LLM-generated critiques to augment reward models, related to multi-criteria evaluation approaches."
     },
     {
       "title": "A survey on evaluating large language models in code generation tasks",
       "authors": ["Liguo Chen", "Qi Guo", "Hongrui Jia"],
       "year": 2024,
       "arxiv_id": "2408.16498",
       "relevance": "Survey of LLM code generation evaluation methods, directly relevant to methodology quality in this domain."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler", "Sandhini Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for the foundation model family used in this paper's experiments."
     }
   ]
 }