ai-research-survey

calibration.json (23833B)

---

 {
   "paper_slug": "agentbased-evaluation-framework-2025",
   "calibrated_by": "opus",
   "calibration_date": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 42,
   "disagreement_count": 8,
   "agreement_rate": 0.84,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet credits the human annotation of the benchmark as 'human evaluation,' but the schema specifies 'human ratings, manual inspection, user studies, expert review of the system's OUTPUTS. The humans must be evaluating what the system produced.' The human experts here scored the LLM-generated code to create ground truth labels for the benchmark -- they were NOT evaluating CodeVisionary's outputs. They evaluated LLM code generation quality, not the evaluation framework's outputs. Manual classification of the benchmark itself does not count per the schema description."
     },
     {
       "category": "evaluation_design",
       "question": "held_out_test_set",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet acknowledges the conflation of tuning and testing ('this does conflate tuning and testing') but still answers true. The schema says 'If unclear whether the reported numbers are on data used for any selection decisions, NO.' RQ3 uses the same 363-sample set for hyperparameter selection (number of judges, rounds), and the final reported results in Tables II and III are on the same data. This is clearly not a held-out test set."
     },
     {
       "category": "evaluation_design",
       "question": "failure_cases_discussed",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet credits the failure analysis of the ICE-Score baseline in Section I as failure case discussion for CodeVisionary. However, the schema asks 'Are failure cases shown or discussed? Look for: error analysis, qualitative examples of failures, discussion of where the approach breaks down. If the paper only shows successes, NO.' The paper discusses failures of the BASELINE (ICE-Score) to motivate CodeVisionary, not failures of CodeVisionary itself. There is no discussion of where CodeVisionary fails or breaks down. The case study in Figure 5 shows a success case."
     },
     {
       "category": "human_studies",
       "question": "pre_registered",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet treats the expert annotation study as having human participants. However, the annotators are providing ground truth labels for a benchmark -- they are expert raters doing professional work, not research participants. The schema says 'NA if no human participants. Mining public repositories or analyzing public data does NOT make participants.' The annotation work is analogous to professional labeling, not a human subjects study. The pre_registered criterion (and most human_studies criteria) are for studies OF human participants, not studies that USE human annotators as tools."
     },
     {
       "category": "human_studies",
       "question": "irb_or_ethics_approval",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Same reasoning as pre_registered. Expert annotators providing benchmark labels are not human research participants. IRB review is required for research ON human subjects, not for using human annotators as part of a data pipeline. This is a benchmark evaluation paper, not a human subjects study."
     },
     {
       "category": "human_studies",
       "question": "demographics_reported",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Same reasoning. The annotators are expert raters creating ground truth labels, not participants in a human study. Demographic reporting requirements apply to studies of human participants, not to studies that employ expert annotators for benchmark construction."
     },
     {
       "category": "human_studies",
       "question": "inclusion_exclusion_criteria",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet treats the expert annotators as human participants and marks applies=true (answer=false since no criteria stated). Opus marks applies=false because expert annotators creating benchmark labels are instruments for data collection, not human research participants. Inclusion/exclusion criteria requirements apply to participant recruitment in human subjects research, not professional annotator selection."
     },
     {
       "category": "human_studies",
       "question": "blinding_described",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet credits the paper's statement that 'All experts remain unaware of the identity of the LLM that generates each response' as blinding in a human subjects study. Opus marks applies=false because this is not a human subjects study -- the annotators are professional raters, not research participants. While blinding annotators to LLM identity is good annotation practice, the human_studies.blinding_described criterion is about blinding participants in experimental studies, which does not apply here."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract provides a GitHub URL: 'The resources of CodeVisionary are available at https://github.com/Eshe0922/CodeVisionary.' This is a working URL provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The constructed benchmark of 363 samples with human annotations is not released. Only the framework code repository is referenced. The paper provides no download link for the benchmark data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Docker containers for code execution and describes tools used, but does not provide a requirements.txt, Dockerfile for the framework itself, or a detailed environment setup section with library versions needed to run CodeVisionary."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository is referenced but the paper itself contains no 'Reproducing Results' section or specific commands to run experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables II, III, and V are point estimates (Pearson, Spearman, Kendall-Tau coefficients) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeVisionary outperforms baselines by comparing numeric coefficients but performs no statistical significance tests (no p-values, t-tests, bootstrap tests, etc.)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with context: 'outperforming the best baseline with average improvements of 0.217, 0.163, and 0.141 in Pearson, Spearman, and Kendall-Tau coefficients.' Both baseline and method values are provided in Table II, giving readers enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 363 samples (121 tasks x 3 LLMs). No power analysis or justification for why this sample size is sufficient is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section V-D states 'we perform multiple trials and take the average as the experimental results' but no standard deviation, IQR, or any spread measure is reported across these trials. Only averaged values appear in result tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: VANILLA (direct LLM prompting), ICE-Score (EACL 2024), and CODEJUDGE (EMNLP 2024)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ICE-Score (EACL 2024) and CODEJUDGE (EMNLP 2024) are both recent contemporary baselines representing the state of the art in LLM-based code evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B (RQ2) presents ablation studies removing the RMCD stage (w/o RMCD), the FSAS stage (w/o FSAS), and individual information types (w/o RT, w/o UI/UX, w/o ST). Results are in Table III."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three correlation metrics are used: Pearson (rp), Spearman (rs), and Kendall-Tau (tau) coefficients."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The human expert annotations create ground truth labels for the benchmark. They evaluate the LLM-generated code quality, NOT CodeVisionary's outputs. Per the schema: 'The humans must be evaluating what the system produced -- manual classification of the benchmark or dataset itself does not count.' No humans evaluated CodeVisionary's evaluation outputs (scores, reports). Evaluation of the system is entirely automated via correlation metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "RQ3 explores hyperparameter choices (number of judges 2-5, rounds 2-5) on the same 363-sample test set that final results are reported on. The schema states 'If unclear whether the reported numbers are on data used for any selection decisions, NO.' The same dataset is used for both hyperparameter selection and evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides per-coding-scenario breakdowns (5 categories) and per-programming-language breakdowns (5 languages), showing Spearman correlation for each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses failure cases of the BASELINES (ICE-Score) in Section I to motivate the work. However, it does not discuss where CodeVisionary itself fails or breaks down. The case study (Figure 5) shows a success case. The schema requires 'discussion of where the approach breaks down.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C tests CodeVisionary on CoNaLa and reports it does not outperform ICE-Score on rp (0.644 vs 0.655). This is an honest negative result. The ablation in Table III shows performance degradation when components are removed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'average improvements of 0.217, 0.163, and 0.141 in Pearson, Spearman, and Kendall-Tau coefficients.' Table II confirms CodeVisionary achieves avg 0.301/0.272/0.241 vs best baseline 0.084/0.109/0.100. The improvements match."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims via ablation studies (e.g., 'The RMCD stage boosts rp, rs, and tau by 27.0%, 32.0%, and 32.4%'). However, the ablation design removes entire multi-component stages, not individual variables. The same dataset is used for hyperparameter selection (RQ3) and final evaluation, creating potential circularity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to be 'the first agent-based evaluation framework for complex code generation' -- a broad framing. It tests on one benchmark (CodeArena hard tasks, 363 samples) and one simpler dataset (CoNaLa). The title and abstract are unbounded; the limitations section acknowledges 'our benchmark may not cover all coding scenarios and programming languages' but does not constrain the claims accordingly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section V-D mentions only benchmark coverage and LLM randomness as threats. The paper does not discuss alternative explanations such as whether improvements stem from compute asymmetry (40 interactions vs. single-call baselines) rather than architectural design, or whether the multi-judge consensus simply averages out noise."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o', 'GPT-3.5-turbo', and 'Claude-3.5-Sonnet' throughout without specific API versions or snapshot dates. References point to product announcement blog posts. Per the schema, 'Marketing names like Gemini-2.5 or GPT-4o without a snapshot date or API version do NOT count.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language and provides one example of clarity evaluation criteria. Full prompt text is not included. The paper says 'remaining aspects and criteria are available in our repository' but the actual prompts sent to GPT-4o for the RMCD and FSAS stages are not reproduced in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D reports: temperature=0.2 for RMCD stage, temperature=0.7 for FSAS stage, max interactions=40, number of judges=3, max negotiation rounds=4."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively describes the agentic scaffolding: Docker container setup, 8 tool types (Dynamic Execution, Static Linter, Unit Tests, Screenshot, Interaction, Web Browsing, General Semantic, Bash Command), the thought/action/observation loop, Execute/Analyze state alternation, multi-judge negotiation protocol with formal definitions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents the benchmark construction pipeline: filter CodeArena to 'hard' tasks, exclude platform-specific tasks (MATLAB, Verilog), generate responses using 3 LLMs, manual scoring by two experts with Kappa>80% and third-expert adjudication. Counts are provided (397 -> 121 tasks, 363 samples)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D is titled 'Threats and Limitations' and provides dedicated discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section V-D mentions only two threats: (1) benchmark may not cover all scenarios and (2) LLM randomness causes variation. These are generic disclaimers. No specific threats like compute asymmetry, hyperparameter tuning on test data, or GPT-4o evaluating GPT-4o-generated code are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section notes 'may not cover all coding scenarios' but does not explicitly state what the results do NOT show. It does not bound results to GPT-4o as evaluator, to 'hard' complexity tasks specifically, or state that results may not generalize to other evaluator models."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 363-sample benchmark with human annotations is not publicly available. Only the framework code repository is referenced. The human-annotated ground truth needed to reproduce correlation results is not accessible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes data collection: filtering CodeArena to hard tasks, excluding platform-specific tasks, generating responses via 3 LLMs, human annotation with inter-rater reliability (Kappa>80%) and adjudication process."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states annotators have 'over five years of expertise in the relevant programming languages' but does not describe how these experts were recruited, selected, or compensated, or whether they could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from CodeArena (397 hard tasks) to final benchmark (121 tasks x 3 LLM responses = 363 samples) is documented with filtering criteria, response generation procedure, and annotation process. Counts are consistent."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section discloses funding: NSFC grants (No. 62472126, 62276075), Guangdong Province Natural Science Foundation (No. 2023A1515011959), and Shenzhen-Hong Kong Jointly Funded Project (No. SGDX20230116091246007)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Xinchen Wang and Ruida Hu from HIT Shenzhen, Pengfei Gao and Chao Peng from ByteDance. A footnote notes 'Work done during an internship at ByteDance.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Chinese government research foundations (NSFC, Guangdong Province, Shenzhen-Hong Kong) with no financial interest in CodeVisionary's performance. ByteDance is an affiliate of some authors but not listed as a funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration regarding patents, equity, or financial interests. Absence of such a statement means this criterion is not met."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is used as the evaluator model but its training data cutoff is not stated. GPT-3.5-turbo, Claude-3.5-Sonnet, and GPT-4o are used for response generation without training cutoff dates. The paper evaluates model-generated code against human labels, so contamination of the evaluation benchmark by the models is relevant."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "CodeArena was published in 2024 (arXiv:2412.05210) and could have been in the training data of GPT-4o. The paper does not discuss whether the tasks or generated code could have appeared in model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeArena tasks were collected from various sources and published in late 2024. GPT-4o's training cutoff is unknown from the paper. No contamination risk assessment is provided for either the benchmark tasks or the evaluation process."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The expert annotators are providing ground truth labels for a benchmark -- they are professional raters, not research participants in a human subjects study. Pre-registration applies to studies OF human participants, not studies that use human annotators as instruments. This is a benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Expert annotators creating benchmark labels are not human research participants. IRB review is for research ON human subjects, not for employing human annotators in a data pipeline. This is a benchmark evaluation paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are expert raters creating ground truth labels, not participants in a human study. Demographic reporting requirements apply to studies of human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning -- expert annotators are instruments for benchmark construction, not human research participants. Inclusion/exclusion criteria requirements are for participant recruitment in human subjects research."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. The expert annotation is an observational scoring task, not an experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. While the paper does note that experts 'remain unaware of the identity of the LLM,' this is good annotation practice, not blinding in a human subjects study context."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Attrition is not applicable to expert annotation of a benchmark."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "CodeVisionary uses GPT-4o with up to 40 interactions in the RMCD stage plus additional FSAS calls. Table IV reports average 6.12 actions per instance, but no API cost, token count, or monetary cost is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, API spend, wall-clock time) is stated for running the 363-sample evaluation. Given the multi-interaction approach with GPT-4o, this is a non-trivial cost that is not quantified."
       }
     }
   }
 }