ai-research-survey

calibration.json (20229B)

---

 {
   "paper_slug": "agentasajudge-evaluate-agents-2024",
   "calibration_date": "2026-02-28",
   "calibration_model": "opus",
   "total_questions": 50,
   "agreement_count": 45,
   "disagreement_count": 5,
   "agreement_rate": 0.90,
   "disagreements": [
     {
       "category": "human_studies",
       "question": "pre_registered",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "The three human evaluators are co-authors performing annotation work as part of their research contribution, not external human participants in a human subjects study. Pre-registration requirements apply to studies with recruited human participants, not co-author annotators. The paper's 'Human-as-a-Judge' analysis examines inter-rater reliability of their own team's annotations, which is a standard methodological practice, not a study ON human subjects."
     },
     {
       "category": "human_studies",
       "question": "irb_or_ethics_approval",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "IRB/ethics approval is required for research conducted ON human subjects. The evaluators here are three co-authors performing research tasks in their capacity as researchers. Their disagreement analysis is a methodological reflection, not a study with recruited human participants. Standard CS research practice does not require IRB review for co-author annotation work."
     },
     {
       "category": "human_studies",
       "question": "demographics_reported",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Demographics reporting applies when there are recruited human participants whose characteristics could affect generalizability. The evaluators here are three identified co-authors ('AI experts from the authors') performing a specific methodological role. Their demographic characteristics are not relevant in the same way as external participant demographics in a user study."
     },
     {
       "category": "human_studies",
       "question": "inclusion_exclusion_criteria",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Inclusion/exclusion criteria apply to studies that recruit participants from a population. The evaluators are three self-selected co-authors, not participants recruited from a pool. There is no 'recruitment' in the meaningful sense — they are researchers contributing to their own paper."
     },
     {
       "category": "human_studies",
       "question": "blinding_described",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Blinding applies to experimental studies with human participants where knowledge of conditions could bias outcomes. The evaluators are co-author annotators performing a methodological validation role, not participants in a controlled experiment. While blinding WOULD have strengthened the evaluation design, the question is about applicability to this paper type, and co-author annotation work does not structurally require blinding in the human_studies sense."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/metauto-ai/agent-as-a-judge, listed in the paper header. This is a working URL to the project repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "DevAI dataset released on HuggingFace: https://huggingface.co/devai-benchmark, stated in the paper header. Dataset is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment setup section is provided in the paper. Dependencies are visible in trajectory samples but no formal environment specification is given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. A GitHub link exists but the paper itself contains no 'Reproducing Results' section or commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2-4 are reported as point estimates (e.g., '90.44% alignment rate') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Agent-as-a-Judge outperforms LLM-as-a-Judge based on comparing alignment rate percentages (e.g., 90.44% vs 60.38%) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw alignment rates are reported side by side in Table 3 but no standardized effect sizes (Cohen's d, odds ratios) are computed. The paper reports absolute differences without formal effect size calculation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has only 55 tasks and 365 requirements. No power analysis or justification for why 55 tasks is sufficient to support the generalization claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run experiments. No standard deviation, variance across runs, or spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "LLM-as-a-Judge is compared as a direct baseline (Table 3). Human-as-a-Judge serves as the gold standard. Three developer agents (MetaGPT, GPT-Pilot, OpenHands) are benchmarked."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LLM-as-a-Judge (Zheng et al., 2024) is contemporary. The three developer agents are all current open-source frameworks with >30K GitHub stars. GPT-4o-2024-05-13 is used as the backend."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 and Appendix K present extensive ablation studies with 8 components (ask, graph, read, locate, retrieve, search, planning, memory). Tables 4-7 show progressive component addition results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Alignment Rate, Judge Shift, PR Curves, Requirements Met (independent and dependency-aware), Task Solve Rate, Self-Termination rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three expert human evaluators performed manual evaluation over 86.5 total hours across two rounds including consensus-building (Section 3). This serves as the ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 55 DevAI tasks are used for both ablation studies (component selection) and final performance reporting. No train/test split. The same data used to select the best Agent-as-a-Judge configuration is used to report its final performance."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are broken down by developer agent (MetaGPT, GPT-Pilot, OpenHands) but not by task category, despite the benchmark defining categories (supervised learning, RL, CV, NLP, etc.) in Figure 2."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F discusses qualitative failure modes of developer agents. Section 4.1 discusses Agent-as-a-Judge failure modes (planning instability, memory chain errors). Ablation results show where components hurt performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies (Section 4.3, Appendix K) show that search, planning, and memory components actually hurt performance relative to configurations without them. These negative results are explicitly reported and analyzed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that Agent-as-a-Judge 'dramatically outperforms LLM-as-a-Judge and is as reliable as our human evaluation baseline' are supported by Table 3 showing ~90% alignment for Agent-as-a-Judge vs ~60-70% for LLM-as-a-Judge, with individual human evaluators averaging ~85-89%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Ablation studies make causal claims (e.g., 'adding the graph component increases performance to 75.95%'). The sequential addition design does not isolate component interactions. Ablations are run only on OpenHands, not validated across all three developer agents."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames Agent-as-a-Judge as a general framework ('Evaluate Agents with Agents') but validates it only on 55 AI development code-generation tasks. The abstract and conclusion make broad claims ('concrete step forward for modern agentic systems') without bounding results to this narrow setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for Agent-as-a-Judge's superior performance. The key confound — Agent-as-a-Judge has workspace/file access while LLM-as-a-Judge does not — is partially noted via gray-box vs black-box comparison but not systematically disentangled. Whether a more capable LLM without agentic features would achieve similar results is not explored."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Paper specifies 'gpt-4o-2024-05-13' as the backend LLM (Section 2.3, Table 1, trajectory samples). This is a specific model version with a date identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix L provides actual prompt text for the Agent-as-a-Judge system: system prompt (L.1), locate module (L.2), retrieve module (L.3), ask module (L.4), locate user prompt (L.5). Templates use placeholders ({criteria}, {evidence}) but the DevAI dataset is released, and Appendix M shows sample evidence, enabling reconstruction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for the LLM calls made by Agent-as-a-Judge or the developer agents."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix C describe the Agent-as-a-Judge scaffold in detail: eight modular components (graph, locate, read, search, retrieve, ask, memory, planning) with their roles. Figures 6 and 9 show pipeline diagrams."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E documents the DevAI creation pipeline: manually drafted queries (E.1), setting judging criteria (E.2), building dependency DAGs (E.3), refinement rounds (E.4), dataset analysis (E.5), auxiliary information (E.6). Filtering criteria at each stage are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. Some limitations are mentioned in passing (e.g., 'a perfect Agent-as-a-Judge is not the focus of this proof of concept') but there is no labeled section with substantive limitation discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No systematic discussion of specific threats to validity. The paper briefly notes limitations like the benchmark being small-scale but does not enumerate specific threats to the validity of its conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper evaluates 55 AI development tasks in Python but does not explicitly state what domains, task types, or agent types the results do NOT generalize to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "DevAI dataset released on HuggingFace (https://huggingface.co/devai-benchmark). Appendix E.7 shows the JSON format. Tasks, requirements, and preferences are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E documents the data collection process in detail: expert annotators manually crafted queries targeting AI development tasks, set binary requirements, built dependency DAGs, and performed two rounds of review and refinement."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Appendix H states 'We recruited three AI experts from the authors to perform human evaluation.' The evaluators are co-authors, introducing a conflict of interest. Their selection process and qualifications beyond 'AI experts' are not described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline from task creation to final evaluation is documented: draft queries -> set requirements -> build dependency DAGs -> refinement rounds -> execute with developer agents (1800-second time limit) -> collect workspaces and trajectories."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section discloses funding from KAUST Center of Excellence for Generative AI under award number 5940 and the SDAIA-KAUST Center of Excellence in Data Science and Artificial Intelligence."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly stated: Meta AI and KAUST. The paper uses GPT-4o (OpenAI), not Meta or KAUST products, so no direct product conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "KAUST is an academic institution with no financial stake in Agent-as-a-Judge outperforming LLM-as-a-Judge or in the specific developer agents evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests (patents, equity, startups) is present in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses gpt-4o-2024-05-13 as the backend but does not state its training data cutoff date. The DevAI benchmark is novel but no confirmation that it post-dates the training cutoff is provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether gpt-4o's training data could overlap with tasks similar to those in DevAI. The benchmark is hand-crafted but overlap risk is not formally addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DevAI is a newly created benchmark, likely created after gpt-4o's training cutoff, but this is not explicitly confirmed. No analysis of contamination risk is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The three human evaluators are co-authors performing annotation work as part of their research contribution, not external human participants in a human subjects study. Pre-registration requirements are structurally inapplicable to co-author annotation work."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The evaluators are co-author researchers, not recruited human participants. IRB review applies to research conducted on human subjects, not to co-authors performing their research roles."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Demographics reporting applies to recruited human participants whose characteristics could affect generalizability. Co-author annotators performing a methodological role do not constitute human participants requiring demographic characterization."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Inclusion/exclusion criteria apply to studies recruiting participants from a population. The evaluators are self-selected co-authors contributing to their own paper, not recruited from a participant pool."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "All evaluators assessed all outputs in a repeated-measures design. Randomization of participant assignment is not applicable, and the evaluators are co-authors, not participants in a human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Co-author annotators performing a research methodology role do not constitute human participants in a controlled experiment requiring blinding. While blinding would have strengthened the evaluation, the human_studies blinding criterion is structurally inapplicable to co-author annotation work."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "All three co-author evaluators completed both rounds. Attrition reporting applies to studies with recruited human participants, not co-author annotation work."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 reports Agent-as-a-Judge cost $30.58 in API calls and took 118.43 minutes, vs Human-as-a-Judge ~$1,297.50 and 86.5 hours. Table 1 reports per-agent costs ($1.19, $3.92, $6.38)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3: 'a full evaluation on DevAI with one of these three took around 210.65 USD and 14 hours.' Section 4.4 reports compute costs for all major experimental components."
       }
     }
   }
 }