loop-benchmarking

analysis.ts (10020B)

---

 import type { Run, AxisName } from "./types";
 import { AXIS_NAMES } from "./types";
 
 export interface ConfidenceIntervalResult {
   mean: number;
   ci: number;
   lower: number;
   upper: number;
   n: number;
 }
 
 export function confidenceInterval(
   values: number[],
   confidence = 0.95
 ): ConfidenceIntervalResult {
   const n = values.length;
   if (n < 2) {
     const mean = n === 1 ? values[0] : 0;
     return { mean, ci: 0, lower: mean, upper: mean, n };
   }
   const mean = values.reduce((a, b) => a + b, 0) / n;
   const stdDev = Math.sqrt(
     values.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1)
   );
   // t-value approximation for 95% CI (exact for small n, 1.96 for large n)
   const tValues: Record<number, number> = {
     2: 12.71,
     3: 4.3,
     4: 3.18,
     5: 2.78,
     6: 2.57,
     7: 2.45,
     8: 2.36,
     9: 2.31,
     10: 2.26,
   };
   const t = tValues[n] ?? (n > 30 ? 1.96 : 2.0);
   const ci = t * stdDev / Math.sqrt(n);
   return { mean, ci, lower: mean - ci, upper: mean + ci, n };
 }
 
 export interface Cell {
   cell_id: string;
   runs: Run[];
   meta: Run["meta"]; // from first run
   n: number;
   score: { avg: number; min: number; max: number; range: number };
   cost: { avg: number; min: number; max: number };
   turns: { avg: number; min: number; max: number };
   wall_time: { avg: number; min: number; max: number };
   gameplay: { avg: number; min: number; max: number };
   code_quality: { avg: number; min: number; max: number };
   quality: { avg: number; min: number; max: number };
   structural: { avg: number; min: number; max: number };
   sonarqube: { avg: number; min: number; max: number };
   transcript: { avg: number; min: number; max: number };
 }
 
 export interface EffectEntry {
   value: string;
   mean: number;
   effect: number;
   variance: number;
   n: number;
 }
 
 export interface AxisEffect {
   axis: string;
   spread: number;
   values: EffectEntry[];
 }
 
 export interface InteractionCell {
   mean: number;
   variance: number;
   n: number;
 }
 
 export interface InteractionResult {
   axisA: string;
   axisB: string;
   table: Record<string, Record<string, InteractionCell>>;
   maxInteraction: number;
 }
 
 const SKIP_KEYS = new Set([
   "task",
   "cell_id",
   "run_id",
   "run_number",
   "runs_per_cell",
   "max_budget_usd",
   "timeout_seconds",
   "base_tools",
   "started_at",
   "completed_at",
   "wall_time_seconds",
   "exit_code",
   "short_id",
   "short_cell_id",
   "claude_version",
   "sub_agents",
   "actual_model",
 ]);
 
 type MetricExtractor = (run: Run) => number | null;
 
 const METRICS: Record<string, MetricExtractor> = {
   score: (r) => r.eval_results?.score ?? null,
   cost: (r) => r.claude_output?.total_cost_usd ?? null,
   turns: (r) => r.claude_output?.num_turns ?? null,
   wall_time: (r) => r.meta.wall_time_seconds ?? null,
   gameplay: (r) => (r.eval_results as Record<string, any>)?.gameplay_bot?.score ?? null,
   code_quality: (r) => (r.eval_results as Record<string, any>)?.code_analysis?.score ?? null,
   structural: (r) => r.eval_results?.structural?.score ?? null,
   quality: (r) => r.eval_results?.quality?.score ?? null,
   transcript: (r) => (r.eval_results as Record<string, any>)?.transcript_analysis?.score ?? null,
   sonarqube: (r) => (r.eval_results as Record<string, any>)?.sonarqube?.score ?? null,
 };
 
 function agg(values: number[]): { avg: number; min: number; max: number } {
   if (values.length === 0) return { avg: 0, min: 0, max: 0 };
   const sum = values.reduce((a, b) => a + b, 0);
   return {
     avg: sum / values.length,
     min: Math.min(...values),
     max: Math.max(...values),
   };
 }
 
 export function groupIntoCells(runs: Run[]): Cell[] {
   const byCell = new Map<string, Run[]>();
   for (const run of runs) {
     const id = run.meta.cell_id;
     const list = byCell.get(id);
     if (list) list.push(run);
     else byCell.set(id, [run]);
   }
 
   const cells: Cell[] = [];
   for (const [cell_id, cellRuns] of byCell) {
     const extractVals = (extractor: MetricExtractor): number[] => {
       const vals: number[] = [];
       for (const r of cellRuns) {
         const v = extractor(r);
         if (v !== null) vals.push(v);
       }
       return vals;
     };
 
     const scoreVals = extractVals(METRICS.score);
     const scoreAgg = agg(scoreVals);
 
     cells.push({
       cell_id,
       runs: cellRuns,
       meta: cellRuns[0].meta,
       n: cellRuns.length,
       score: { ...scoreAgg, range: scoreAgg.max - scoreAgg.min },
       cost: agg(extractVals(METRICS.cost)),
       turns: agg(extractVals(METRICS.turns)),
       wall_time: agg(extractVals(METRICS.wall_time)),
       gameplay: agg(extractVals(METRICS.gameplay)),
       code_quality: agg(extractVals(METRICS.code_quality)),
       quality: agg(extractVals(METRICS.quality)),
       structural: agg(extractVals(METRICS.structural)),
       sonarqube: agg(extractVals(METRICS.sonarqube)),
       transcript: agg(extractVals(METRICS.transcript)),
     });
   }
 
   return cells;
 }
 
 export function computeMainEffects(
   runs: Run[],
   metric: string = "score"
 ): AxisEffect[] {
   const extract = METRICS[metric];
   if (!extract) return [];
 
   const cells = groupIntoCells(runs);
 
   // Compute per-cell metric averages and ranges
   const scored: Array<{ meta: Run["meta"]; avg: number; range: number }> = [];
   for (const cell of cells) {
     const vals: number[] = [];
     for (const run of cell.runs) {
       const v = extract(run);
       if (v !== null) vals.push(v);
     }
     if (vals.length === 0) continue;
     const cellAvg = vals.reduce((a, b) => a + b, 0) / vals.length;
     const cellRange = vals.length > 1 ? Math.max(...vals) - Math.min(...vals) : 0;
     scored.push({ meta: cell.meta, avg: cellAvg, range: cellRange });
   }
   if (scored.length === 0) return [];
 
   const grandMean = scored.reduce((s, c) => s + c.avg, 0) / scored.length;
 
   // Find axis keys from meta
   const axisKeys = Object.keys(scored[0].meta).filter(
     (k) => !SKIP_KEYS.has(k)
   );
 
   const effects: AxisEffect[] = [];
 
   for (const axis of axisKeys) {
     const groups: Record<string, { avgs: number[]; ranges: number[] }> = {};
     for (const { meta, avg, range } of scored) {
       const key = String((meta as Record<string, unknown>)[axis] ?? "unknown");
       const g = groups[key] ??= { avgs: [], ranges: [] };
       g.avgs.push(avg);
       g.ranges.push(range);
     }
 
     if (Object.keys(groups).length < 2) continue;
 
     const values: EffectEntry[] = [];
     for (const [val, { avgs, ranges }] of Object.entries(groups)) {
       const mean = avgs.reduce((a, b) => a + b, 0) / avgs.length;
       const variance = ranges.reduce((a, b) => a + b, 0) / ranges.length;
       values.push({
         value: val,
         mean: Math.round(mean * 10000) / 10000,
         effect: Math.round((mean - grandMean) * 10000) / 10000,
         variance: Math.round(variance * 10000) / 10000,
         n: avgs.length,
       });
     }
 
     const means = values.map((v) => v.mean);
     const spread = Math.max(...means) - Math.min(...means);
 
     effects.push({
       axis,
       spread: Math.round(spread * 10000) / 10000,
       values: values.sort((a, b) => b.effect - a.effect),
     });
   }
 
   return effects.sort((a, b) => b.spread - a.spread);
 }
 
 export function computeInteraction(
   runs: Run[],
   axisA: string,
   axisB: string,
   metric: string = "score"
 ): InteractionResult {
   const extract = METRICS[metric];
   if (!extract)
     return { axisA, axisB, table: {}, maxInteraction: 0 };
 
   const cells = groupIntoCells(runs);
 
   // Group cells by (axisA, axisB) combination
   const groups: Record<string, Record<string, { avgs: number[]; ranges: number[] }>> = {};
 
   for (const cell of cells) {
     const vals: number[] = [];
     for (const run of cell.runs) {
       const v = extract(run);
       if (v !== null) vals.push(v);
     }
     if (vals.length === 0) continue;
 
     const cellAvg = vals.reduce((a, b) => a + b, 0) / vals.length;
     const cellRange = vals.length > 1 ? Math.max(...vals) - Math.min(...vals) : 0;
 
     const a = String((cell.meta as Record<string, unknown>)[axisA] ?? "?");
     const b = String((cell.meta as Record<string, unknown>)[axisB] ?? "?");
     const g = ((groups[a] ??= {})[b] ??= { avgs: [], ranges: [] });
     g.avgs.push(cellAvg);
     g.ranges.push(cellRange);
   }
 
   const table: Record<string, Record<string, InteractionCell>> = {};
   const allMeans: number[] = [];
 
   for (const [a, bGroups] of Object.entries(groups)) {
     table[a] = {};
     for (const [b, { avgs, ranges }] of Object.entries(bGroups)) {
       const mean = avgs.reduce((s, v) => s + v, 0) / avgs.length;
       const variance = ranges.reduce((s, v) => s + v, 0) / ranges.length;
       table[a][b] = {
         mean: Math.round(mean * 10000) / 10000,
         variance: Math.round(variance * 10000) / 10000,
         n: avgs.length,
       };
       allMeans.push(mean);
     }
   }
 
   const grandMean =
     allMeans.length > 0
       ? allMeans.reduce((a, b) => a + b, 0) / allMeans.length
       : 0;
 
   // Row and column means
   const aMeans: Record<string, number> = {};
   const bMeans: Record<string, number> = {};
   const bKeys = new Set<string>();
 
   for (const [a, bGroups] of Object.entries(table)) {
     const vals = Object.values(bGroups).map((c) => c.mean);
     aMeans[a] = vals.reduce((s, v) => s + v, 0) / vals.length;
     for (const b of Object.keys(bGroups)) bKeys.add(b);
   }
 
   for (const b of bKeys) {
     const vals: number[] = [];
     for (const a of Object.keys(table)) {
       if (table[a][b]) vals.push(table[a][b].mean);
     }
     bMeans[b] = vals.length > 0 ? vals.reduce((s, v) => s + v, 0) / vals.length : grandMean;
   }
 
   // Max interaction = max deviation from additive model
   let maxInteraction = 0;
   for (const a of Object.keys(table)) {
     for (const b of Object.keys(table[a])) {
       const expected = aMeans[a] + bMeans[b] - grandMean;
       const actual = table[a][b].mean;
       maxInteraction = Math.max(maxInteraction, Math.abs(actual - expected));
     }
   }
 
   return {
     axisA,
     axisB,
     table,
     maxInteraction: Math.round(maxInteraction * 10000) / 10000,
   };
 }