loop-benchmarking

compare.astro (6584B)

---

 ---
 import Base from "../layouts/Base.astro";
 import { loadAllRuns, getAxisValues, getTaskNames, AXIS_NAMES, projectRunForIndex } from "../lib/data";
 import type { Run, AxisName } from "../lib/data";
 import VariabilityViolin from "../components/VariabilityViolin";
 
 const runs = loadAllRuns();
 const axisValues = getAxisValues(runs);
 const tasks = getTaskNames(runs);
 
 // Projected slim runs for the client:load VariabilityViolin island.
 const slimRuns = runs.map(projectRunForIndex);
 
 // Build comparison data using cell-based aggregation.
 // A "cell" is a unique configuration (cell_id). Multiple runs share a cell_id
 // when they are repeat trials of the same config. Averaging per-cell first,
 // then aggregating across cells, prevents configs with more repeats from
 // dominating the average.
 
 interface ComparisonRow {
   axis: string;
   value: string;
   cells: number;   // number of unique configs
   runs: number;    // total runs
   avg_score: string;
   score_range: string;  // "68%-80%"
   avg_cost: string;
   cost_range: string;   // "$0.15-$0.22"
   avg_time: string;
 }
 
 interface CellStats {
   avg_score: number | null;
   avg_cost: number | null;
   avg_time: number | null;
   run_count: number;
 }
 
 /** Compute per-cell averages from a list of runs. */
 function getCellStats(runs: Run[]): Map<string, CellStats> {
   const cells = new Map<string, Run[]>();
   for (const run of runs) {
     const id = run.meta.cell_id;
     if (!cells.has(id)) cells.set(id, []);
     cells.get(id)!.push(run);
   }
 
   const result = new Map<string, CellStats>();
   for (const [cellId, cellRuns] of cells) {
     const scores = cellRuns
       .map((r) => r.eval_results?.score)
       .filter((s): s is number => s != null);
     const costs = cellRuns
       .map((r) => r.claude_output?.total_cost_usd)
       .filter((c): c is number => c != null);
     const times = cellRuns
       .map((r) => r.meta.wall_time_seconds)
       .filter((t): t is number => t != null);
 
     const avg = (arr: number[]) =>
       arr.length > 0 ? arr.reduce((a, b) => a + b, 0) / arr.length : null;
 
     result.set(cellId, {
       avg_score: avg(scores),
       avg_cost: avg(costs),
       avg_time: avg(times),
       run_count: cellRuns.length,
     });
   }
   return result;
 }
 
 function formatRange(values: number[], formatter: (v: number) => string): string {
   if (values.length === 0) return "-";
   if (values.length === 1) return formatter(values[0]);
   const min = Math.min(...values);
   const max = Math.max(...values);
   if (min === max) return formatter(min);
   return formatter(min) + "-" + formatter(max);
 }
 
 const comparisons: ComparisonRow[] = [];
 
 const AXIS_LABELS: Record<AxisName, string> = {
   model: "Model",
   effort: "Effort",
   prompt_style: "Prompt Style",
   language: "Language",
   human_language: "Human Language",
   tool_read: "Tool: Read",
   tool_write: "Tool: Write",
   tool_edit: "Tool: Edit",
   tool_glob: "Tool: Glob",
   tool_grep: "Tool: Grep",
   linter: "Linter",
   playwright: "Playwright",
   context_file: "Context File",
   web_search: "Web Search",
   max_budget: "Budget",
   tests_provided: "Tests Provided",
   strategy: "Strategy",
   design_guidance: "Design Guidance",
   architecture: "Architecture",
   error_checking: "Error Checking",
   context_noise: "Context Noise",
   renderer: "Renderer",
   provider: "Provider",
 };
 
 // Pre-compute all cell stats once
 const allCellStats = getCellStats(runs);
 
 for (const axis of AXIS_NAMES) {
   for (const value of axisValues[axis]) {
     const filtered = runs.filter(
       (r: Run) => String(r.meta[axis as keyof typeof r.meta]) === value
     );
 
     // Find the unique cell_ids in these runs and gather their stats
     const cellIds = new Set(filtered.map((r) => r.meta.cell_id));
     const matchingCells: CellStats[] = [];
     for (const id of cellIds) {
       const cs = allCellStats.get(id);
       if (cs) matchingCells.push(cs);
     }
 
     const cellScores = matchingCells
       .map((c) => c.avg_score)
       .filter((s): s is number => s != null);
     const cellCosts = matchingCells
       .map((c) => c.avg_cost)
       .filter((c): c is number => c != null);
     const cellTimes = matchingCells
       .map((c) => c.avg_time)
       .filter((t): t is number => t != null);
 
     const avg = (arr: number[]) =>
       arr.length > 0 ? arr.reduce((a, b) => a + b, 0) / arr.length : null;
 
     const avgScore = avg(cellScores);
     const avgCost = avg(cellCosts);
     const avgTime = avg(cellTimes);
 
     comparisons.push({
       axis: AXIS_LABELS[axis],
       value,
       cells: cellIds.size,
       runs: filtered.length,
       avg_score: avgScore != null ? (avgScore * 100).toFixed(0) + "%" : "-",
       score_range: formatRange(cellScores, (v) => (v * 100).toFixed(0) + "%"),
       avg_cost: avgCost != null ? "$" + avgCost.toFixed(2) : "-",
       cost_range: formatRange(cellCosts, (v) => "$" + v.toFixed(2)),
       avg_time: avgTime != null ? Math.round(avgTime) + "s" : "-",
     });
   }
 }
 ---
 
 <Base title="Compare">
   <h1 style="margin-bottom: 8px;">Compare Configurations</h1>
   <p style="color: var(--text-muted); margin-bottom: 24px; font-size: 0.875rem;">
     Aggregate performance across each axis value.
   </p>
 
   {runs.length === 0 ? (
     <div class="card" style="text-align: center; padding: 40px; color: var(--text-muted);">
       No benchmark results yet. Run the harness to generate data.
     </div>
   ) : (
     <div class="card" style="overflow-x: auto;">
       <table>
         <thead>
           <tr>
             <th>Axis</th>
             <th>Value</th>
             <th>Cells</th>
             <th>Runs</th>
             <th>Avg Score</th>
             <th>Score Range</th>
             <th>Avg Cost</th>
             <th>Cost Range</th>
             <th>Avg Time</th>
           </tr>
         </thead>
         <tbody>
           {comparisons.map((row) => (
             <tr>
               <td style="color: var(--text-muted);">{row.axis}</td>
               <td>
                 <span class="badge badge-neutral">{row.value}</span>
               </td>
               <td>{row.cells}</td>
               <td>{row.runs}</td>
               <td class="score-cell">{row.avg_score}</td>
               <td style="color: var(--text-muted); font-size: 0.85rem;">{row.score_range}</td>
               <td>{row.avg_cost}</td>
               <td style="color: var(--text-muted); font-size: 0.85rem;">{row.cost_range}</td>
               <td>{row.avg_time}</td>
             </tr>
           ))}
         </tbody>
       </table>
     </div>
   )}
 
   <div style="margin-top: 24px;">
     <VariabilityViolin client:load runs={slimRuns} />
   </div>
 </Base>