loop-benchmarking

Charts.tsx (13409B)

---

 import { useState, useMemo } from "react";
 import {
   ComposedChart,
   Bar,
   Scatter,
   XAxis,
   YAxis,
   CartesianGrid,
   Tooltip,
   ResponsiveContainer,
   Cell,
 } from "recharts";
 import type { Run } from "../lib/types";
 import { getModelColor, modelSortOrder } from "../lib/colors";
 import { confidenceInterval } from "../lib/analysis";
 import ModelSelector from "./ModelSelector";
 
 interface ChartsProps {
   runs: Run[];
 }
 
 interface BoxPlotData {
   label: string;
   min: number;
   q1: number;
   median: number;
   q3: number;
   max: number;
   cellCount: number;
   runCount: number;
   scores: number[];
   // Derived fields for recharts stacked bar trick
   base: number;    // invisible bar height = q1
   iqr: number;     // visible box height = q3 - q1
   color: string;
   // 95% confidence interval of the mean
   ciMean: number;
   ciLower: number;
   ciUpper: number;
 }
 
 
 const SMUI = {
   surface0: "hsl(213 16% 12%)",
   surface1: "hsl(217 16% 15.5%)",
   surface2: "hsl(216 15% 19%)",
   border: "hsl(217 17% 28%)",
   muted: "hsl(213 14% 65%)",
   frost1: "hsl(176 25% 65%)",
   frost2: "hsl(193 44% 67%)",
   frost3: "hsl(210 34% 63%)",
   frost4: "hsl(213 32% 52%)",
   green: "hsl(92 28% 65%)",
   yellow: "hsl(40 71% 73%)",
   red: "hsl(355 52% 64%)",
   purple: "hsl(311 24% 63%)",
 };
 
 // MODEL_COLORS imported from ../lib/colors
 
 const TOOLTIP_STYLE = {
   background: SMUI.surface1,
   border: `1px solid ${SMUI.border}`,
   borderRadius: "0",
   fontFamily: "'JetBrains Mono', monospace",
   fontSize: "11px",
   padding: "8px 12px",
 };
 
 interface CellAggregate {
   cell_id: string;
   model: string;
   task: string;
   avgScore: number;
   avgCost: number;
   passRate: number;
   runCount: number;
 }
 
 function aggregateCells(runs: Run[]): CellAggregate[] {
   const byCell: Record<string, {
     model: string;
     task: string;
     scores: number[];
     costs: number[];
     passes: number;
     total: number;
   }> = {};
 
   for (const run of runs) {
     const cellId = run.meta.cell_id;
     if (!byCell[cellId]) {
       byCell[cellId] = {
         model: run.meta.actual_model || run.meta.model,
         task: run.meta.task,
         scores: [],
         costs: [],
         passes: 0,
         total: 0,
       };
     }
 
     byCell[cellId].total++;
     if (run.eval_results?.score != null) {
       byCell[cellId].scores.push(run.eval_results.score);
     }
     if (run.claude_output?.total_cost_usd != null) {
       byCell[cellId].costs.push(run.claude_output.total_cost_usd);
     }
     if (run.eval_results?.functional?.pass) {
       byCell[cellId].passes++;
     }
   }
 
   return Object.entries(byCell).map(([cell_id, data]) => ({
     cell_id,
     model: data.model,
     task: data.task,
     avgScore: data.scores.length > 0
       ? data.scores.reduce((a, b) => a + b, 0) / data.scores.length
       : 0,
     avgCost: data.costs.length > 0
       ? data.costs.reduce((a, b) => a + b, 0) / data.costs.length
       : 0,
     passRate: data.total > 0
       ? data.passes / data.total
       : 0,
     runCount: data.total,
   }));
 }
 
 // MODEL_ORDER imported via modelSortOrder from ../lib/colors
 
 function quantile(sorted: number[], q: number): number {
   if (sorted.length === 0) return 0;
   if (sorted.length === 1) return sorted[0];
   const pos = q * (sorted.length - 1);
   const lo = Math.floor(pos);
   const hi = Math.ceil(pos);
   const frac = pos - lo;
   return sorted[lo] + frac * (sorted[hi] - sorted[lo]);
 }
 
 function computeBoxStats(values: number[]): { min: number; q1: number; median: number; q3: number; max: number } {
   const sorted = [...values].sort((a, b) => a - b);
   return {
     min: sorted.length > 0 ? sorted[0] : 0,
     q1: quantile(sorted, 0.25),
     median: quantile(sorted, 0.5),
     q3: quantile(sorted, 0.75),
     max: sorted.length > 0 ? sorted[sorted.length - 1] : 0,
   };
 }
 
 function aggregateByModel(runs: Run[]): BoxPlotData[] {
   const cells = aggregateCells(runs);
   const byModel: Record<string, CellAggregate[]> = {};
 
   for (const cell of cells) {
     if (!byModel[cell.model]) byModel[cell.model] = [];
     byModel[cell.model].push(cell);
   }
 
   const sortedEntries = Object.entries(byModel).sort(([a], [b]) =>
     modelSortOrder(a) - modelSortOrder(b) || a.localeCompare(b)
   );
 
   return sortedEntries.map(([model, modelCells]) => {
     const scores = modelCells.map((c) => Math.round(c.avgScore * 100));
     const totalRuns = modelCells.reduce((sum, c) => sum + c.runCount, 0);
     const stats = computeBoxStats(scores);
     const baseModel = model;
     const ci = confidenceInterval(scores);
     return {
       label: `${model}|(n=${totalRuns})`,
       ...stats,
       base: stats.q1,
       iqr: stats.q3 - stats.q1,
       cellCount: modelCells.length,
       runCount: totalRuns,
       scores,
       color: getModelColor(baseModel),
       ciMean: ci.mean,
       ciLower: ci.lower,
       ciUpper: ci.upper,
     };
   });
 }
 
 
 // Custom shape: draws a box from q1 to q3 with whiskers from min to max and a median line
 function BoxPlotShape(props: any) {
   const { x, y, width, height, payload } = props as {
     x: number; y: number; width: number; height: number;
     payload: BoxPlotData;
   };
   if (!payload || height === undefined) return null;
 
   const { min, median, max, color, cellCount, ciLower, ciUpper, ciMean } = payload;
   const lowN = cellCount < 3;
   const boxOpacity = lowN ? 0.4 : 1;
   // The bar is rendered from q1 (base) with height iqr (q3-q1).
   // y is the top of the bar (q3 in chart coords), y+height is the bottom (q1).
   const boxTop = y;
   const boxBottom = y + height;
   const boxQ3 = payload.q3;
   const boxQ1 = payload.q1;
   const centerX = x + width / 2;
 
   // Scale: we need to convert data values to pixel positions.
   // We know q1 maps to boxBottom and q3 maps to boxTop.
   const dataToY = (val: number): number => {
     if (boxQ3 === boxQ1) return boxTop;
     return boxTop + ((boxQ3 - val) / (boxQ3 - boxQ1)) * (boxBottom - boxTop);
   };
 
   const minY = dataToY(min);
   const maxY = dataToY(max);
   const medianY = dataToY(median);
   const whiskerHalfW = width * 0.3;
 
   return (
     <g opacity={boxOpacity}>
       {/* Whisker line: min to max */}
       <line x1={centerX} y1={minY} x2={centerX} y2={maxY} stroke={SMUI.muted} strokeWidth={1} />
       {/* Min whisker cap */}
       <line x1={centerX - whiskerHalfW} y1={minY} x2={centerX + whiskerHalfW} y2={minY} stroke={SMUI.muted} strokeWidth={1} />
       {/* Max whisker cap */}
       <line x1={centerX - whiskerHalfW} y1={maxY} x2={centerX + whiskerHalfW} y2={maxY} stroke={SMUI.muted} strokeWidth={1} />
       {/* Box (IQR) -- dashed stroke when low sample size */}
       <rect x={x} y={boxTop} width={width} height={Math.max(height, 1)} fill={color} fillOpacity={0.3} stroke={color} strokeWidth={1} strokeDasharray={lowN ? "4 2" : undefined} />
       {/* Median line */}
       <line x1={x} y1={medianY} x2={x + width} y2={medianY} stroke={color} strokeWidth={2} />
       {/* 95% CI band on the mean */}
       {ciLower !== ciUpper && (() => {
         const ciTopY = dataToY(Math.min(ciUpper, max));
         const ciBotY = dataToY(Math.max(ciLower, min));
         const ciMeanY = dataToY(ciMean);
         const ciHalfW = width * 0.45;
         return (
           <>
             {/* Shaded CI band */}
             <rect
               x={centerX - ciHalfW}
               y={ciTopY}
               width={ciHalfW * 2}
               height={Math.max(ciBotY - ciTopY, 1)}
               fill={color}
               fillOpacity={0.2}
               stroke="none"
             />
             {/* CI vertical line */}
             <line x1={centerX} y1={ciTopY} x2={centerX} y2={ciBotY} stroke={color} strokeWidth={1.5} strokeDasharray="2 2" />
             {/* CI top cap */}
             <line x1={centerX - 4} y1={ciTopY} x2={centerX + 4} y2={ciTopY} stroke={color} strokeWidth={1.5} />
             {/* CI bottom cap */}
             <line x1={centerX - 4} y1={ciBotY} x2={centerX + 4} y2={ciBotY} stroke={color} strokeWidth={1.5} />
             {/* Mean dot */}
             <circle cx={centerX} cy={ciMeanY} r={2.5} fill={color} stroke="none" />
           </>
         );
       })()}
     </g>
   );
 }
 
 
 
 
 // Custom tooltip for model box plot
 function ModelBoxTooltipContent({ active, payload, label }: { active?: boolean; payload?: Array<{ payload: BoxPlotData }>; label?: string }) {
   if (!active || !payload || payload.length === 0) return null;
   const d = payload[0].payload;
   return (
     <div style={TOOLTIP_STYLE}>
       <div style={{ marginBottom: 4, fontWeight: 600 }}>{label?.split("|")[0]}</div>
       <div style={{ marginBottom: 4, color: SMUI.muted, fontSize: 10 }}>n={d.runCount} runs across {d.cellCount} cells</div>
       <div>Max: {d.max}%</div>
       <div>Q3: {Math.round(d.q3)}%</div>
       <div>Median: {Math.round(d.median)}%</div>
       <div>Q1: {Math.round(d.q1)}%</div>
       <div>Min: {d.min}%</div>
       {d.ciLower !== d.ciUpper && (
         <div style={{ marginTop: 4, borderTop: `1px solid ${SMUI.border}`, paddingTop: 4 }}>
           <div style={{ color: SMUI.frost2 }}>Mean: {Math.round(d.ciMean)}%</div>
           <div style={{ color: SMUI.frost2 }}>95% CI: [{Math.round(d.ciLower)}% - {Math.round(d.ciUpper)}%]</div>
         </div>
       )}
     </div>
   );
 }
 
 
 export default function Charts({ runs }: ChartsProps) {
   // Extract unique models sorted consistently
   const allModels = useMemo(() => {
     const models = new Set<string>();
     for (const run of runs) {
       models.add(run.meta.actual_model || run.meta.model);
     }
     return [...models].sort((a, b) => modelSortOrder(a) - modelSortOrder(b) || a.localeCompare(b));
   }, [runs]);
 
   const [selectedModels, setSelectedModels] = useState<Set<string>>(() => new Set(allModels));
 
   if (runs.length === 0) {
     return (
       <div className="card" style={{ textAlign: "center", padding: "40px", color: SMUI.muted }}>
         No data to chart yet.
       </div>
     );
   }
 
   const filteredRuns = runs.filter((r) => selectedModels.has(r.meta.actual_model || r.meta.model));
   const modelData = aggregateByModel(filteredRuns);
 
   const detectableDifference = useMemo(() => {
     const ciWidths = modelData
       .filter((d) => d.ciLower !== d.ciUpper)
       .map((d) => d.ciUpper - d.ciLower);
     if (ciWidths.length === 0) return null;
     return Math.round(Math.max(...ciWidths));
   }, [modelData]);
 
   return (
     <div className="card">
       <div style={{ display: "flex", justifyContent: "space-between", alignItems: "flex-start", marginBottom: "16px", flexWrap: "wrap", gap: "8px" }}>
         <div>
           <h3 style={{ margin: 0 }}>Score Distribution by Model</h3>
           <div style={{ fontSize: "10px", color: "var(--text-muted, hsl(213 14% 65%))", fontFamily: "'JetBrains Mono', monospace", marginTop: "2px" }}>
             (n={filteredRuns.length} runs across {modelData.reduce((sum, d) => sum + d.cellCount, 0)} cells)
           </div>
         </div>
         <ModelSelector
           allModels={allModels}
           selectedModels={selectedModels}
           onChange={setSelectedModels}
         />
       </div>
       <ResponsiveContainer width="100%" height={270}>
         <ComposedChart data={modelData} barCategoryGap="20%">
           <CartesianGrid strokeDasharray="3 3" stroke={SMUI.border} vertical={false} />
           <XAxis
             dataKey="label"
             stroke={SMUI.muted}
             tickLine={false}
             axisLine={{ stroke: SMUI.border }}
             interval={0}
             tick={({ x, y, payload }: any) => {
               const [name, count] = (payload.value as string).split("|");
               return (
                 <g>
                   <text x={x} y={y + 12} textAnchor="middle" fill={SMUI.muted} fontSize={10} fontFamily="'JetBrains Mono', monospace">{name}</text>
                   <text x={x} y={y + 24} textAnchor="middle" fill={SMUI.muted} fontSize={8} fontFamily="'JetBrains Mono', monospace" opacity={0.6}>{count}</text>
                 </g>
               );
             }}
             height={40}
           />
           <YAxis
             stroke={SMUI.muted}
             fontSize={11}
             fontFamily="'JetBrains Mono', monospace"
             domain={[0, 100]}
             tickLine={false}
             axisLine={false}
             yAxisId="score"
           />
           <Tooltip content={<ModelBoxTooltipContent />} cursor={{ fill: "hsl(217 17% 28% / 0.3)" }} />
           {/* Invisible base bar to push the visible box up to q1 */}
           <Bar dataKey="base" stackId="box" fill="transparent" barSize={40} yAxisId="score" />
           {/* Visible IQR box with custom shape for whiskers and median */}
           <Bar dataKey="iqr" stackId="box" barSize={40} yAxisId="score" shape={<BoxPlotShape />}>
             {modelData.map((entry) => (
               <Cell key={entry.label} fill={entry.color} />
             ))}
           </Bar>
           {/* Hidden scatter to keep recharts scale consistent */}
           <Scatter data={[]} dataKey="score" yAxisId="score" fill="transparent" />
         </ComposedChart>
       </ResponsiveContainer>
       {detectableDifference != null && (
         <div style={{
           fontSize: "10px",
           fontFamily: "'JetBrains Mono', monospace",
           color: SMUI.muted,
           marginTop: "4px",
           textAlign: "center",
         }}>
           Detectable difference: differences of ±{detectableDifference}% are statistically significant with current data
         </div>
       )}
     </div>
   );
 }