loop-benchmarking

BumpChart.tsx (17657B)

---

 import { useState, useMemo } from "react";
 import {
   LineChart,
   Line,
   XAxis,
   YAxis,
   CartesianGrid,
   Tooltip,
   ResponsiveContainer,
   ReferenceDot,
 } from "recharts";
 import type { Run } from "../lib/types";
 import { AXIS_NAMES, type AxisName } from "../lib/types";
 import { groupIntoCells, type Cell } from "../lib/analysis";
 import { getModelColor } from "../lib/colors";
 
 interface BumpChartProps {
   runs: Run[];
 }
 
 const AXIS_LABELS: Record<AxisName, string> = {
   model: "Model",
   effort: "Effort",
   prompt_style: "Prompt Style",
   language: "Language",
   human_language: "Human Language",
   tool_read: "Read Tool",
   tool_write: "Write Tool",
   tool_edit: "Edit Tool",
   tool_glob: "Glob Tool",
   tool_grep: "Grep Tool",
   linter: "Linter",
   playwright: "Playwright",
   context_file: "Context File",
   web_search: "Web Search",
   max_budget: "Budget",
   tests_provided: "Tests Provided",
   strategy: "Strategy",
   design_guidance: "Design Guidance",
   architecture: "Architecture",
   error_checking: "Error Checking",
   context_noise: "Context Noise",
   renderer: "Renderer",
   provider: "Provider",
 };
 
 // All axes except "model" since we rank by model
 const CONDITION_AXES = AXIS_NAMES.filter((a) => a !== "model");
 
 interface RankedPoint {
   conditionValue: string;
   rank: number;
   avgScore: number;
   model: string;
   n: number;
 }
 
 interface CrossingPoint {
   conditionValue: string;
   x: number;
   rank: number;
   models: [string, string];
 }
 
 function computeRankings(
   runs: Run[],
   axis: AxisName
 ): { ranked: Record<string, RankedPoint[]>; crossings: CrossingPoint[] } {
   // Group runs into cells, then work with cell averages
   const cells = groupIntoCells(runs);
   const scoredCells = cells.filter((c) => c.score.avg > 0);
 
   // Get unique condition values for the selected axis
   const conditionValues = Array.from(
     new Set(scoredCells.map((c) => String(c.meta[axis])))
   ).sort();
 
   // Get unique models
   const models = Array.from(new Set(scoredCells.map((c) => c.meta.model))).sort();
 
   // For each condition value, compute average of cell averages per model, then rank
   const ranked: Record<string, RankedPoint[]> = {};
   for (const model of models) {
     ranked[model] = [];
   }
 
   const prevRanks: Record<string, number> = {};
 
   const crossings: CrossingPoint[] = [];
 
   for (let ci = 0; ci < conditionValues.length; ci++) {
     const cv = conditionValues[ci];
     const cellsForCondition = scoredCells.filter(
       (c) => String(c.meta[axis]) === cv
     );
 
     // Compute average of cell averages per model for this condition
     const modelScores: Array<{
       model: string;
       avgScore: number;
       n: number;
     }> = [];
     for (const model of models) {
       const modelCells = cellsForCondition.filter(
         (c) => c.meta.model === model
       );
 
       if (modelCells.length > 0) {
         const avg = modelCells.reduce((s, c) => s + c.score.avg, 0) / modelCells.length;
         modelScores.push({ model, avgScore: avg, n: modelCells.length });
       }
     }
 
     // Sort by avgScore descending (higher score = rank 1)
     modelScores.sort((a, b) => b.avgScore - a.avgScore);
 
     // Assign ranks
     const currentRanks: Record<string, number> = {};
     for (let i = 0; i < modelScores.length; i++) {
       const ms = modelScores[i];
       const rank = i + 1;
       currentRanks[ms.model] = rank;
       ranked[ms.model].push({
         conditionValue: cv,
         rank,
         avgScore: ms.avgScore,
         model: ms.model,
         n: ms.n,
       });
     }
 
     // Detect crossings: if any two models swapped relative rank order
     if (ci > 0) {
       for (let i = 0; i < models.length; i++) {
         for (let j = i + 1; j < models.length; j++) {
           const m1 = models[i];
           const m2 = models[j];
           const prev1 = prevRanks[m1];
           const prev2 = prevRanks[m2];
           const curr1 = currentRanks[m1];
           const curr2 = currentRanks[m2];
 
           if (
             prev1 !== undefined &&
             prev2 !== undefined &&
             curr1 !== undefined &&
             curr2 !== undefined
           ) {
             // Check if they crossed: relative order changed
             if (
               (prev1 < prev2 && curr1 > curr2) ||
               (prev1 > prev2 && curr1 < curr2)
             ) {
               // Approximate crossing rank as average of the two at the crossing point
               const crossRank = (curr1 + curr2) / 2;
               crossings.push({
                 conditionValue: cv,
                 x: ci,
                 rank: crossRank,
                 models: [m1, m2],
               });
             }
           }
         }
       }
     }
 
     Object.assign(prevRanks, currentRanks);
   }
 
   return { ranked, crossings };
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type DotProps = { cx?: number; cy?: number; payload?: any; stroke?: string };
 
 function makeRankDot(
   model: string,
   color: string,
   lookup: Record<string, Record<string, RankedPoint>>
 ) {
   return function RankDot({ cx, cy, payload }: DotProps) {
     if (cx === undefined || cy === undefined || !payload) return null;
     const point = lookup[model]?.[payload.conditionValue];
     if (!point) return null;
     return (
       <g>
         <circle
           cx={cx}
           cy={cy}
           r={5}
           fill={color}
           stroke="var(--surface-1)"
           strokeWidth={2}
         />
         <text
           x={cx + 10}
           y={cy - 8}
           fill="var(--text)"
           fontSize={10}
           fontFamily="'JetBrains Mono', monospace"
           textAnchor="start"
         >
           {(point.avgScore * 100).toFixed(0)}%
         </text>
       </g>
     );
   };
 }
 
 function CustomTooltipContent({
   active,
   payload,
   lookup,
 }: {
   active?: boolean;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   payload?: Array<{ dataKey?: string; payload?: any; stroke: string }>;
   label?: string;
   lookup: Record<string, Record<string, RankedPoint>>;
 }) {
   if (!active || !payload || payload.length === 0) return null;
 
   const conditionValue = payload[0]?.payload?.conditionValue;
   if (!conditionValue) return null;
 
   // Resolve actual RankedPoint data from lookup
   const resolved = payload
     .filter((entry) => entry.dataKey && lookup[entry.dataKey])
     .map((entry) => ({
       point: lookup[entry.dataKey!]?.[conditionValue],
       stroke: entry.stroke,
     }))
     .filter((r) => r.point);
 
   const sorted = [...resolved].sort(
     (a, b) => a.point!.rank - b.point!.rank
   );
 
   return (
     <div
       style={{
         background: "var(--surface-1)",
         border: "1px solid var(--border)",
         padding: "8px 12px",
         fontFamily: "'JetBrains Mono', monospace",
         fontSize: "11px",
       }}
     >
       <div
         style={{
           color: "var(--text)",
           fontWeight: 600,
           marginBottom: "6px",
         }}
       >
         {conditionValue}
       </div>
       {sorted.map((entry) => (
         <div
           key={entry.point!.model}
           style={{
             display: "flex",
             alignItems: "center",
             gap: "8px",
             marginBottom: "2px",
           }}
         >
           <span
             style={{
               display: "inline-block",
               width: 8,
               height: 8,
               background: entry.stroke,
               flexShrink: 0,
             }}
           />
           <span style={{ color: "var(--text-muted)", width: "16px" }}>
             #{entry.point!.rank}
           </span>
           <span style={{ color: "var(--text)" }}>
             {entry.point!.model}
           </span>
           <span style={{ color: "var(--text-muted)", marginLeft: "auto" }}>
             {(entry.point!.avgScore * 100).toFixed(1)}% ({entry.point!.n} cells)
           </span>
         </div>
       ))}
     </div>
   );
 }
 
 export default function BumpChart({ runs }: BumpChartProps) {
   // Pre-compute which axes are useful: need 2+ condition values AND 2+ models with scores
   const validAxes = useMemo(() => {
     const cells = groupIntoCells(runs);
     const scoredCells = cells.filter((c) => c.score.avg > 0);
     return CONDITION_AXES.filter((axis) => {
       const conditionValues = Array.from(
         new Set(scoredCells.map((c) => String(c.meta[axis])))
       );
       if (conditionValues.length < 2) return false;
       // Check that at least one condition value has 2+ models with scores
       for (const cv of conditionValues) {
         const modelsWithScores = new Set(
           scoredCells.filter((c) => String(c.meta[axis]) === cv).map((c) => c.meta.model)
         );
         if (modelsWithScores.size >= 2) return true;
       }
       return false;
     });
   }, [runs]);
 
   const [selectedAxis, setSelectedAxis] = useState<AxisName>(
     validAxes.includes("prompt_style") ? "prompt_style" : validAxes[0] ?? "prompt_style"
   );
 
   const { ranked, crossings, conditionValues, models } = useMemo(() => {
     const { ranked, crossings } = computeRankings(runs, selectedAxis);
     const conditionValues = Array.from(
       new Set(runs.map((r) => String(r.meta[selectedAxis])))
     ).sort();
     const models = Object.keys(ranked).filter(
       (m) => ranked[m].length > 0
     );
     return { ranked, crossings, conditionValues, models };
   }, [runs, selectedAxis]);
 
   // Build a lookup: model -> conditionValue -> RankedPoint
   const pointLookup = useMemo(() => {
     const lookup: Record<string, Record<string, RankedPoint>> = {};
     for (const model of models) {
       lookup[model] = {};
       for (const point of ranked[model]) {
         lookup[model][point.conditionValue] = point;
       }
     }
     return lookup;
   }, [models, ranked]);
 
   // Build recharts data: one entry per condition value
   const chartData = useMemo(() => {
     return conditionValues.map((cv) => {
       const entry: Record<string, unknown> = { conditionValue: cv };
       for (const model of models) {
         const point = pointLookup[model]?.[cv];
         if (point) {
           entry[model] = point.rank;
         }
       }
       return entry;
     });
   }, [conditionValues, models, pointLookup]);
 
   const maxRank = models.length;
 
   const scoredCells = groupIntoCells(runs).filter((c) => c.score.avg > 0);
 
   if (scoredCells.length === 0) {
     return (
       <div
         className="card"
         style={{
           textAlign: "center",
           padding: "40px",
           color: "var(--text-muted)",
         }}
       >
         No scored cells available for ranking.
       </div>
     );
   }
 
   if (validAxes.length === 0) {
     return (
       <div className="card">
         <h3 style={{ margin: 0 }}>Model Rankings by Condition</h3>
         <div
           style={{
             textAlign: "center",
             padding: "40px",
             color: "var(--text-muted)",
             fontSize: "0.8rem",
           }}
         >
           Not enough data to compare models. Rankings need at least 2 condition
           values where 2 or more models have scored cells.
         </div>
       </div>
     );
   }
 
   return (
     <div className="card">
       <div
         style={{
           display: "flex",
           alignItems: "center",
           justifyContent: "space-between",
           marginBottom: "16px",
           flexWrap: "wrap",
           gap: "12px",
         }}
       >
         <div>
           <h3 style={{ margin: 0 }}>Model Rankings by Condition</h3>
           <p
             style={{
               color: "var(--text-muted)",
               fontSize: "0.75rem",
               margin: "4px 0 0",
             }}
           >
             Rank 1 = best average cell score. Crossings indicate rank swaps.
           </p>
         </div>
         <div style={{ display: "flex", alignItems: "center", gap: "8px" }}>
           <label
             style={{
               fontSize: "0.75rem",
               color: "var(--text-muted)",
             }}
           >
             Condition:
           </label>
           <select
             value={selectedAxis}
             onChange={(e) => setSelectedAxis(e.target.value as AxisName)}
             style={{
               background: "var(--surface-2)",
               color: "var(--text)",
               border: "1px solid var(--border)",
               padding: "4px 8px",
               fontFamily: "'JetBrains Mono', monospace",
               fontSize: "0.75rem",
               borderRadius: "0",
               cursor: "pointer",
             }}
           >
             {validAxes.map((axis) => (
               <option key={axis} value={axis}>
                 {AXIS_LABELS[axis]}
               </option>
             ))}
           </select>
         </div>
       </div>
 
       {conditionValues.length < 2 || models.length < 2 ? (
         <div
           style={{
             textAlign: "center",
             padding: "40px",
             color: "var(--text-muted)",
             fontSize: "0.8rem",
           }}
         >
           {models.length < 2
             ? `Need at least 2 models with scored runs for "${AXIS_LABELS[selectedAxis]}" to show rankings. Currently only: ${models.join(", ") || "none"}`
             : `Need at least 2 values for "${AXIS_LABELS[selectedAxis]}" to show rankings. Currently only: ${conditionValues.join(", ") || "none"}`}
         </div>
       ) : (
         <>
           <ResponsiveContainer width="100%" height={300}>
             <LineChart
               data={chartData}
               margin={{ top: 20, right: 60, bottom: 10, left: 10 }}
             >
               <CartesianGrid
                 strokeDasharray="3 3"
                 stroke="var(--border)"
                 vertical={false}
               />
               <XAxis
                 dataKey="conditionValue"
                 stroke="var(--text-muted)"
                 fontSize={11}
                 fontFamily="'JetBrains Mono', monospace"
                 tickLine={false}
                 axisLine={{ stroke: "var(--border)" }}
               />
               <YAxis
                 domain={[0.5, maxRank + 0.5]}
                 ticks={Array.from({ length: maxRank }, (_, i) => i + 1)}
                 reversed
                 stroke="var(--text-muted)"
                 fontSize={11}
                 fontFamily="'JetBrains Mono', monospace"
                 tickLine={false}
                 axisLine={{ stroke: "var(--border)" }}
                 label={{
                   value: "Rank",
                   angle: -90,
                   position: "insideLeft",
                   fill: "var(--text-muted)",
                   fontSize: 11,
                   fontFamily: "'JetBrains Mono', monospace",
                 }}
                 tickFormatter={(v: number) => `#${v}`}
               />
               <Tooltip
                 content={<CustomTooltipContent lookup={pointLookup} />}
                 cursor={{ stroke: "var(--border)", strokeDasharray: "3 3" }}
               />
               {models.map((model) => (
                 <Line
                   key={model}
                   type="linear"
                   dataKey={model}
                   stroke={getModelColor(model)}
                   strokeWidth={2.5}
                   dot={makeRankDot(
                     model,
                     getModelColor(model),
                     pointLookup
                   )}
                   activeDot={false}
                   name={model}
                   connectNulls
                 />
               ))}
               {crossings.map((crossing, i) => (
                 <ReferenceDot
                   key={`crossing-${i}`}
                   x={crossing.conditionValue}
                   y={crossing.rank}
                   r={10}
                   fill="none"
                   stroke="var(--yellow)"
                   strokeWidth={1.5}
                   strokeDasharray="3 2"
                 />
               ))}
             </LineChart>
           </ResponsiveContainer>
 
           {/* Legend */}
           <div
             style={{
               display: "flex",
               alignItems: "center",
               justifyContent: "center",
               gap: "20px",
               marginTop: "12px",
               flexWrap: "wrap",
             }}
           >
             {models.map((model) => (
               <div
                 key={model}
                 style={{
                   display: "flex",
                   alignItems: "center",
                   gap: "6px",
                   fontSize: "0.75rem",
                   fontFamily: "'JetBrains Mono', monospace",
                 }}
               >
                 <span
                   style={{
                     display: "inline-block",
                     width: 12,
                     height: 3,
                     background: getModelColor(model),
                   }}
                 />
                 <span style={{ color: "var(--text)" }}>{model}</span>
               </div>
             ))}
             {crossings.length > 0 && (
               <div
                 style={{
                   display: "flex",
                   alignItems: "center",
                   gap: "6px",
                   fontSize: "0.75rem",
                   fontFamily: "'JetBrains Mono', monospace",
                 }}
               >
                 <span
                   style={{
                     display: "inline-block",
                     width: 12,
                     height: 12,
                     borderRadius: "50%",
                     border: "1.5px dashed var(--yellow)",
                   }}
                 />
                 <span style={{ color: "var(--text-muted)" }}>
                   rank swap
                 </span>
               </div>
             )}
           </div>
         </>
       )}
     </div>
   );
 }