loop-benchmarking

data.ts (7652B)

---

 /**
  * Data loading module. Server-side only (uses Node.js fs).
  * Types are re-exported from ./types.ts which is client-safe.
  */
 import { createHash } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 
 // Use process.cwd() which in Astro build is the dashboard/ directory.
 const RESULTS_DIR = path.resolve(process.cwd(), "../results");
 
 // Re-export types so existing imports from "./data" still work
 export type { RunMeta, EvalResults, ClaudeOutput, Run, AxisName } from "./types";
 export { AXIS_NAMES } from "./types";
 
 import type { RunMeta, EvalResults, ClaudeOutput, Run, AxisName } from "./types";
 import { AXIS_NAMES } from "./types";
 
 function shortHash(str: string): string {
   return createHash("sha256").update(str).digest("hex").slice(0, 8);
 }
 
 export function loadAllRuns(): Run[] {
   const runsDir = path.join(RESULTS_DIR, "runs");
   if (!fs.existsSync(runsDir)) return [];
 
   const runs: Run[] = [];
 
   for (const runId of fs.readdirSync(runsDir)) {
     const runDir = path.join(runsDir, runId);
     if (!fs.statSync(runDir).isDirectory()) continue;
 
     const metaPath = path.join(runDir, "meta.json");
     if (!fs.existsSync(metaPath)) continue;
 
     try {
       const meta: RunMeta = JSON.parse(fs.readFileSync(metaPath, "utf-8"));
 
       // Normalize old schema to new
       if ((meta as any).sub_agents && !meta.strategy) {
           meta.strategy = (meta as any).sub_agents === "on" ? "use_subagents" : "none";
       }
       if (meta.playwright === "on") {
           (meta as any).playwright = "available";
       }
       // Default new axes for old runs
       meta.tests_provided = meta.tests_provided || "none";
       meta.strategy = meta.strategy || "none";
       meta.design_guidance = meta.design_guidance || "none";
       meta.architecture = meta.architecture || "none";
       meta.error_checking = meta.error_checking || "none";
       meta.context_noise = meta.context_noise || "clean";
       meta.renderer = meta.renderer || "none";
       meta.provider = meta.provider || "anthropic";
       meta.actual_model = meta.actual_model || meta.model;
 
       // Normalize legacy model names to versioned
       const MODEL_RENAME: Record<string, string> = {
         haiku: "haiku-4.5", sonnet: "sonnet-4.6", opus: "opus-4.6",
       };
       if (MODEL_RENAME[meta.model]) meta.model = MODEL_RENAME[meta.model];
       if (MODEL_RENAME[meta.actual_model]) meta.actual_model = MODEL_RENAME[meta.actual_model];
 
       // Compute short IDs if not in meta (backwards compat)
       if (!meta.short_id && meta.run_id) {
         meta.short_id = shortHash(meta.run_id);
       }
       if (!meta.short_cell_id && meta.cell_id) {
         meta.short_cell_id = shortHash(meta.cell_id);
       }
 
       let eval_results: EvalResults | null = null;
       const evalPath = path.join(runDir, "eval_results.json");
       if (fs.existsSync(evalPath)) {
         eval_results = JSON.parse(fs.readFileSync(evalPath, "utf-8"));
       }
 
       let claude_output: ClaudeOutput | null = null;
       const outputPath = path.join(runDir, "claude_output.json");
       if (fs.existsSync(outputPath)) {
         claude_output = JSON.parse(fs.readFileSync(outputPath, "utf-8"));
       }
 
       const has_transcript = fs.existsSync(
         path.join(runDir, "transcript.jsonl")
       );
 
       runs.push({ meta, eval_results, claude_output, has_transcript });
     } catch {
       // Skip corrupted run data
       continue;
     }
   }
 
   return runs.sort((a, b) => a.meta.run_id.localeCompare(b.meta.run_id));
 }
 
 export function loadTranscript(runId: string): string[] {
   const transcriptPath = path.join(
     RESULTS_DIR,
     "runs",
     runId,
     "transcript.jsonl"
   );
   if (!fs.existsSync(transcriptPath)) return [];
   return fs
     .readFileSync(transcriptPath, "utf-8")
     .split("\n")
     .filter((line) => line.trim());
 }
 
 export function getAxisValues(runs: Run[]): Record<AxisName, string[]> {
   const values: Record<string, Set<string>> = {};
   for (const axis of AXIS_NAMES) {
     values[axis] = new Set();
   }
   for (const run of runs) {
     for (const axis of AXIS_NAMES) {
       values[axis].add(String(run.meta[axis]));
     }
   }
   const result: Record<string, string[]> = {};
   for (const axis of AXIS_NAMES) {
     result[axis] = Array.from(values[axis]).sort();
   }
   return result as Record<AxisName, string[]>;
 }
 
 export function getTaskNames(runs: Run[]): string[] {
   return Array.from(new Set(runs.map((r) => r.meta.task))).sort();
 }
 
 export interface AggregateStats {
   count: number;
   avg_score: number | null;
   avg_cost: number | null;
   avg_wall_time: number | null;
   avg_turns: number | null;
   pass_rate: number | null;
 }
 
 export function aggregateRuns(runs: Run[]): AggregateStats {
   if (runs.length === 0) {
     return {
       count: 0,
       avg_score: null,
       avg_cost: null,
       avg_wall_time: null,
       avg_turns: null,
       pass_rate: null,
     };
   }
 
   const scores = runs
     .map((r) => r.eval_results?.score)
     .filter((s): s is number => s !== null && s !== undefined);
 
   const costs = runs
     .map((r) => r.claude_output?.total_cost_usd)
     .filter((c): c is number => c !== undefined && c !== null);
 
   const wallTimes = runs
     .map((r) => r.meta.wall_time_seconds)
     .filter((t): t is number => t !== undefined && t !== null);
 
   const turns = runs
     .map((r) => r.claude_output?.num_turns)
     .filter((t): t is number => t !== undefined && t !== null);
 
   const passes = runs.filter(
     (r) => r.eval_results?.functional?.pass === true
   ).length;
 
   const avg = (arr: number[]) =>
     arr.length > 0 ? arr.reduce((a, b) => a + b, 0) / arr.length : null;
 
   return {
     count: runs.length,
     avg_score: avg(scores),
     avg_cost: avg(costs),
     avg_wall_time: avg(wallTimes),
     avg_turns: avg(turns),
     pass_rate: runs.length > 0 ? passes / runs.length : null,
   };
 }
 
 /**
  * Trim a Run for serialization into an Astro island that only needs
  * summary-level fields. Drops per-test details, full SonarQube payloads,
  * code_analysis/transcript_analysis payloads, etc. -- the bulk of
  * eval_results.json content that bloats the index page HTML.
  *
  * Fields kept are the union of everything the index page islands
  * (Charts, Grid, TopBottomConfigs, StatisticalPowerCard) plus
  * analysis.groupIntoCells actually read.
  */
 export function projectRunForIndex(run: Run): Run {
   const er = run.eval_results as Record<string, any> | null;
   const slimEval: EvalResults | null = er
     ? {
         score: er.score ?? null,
         functional: er.functional
           ? { pass: er.functional.pass, score: er.functional.score }
           : undefined,
         structural: er.structural
           ? { pass: er.structural.pass, score: er.structural.score, checks: [] }
           : undefined,
         quality: er.quality ? { score: er.quality.score } : undefined,
         // Non-interface fields the analysis layer reads via `as any`.
         ...(er.gameplay_bot ? { gameplay_bot: { score: er.gameplay_bot.score } } : {}),
         ...(er.code_analysis ? { code_analysis: { score: er.code_analysis.score } } : {}),
         ...(er.transcript_analysis ? { transcript_analysis: { score: er.transcript_analysis.score } } : {}),
         ...(er.sonarqube ? { sonarqube: { score: er.sonarqube.score } } : {}),
       } as EvalResults
     : null;
 
   const slimOutput: ClaudeOutput | null = run.claude_output
     ? {
         total_cost_usd: run.claude_output.total_cost_usd,
         num_turns: run.claude_output.num_turns,
       }
     : null;
 
   return {
     meta: run.meta,
     eval_results: slimEval,
     claude_output: slimOutput,
     has_transcript: run.has_transcript,
   };
 }