loop-benchmarking

index.ts (11441B)

---

 import { test } from "@playwright/test";
 import { execSync, spawn, type ChildProcess } from "node:child_process";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import * as net from "node:net";
 import type { BotReport } from "./types";
 import { PlaywrightDriver } from "./driver";
 import { runAllTests } from "./bot";
 
 /**
  * Find an available port by briefly binding to port 0.
  */
 async function findFreePort(): Promise<number> {
   return new Promise((resolve, reject) => {
     const server = net.createServer();
     server.listen(0, () => {
       const addr = server.address();
       if (addr && typeof addr === "object") {
         const port = addr.port;
         server.close(() => resolve(port));
       } else {
         server.close(() => reject(new Error("could not determine port")));
       }
     });
     server.on("error", reject);
   });
 }
 
 /**
  * Start a simple HTTP server to serve workspace files.
  */
 async function startServer(workspacePath: string, port: number): Promise<ChildProcess> {
   let serverProc: ChildProcess;
 
   try {
     execSync("npx serve --version", { stdio: "ignore", timeout: 5000 });
     serverProc = spawn("npx", ["serve", "-l", String(port), "-s", "--no-clipboard"], {
       cwd: workspacePath,
       stdio: "ignore",
     });
   } catch {
     serverProc = spawn("python3", ["-m", "http.server", String(port)], {
       cwd: workspacePath,
       stdio: "ignore",
     });
   }
 
   const maxWait = 10000;
   const start = Date.now();
   while (Date.now() - start < maxWait) {
     try {
       await new Promise<void>((resolve, reject) => {
         const socket = net.createConnection({ port, host: "127.0.0.1" }, () => {
           socket.destroy();
           resolve();
         });
         socket.on("error", reject);
         socket.setTimeout(500, () => {
           socket.destroy();
           reject(new Error("timeout"));
         });
       });
       return serverProc;
     } catch {
       await new Promise((r) => setTimeout(r, 200));
     }
   }
 
   throw new Error(`server did not start on port ${port} within ${maxWait}ms`);
 }
 
 test.describe("Tetris Gameplay Bot v2", () => {
   let serverProc: ChildProcess | null = null;
   let serverUrl: string;
 
   test.beforeAll(async () => {
     const workspacePath =
       process.env.WORKSPACE_PATH || process.env.TETRIS_WORKSPACE || process.cwd();
     const port = await findFreePort();
     serverProc = await startServer(workspacePath, port);
     serverUrl = `http://127.0.0.1:${port}`;
   });
 
   test.afterAll(async () => {
     if (serverProc) {
       serverProc.kill("SIGTERM");
       serverProc = null;
     }
   });
 
   test("run gameplay bot", async ({ page }) => {
     test.setTimeout(900_000); // 15-minute total timeout; driver watchdog aborts earlier on inactivity
 
     // Measure page load time
     let loadTimeMs = -1;
     try {
       const loadStart = Date.now();
       await page.goto(serverUrl, { waitUntil: "load", timeout: 10000 });
       loadTimeMs = Date.now() - loadStart;
       await page.goto("about:blank");
     } catch {
       // Load time measurement failed, not critical
     }
 
     // Create the Driver (which gets the Playwright Page)
     const driver = new PlaywrightDriver(page);
 
     // Create the Bot (which gets the Driver) and run everything
     const { testResults, calibration, gameplay, session, survey, competitivePlay, calibrationDrift } =
       await runAllTests(driver, serverUrl);
 
     // Accessibility check
     let a11yIssues: string[] = [];
     try {
       await page.goto(serverUrl, { timeout: 10000 });
       await page.waitForTimeout(1000);
       a11yIssues = await page.evaluate(() => {
         const issues: string[] = [];
         if (!document.title || document.title.trim() === "") issues.push("missing page title");
         if (document.querySelectorAll("h1, h2, h3, [role='heading']").length === 0) issues.push("no headings found");
         document.querySelectorAll("img").forEach((img) => {
           if (!img.alt && !img.getAttribute("aria-label")) issues.push("image without alt text");
         });
         document.querySelectorAll("canvas").forEach((canvas) => {
           if (!canvas.getAttribute("aria-label") && !canvas.getAttribute("role")) issues.push("canvas without aria-label or role");
         });
         const focusable = document.querySelectorAll("button, a, input, [tabindex]");
         if (focusable.length === 0 && document.querySelectorAll("canvas").length === 0) issues.push("no focusable elements");
         const body = window.getComputedStyle(document.body);
         if (body.backgroundColor === body.color) issues.push("text color matches background color");
         return issues;
       });
     } catch {
       // a11y check failed, not critical
     }
 
     const passed = testResults.filter((t) => t.pass).length;
     const skipped = testResults.filter((t) => t.detail.startsWith("skipped:")).length;
     const failed = testResults.filter((t) => !t.pass && !t.detail.startsWith("skipped:")).length;
     const total = testResults.length;
     const scorable = total - skipped;
 
     const totalReads = session.gridReadSuccess + session.gridReadFail;
     const gridSuccessRate = totalReads > 0 ? session.gridReadSuccess / totalReads : 0;
 
     // Clean competitive play result
     let cleanCompetitivePlay = competitivePlay;
     if (cleanCompetitivePlay) {
       const { _ccwResult, _ccwTestDone, _softDropDistinct, _softDropTestDone, ...clean } =
         cleanCompetitivePlay as any;
       cleanCompetitivePlay = clean;
     }
 
     // Build the control_discovery report field from the discovered map
     // (may be null if discovery never ran because the game didn't start).
     let controlDiscoveryReport: Record<string, string> | undefined;
     if (calibration.controlMap) {
       const cm = calibration.controlMap;
       controlDiscoveryReport = {};
       const actionDescriptions: Array<[string, { key: string | null; observation: string; confidence: string }]> = [
         ["move_left", cm.move_left],
         ["move_right", cm.move_right],
         ["soft_drop", cm.soft_drop],
         ["hard_drop", cm.hard_drop],
         ["rotate_cw", cm.rotate_cw],
         ["rotate_ccw", cm.rotate_ccw],
       ];
       for (const [name, mapping] of actionDescriptions) {
         if (mapping.key) {
           controlDiscoveryReport[name] = `${mapping.key}${mapping.observation ? ` (${mapping.observation})` : ""}`;
         } else {
           controlDiscoveryReport[name] = "NOT FOUND";
         }
       }
       // Also report per-key observations so readers can see what every
       // candidate did during discovery.
       for (const [key, obs] of Object.entries(cm.key_observations)) {
         if (obs) controlDiscoveryReport[`key:${key}`] = obs;
       }
     }
 
     const report: BotReport = {
       implementation: {
         renderer: calibration.renderer,
         grid_detected: calibration.gridDetected,
         grid_detected_at: calibration.gridDetectedAt || "initial",
         grid_bounds: calibration.gridBounds,
         controls: calibration.controls as unknown as Record<string, string>,
         control_discovery: controlDiscoveryReport,
         start_mechanism: calibration.startMechanism,
         score_element_found: calibration.scoreElementSelector !== null,
         grid_confidence: calibration.gridConfidence,
         survey,
       },
       tests: testResults.map((t) => ({ name: t.name, pass: t.pass, detail: t.detail })),
       summary: {
         total,
         passed,
         failed,
         skipped,
         score: scorable > 0 ? Math.round((passed / scorable) * 100) / 100 : 0,
       },
       gameplay,
       competitive_play: cleanCompetitivePlay,
       session: {
         frames: session.frames,
         events_count: session.events.length,
         pieces_spawned: session.piecesSpawned,
         pieces_locked: session.piecesLocked,
         lines_cleared: session.linesCleared,
         piece_types_seen: [...session.pieceTypes],
         grid_read_success_rate: Math.round(gridSuccessRate * 100) / 100,
       },
       performance: {
         load_time_ms: loadTimeMs,
       },
       accessibility: {
         issues: a11yIssues,
         issue_count: a11yIssues.length,
         pass: a11yIssues.length === 0,
       },
       calibration_drift: calibrationDrift,
     };
 
     // Write report to file
     const reportPath =
       process.env.REPORT_OUTPUT_PATH ||
       path.join(process.cwd(), "gameplay-bot-report.json");
 
     const reportDir = path.dirname(reportPath);
     if (!fs.existsSync(reportDir)) {
       fs.mkdirSync(reportDir, { recursive: true });
     }
 
     fs.writeFileSync(reportPath, JSON.stringify(report, null, 2), "utf-8");
 
     // Log summary
     console.log("\n=== Gameplay Bot v2 Report ===");
     console.log(`Renderer: ${calibration.renderer}`);
     console.log(`Grid detected: ${calibration.gridDetected} (at: ${calibration.gridDetectedAt})`);
     console.log(`Grid confidence: ${Math.round(calibration.gridConfidence * 100)}%`);
     console.log(`Grid read success rate: ${Math.round(gridSuccessRate * 100)}%`);
     console.log(`Start mechanism: ${calibration.startMechanism}`);
     console.log(`Score element: ${calibration.scoreElementSelector ?? "none"}`);
     if (calibration.controlMap) {
       const cm = calibration.controlMap;
       console.log(
         `Controls: left=${cm.move_left.key ?? "?"}, right=${cm.move_right.key ?? "?"}, ` +
         `rotate=${cm.rotate_cw.key ?? "?"}, hard_drop=${cm.hard_drop.key ?? "?"}, ` +
         `soft_drop=${cm.soft_drop.key ?? "NONE"}`
       );
     }
     console.log(`\nTests: ${passed}/${total} passed, ${skipped} skipped, ${failed} failed`);
     console.log(`Score: ${report.summary.score} (${passed}/${scorable} scorable)`);
     for (const t of testResults) {
       const status = t.detail.startsWith("skipped:") ? "SKIP" : t.pass ? "PASS" : "FAIL";
       console.log(`  ${status} ${t.name}: ${t.detail}`);
     }
     console.log(`\nSession: ${session.frames} frames, ${session.events.length} events`);
     console.log(`  Pieces spawned: ${session.piecesSpawned}, locked: ${session.piecesLocked}`);
     console.log(`  Lines cleared: ${session.linesCleared}`);
     console.log(`  Piece types: [${[...session.pieceTypes].join(", ")}]`);
     console.log(`\nGameplay: ${gameplay.pieces_placed} pieces, ${gameplay.lines_cleared} lines`);
     if (competitivePlay) {
       console.log(`\nCompetitive play: ${competitivePlay.pieces_placed} pieces, ${competitivePlay.total_lines_cleared} lines`);
       console.log(`  Clears: ${competitivePlay.single_clears}x single, ${competitivePlay.double_clears}x double, ${competitivePlay.triple_clears}x triple, ${competitivePlay.tetris_clears}x tetris`);
       console.log(`  Score: ${competitivePlay.score_final}, Level: ${competitivePlay.level_final}`);
       if (competitivePlay.bugs_detected.length > 0) {
         console.log(`  Bugs: [${competitivePlay.bugs_detected.join(", ")}]`);
       }
     }
     console.log(
       `\nCalibration cache: ${calibrationDrift.cacheHits} hits / ${calibrationDrift.cacheMisses} misses ` +
       `(${calibrationDrift.recalibrations} recalibrations)` +
       (calibrationDrift.drifted ? ` -- DRIFTED: [${calibrationDrift.changes.join(", ")}]` : "")
     );
     console.log(`\nSurvey: canvas=${survey.has_canvas}, dom_grid=${survey.has_dom_grid}, overlay=${survey.has_overlay}, clickable=${survey.clickable_elements}`);
     console.log(`Report written to: ${reportPath}`);
     console.log("==============================\n");
   });
 });