driver.ts (134928B)
1 // PlaywrightDriver: TetrisDriver implementation using Playwright 2 // "The Eyes and Hands" -- handles ALL webpage interaction 3 4 import type { Page } from "@playwright/test"; 5 import type { 6 Grid, 7 GridBounds, 8 RendererType, 9 Controls, 10 ControlMap, 11 ControlMapping, 12 GameAction, 13 StartMechanism, 14 StartCandidate, 15 TryStartResult, 16 SurveyData, 17 GameLandmarks, 18 PieceType, 19 DriverCalibration, 20 CalibrationDrift, 21 GridSnapshot, 22 TetrisDriver, 23 } from "./types"; 24 25 const GRID_ROWS = 20; 26 const GRID_COLS = 10; 27 28 export class InactivityAbortError extends Error { 29 constructor(message: string) { 30 super(message); 31 this.name = "InactivityAbortError"; 32 } 33 } 34 35 const DEFAULT_CONTROLS: Controls = { 36 left: "ArrowLeft", 37 right: "ArrowRight", 38 down: "ArrowDown", 39 rotate: "ArrowUp", 40 drop: "Space", 41 }; 42 43 /** 44 * Candidate keys to try for each abstract game action, in priority order. 45 * The discovery loop tries these from top to bottom until one matches the 46 * expected behaviour (e.g. moves piece 1 column left). 47 */ 48 const CONTROL_CANDIDATES: Record<GameAction, string[]> = { 49 move_left: ["ArrowLeft", "a", "h"], 50 move_right: ["ArrowRight", "d", "l"], 51 // ArrowDown LAST: some games treat it as hard_drop, so we try alternatives 52 // first and fall through. This is the key insight of the discovery system. 53 soft_drop: ["s", "ArrowDown"], 54 // ArrowDown also gets tried as hard_drop because some games bind the down 55 // arrow to hard drop instead of soft drop. Order: conventional hard-drop 56 // keys first, then the ambiguous ArrowDown last. 57 hard_drop: ["Space", "Enter", "ArrowUp", "ArrowDown"], 58 rotate_cw: ["ArrowUp", "x", "w"], 59 rotate_ccw: ["z", "Control"], 60 // pause/hold are not currently discovered; we keep these entries for type 61 // completeness but they remain at default "not_found". 62 pause: ["p", "Escape"], 63 hold: ["c", "Shift"], 64 }; 65 66 function emptyControlMap(): ControlMap { 67 const notFound = (): ControlMapping => ({ 68 key: null, 69 confidence: "not_found", 70 observation: "", 71 }); 72 return { 73 move_left: notFound(), 74 move_right: notFound(), 75 soft_drop: notFound(), 76 hard_drop: notFound(), 77 rotate_cw: notFound(), 78 rotate_ccw: notFound(), 79 key_observations: {}, 80 }; 81 } 82 83 // --------------------------------------------------------------------------- 84 // GridSnapshot factory 85 // --------------------------------------------------------------------------- 86 87 function makeSnapshot( 88 grid: Grid | null, 89 settledGrid?: Grid | null 90 ): GridSnapshot { 91 const filledCount = grid 92 ? grid.reduce((s, row) => s + row.filter(Boolean).length, 0) 93 : 0; 94 95 const activePieceCells = detectActivePieceCells(grid, settledGrid ?? null); 96 const activePieceType = activePieceCells ? identifyPieceType(activePieceCells) : null; 97 const completeRows = grid ? countCompleteRows(grid) : 0; 98 99 return { 100 grid, 101 filledCount, 102 filledInBottom(rows: number): number { 103 if (!grid) return 0; 104 let count = 0; 105 const start = Math.max(0, grid.length - rows); 106 for (let r = start; r < grid.length; r++) { 107 for (let c = 0; c < grid[r].length; c++) { 108 if (grid[r][c]) count++; 109 } 110 } 111 return count; 112 }, 113 hasFilledInTop(rows: number): boolean { 114 if (!grid) return false; 115 for (let r = 0; r < Math.min(rows, grid.length); r++) { 116 for (let c = 0; c < grid[r].length; c++) { 117 if (grid[r][c]) return true; 118 } 119 } 120 return false; 121 }, 122 completeRows, 123 activePieceCells, 124 activePieceType, 125 }; 126 } 127 128 // --------------------------------------------------------------------------- 129 // Pure grid utility functions 130 // --------------------------------------------------------------------------- 131 132 function countCompleteRows(grid: Grid): number { 133 let count = 0; 134 for (const row of grid) { 135 if (row.every(Boolean)) count++; 136 } 137 return count; 138 } 139 140 function detectActivePieceCells( 141 current: Grid | null, 142 settled: Grid | null 143 ): [number, number][] | null { 144 if (!current) return null; 145 146 if (settled && settled.length === current.length) { 147 const cells: [number, number][] = []; 148 for (let row = 0; row < current.length; row++) { 149 for (let col = 0; col < current[row].length; col++) { 150 if (current[row][col] && !settled[row][col]) { 151 cells.push([row, col]); 152 } 153 } 154 } 155 if (cells.length < 3 || cells.length > 5) return null; 156 return cells; 157 } 158 159 // Fallback: find a connected 3-5 cell cluster in the top 4 rows. 160 // (A fresh tetromino spawns at the top and forms a single connected 161 // component; filled cells elsewhere are usually UI chrome or false 162 // positives from canvas reading.) 163 const topRows = Math.min(4, current.length); 164 const filled: [number, number][] = []; 165 for (let row = 0; row < topRows; row++) { 166 for (let col = 0; col < current[row].length; col++) { 167 if (current[row][col]) filled.push([row, col]); 168 } 169 } 170 if (filled.length < 3) return null; 171 172 // BFS to find connected components (4-connectivity) 173 const key = (r: number, c: number) => `${r},${c}`; 174 const filledSet = new Set(filled.map(([r, c]) => key(r, c))); 175 const seen = new Set<string>(); 176 const components: [number, number][][] = []; 177 178 for (const [r0, c0] of filled) { 179 if (seen.has(key(r0, c0))) continue; 180 const stack: [number, number][] = [[r0, c0]]; 181 const comp: [number, number][] = []; 182 while (stack.length > 0) { 183 const [r, c] = stack.pop()!; 184 const k = key(r, c); 185 if (seen.has(k)) continue; 186 seen.add(k); 187 comp.push([r, c]); 188 for (const [dr, dc] of [[-1, 0], [1, 0], [0, -1], [0, 1]] as const) { 189 const nr = r + dr, nc = c + dc; 190 if (nr >= 0 && nr < topRows && nc >= 0 && current[nr] && nc < current[nr].length) { 191 if (filledSet.has(key(nr, nc)) && !seen.has(key(nr, nc))) { 192 stack.push([nr, nc]); 193 } 194 } 195 } 196 } 197 components.push(comp); 198 } 199 200 // Pick the first component sized 3-5 (a tetromino). If several match, 201 // prefer the one closest to the center column (spawn position). 202 const tetrominoCandidates = components.filter((c) => c.length >= 3 && c.length <= 5); 203 if (tetrominoCandidates.length === 0) return null; 204 205 tetrominoCandidates.sort((a, b) => { 206 const avgColA = a.reduce((s, [, c]) => s + c, 0) / a.length; 207 const avgColB = b.reduce((s, [, c]) => s + c, 0) / b.length; 208 const centerDistA = Math.abs(avgColA - 4.5); 209 const centerDistB = Math.abs(avgColB - 4.5); 210 return centerDistA - centerDistB; 211 }); 212 213 return tetrominoCandidates[0]; 214 } 215 216 function identifyPieceType(cells: [number, number][]): PieceType { 217 if (cells.length !== 4) return "unknown"; 218 219 const minRow = Math.min(...cells.map(([r]) => r)); 220 const maxRow = Math.max(...cells.map(([r]) => r)); 221 const minCol = Math.min(...cells.map(([, c]) => c)); 222 const maxCol = Math.max(...cells.map(([, c]) => c)); 223 const w = maxCol - minCol + 1; 224 const h = maxRow - minRow + 1; 225 226 const norm = cells.map(([r, c]) => [r - minRow, c - minCol] as [number, number]); 227 const key = norm 228 .sort((a, b) => a[0] - b[0] || a[1] - b[1]) 229 .map(([r, c]) => `${r},${c}`) 230 .join("|"); 231 232 if (w === 4 && h === 1) return "I"; 233 if (w === 1 && h === 4) return "I"; 234 if (w === 2 && h === 2) return "O"; 235 236 const tPatterns = [ 237 "0,0|0,1|0,2|1,1", "0,0|1,0|1,1|2,0", "0,1|1,0|1,1|1,2", 238 "0,0|0,1|1,0|2,0", "0,1|1,0|1,1|2,1", "0,0|0,1|1,1|2,1", 239 ]; 240 if (tPatterns.includes(key)) return "T"; 241 242 const sPatterns = ["0,1|0,2|1,0|1,1", "0,0|1,0|1,1|2,1"]; 243 if (sPatterns.includes(key)) return "S"; 244 245 const zPatterns = ["0,0|0,1|1,1|1,2", "0,1|1,0|1,1|2,0"]; 246 if (zPatterns.includes(key)) return "Z"; 247 248 const jPatterns = [ 249 "0,0|1,0|1,1|1,2", "0,0|0,1|1,0|2,0", "0,0|0,1|0,2|1,2", 250 "0,0|1,0|2,0|2,1", "0,1|1,1|2,0|2,1", 251 ]; 252 if (jPatterns.includes(key)) return "J"; 253 254 const lPatterns = [ 255 "0,2|1,0|1,1|1,2", "0,0|1,0|2,0|2,1", "0,0|0,1|0,2|1,0", 256 "0,0|0,1|1,1|2,1", 257 ]; 258 if (lPatterns.includes(key)) return "L"; 259 260 return "unknown"; 261 } 262 263 // --------------------------------------------------------------------------- 264 // Calibration cache helpers 265 // --------------------------------------------------------------------------- 266 267 function cloneCalibration(cal: DriverCalibration): DriverCalibration { 268 const copy: DriverCalibration = { 269 renderer: cal.renderer, 270 gridDetected: cal.gridDetected, 271 gridBounds: cal.gridBounds ? { ...cal.gridBounds } : null, 272 cellWidth: cal.cellWidth, 273 cellHeight: cal.cellHeight, 274 controls: { ...cal.controls }, 275 controlMap: cal.controlMap ? cloneControlMap(cal.controlMap) : cal.controlMap ?? null, 276 startMechanism: cal.startMechanism, 277 scoreElementSelector: cal.scoreElementSelector, 278 levelElementSelector: cal.levelElementSelector, 279 backgroundColor: cal.backgroundColor ? [...cal.backgroundColor] as [number, number, number] : null, 280 consoleErrors: [...cal.consoleErrors], 281 gridConfidence: cal.gridConfidence, 282 gridDetectedAt: cal.gridDetectedAt, 283 }; 284 if (cal.startButton) { 285 copy.startButton = { 286 selector: cal.startButton.selector, 287 text: cal.startButton.text, 288 disappeared: cal.startButton.disappeared, 289 position: { ...cal.startButton.position }, 290 }; 291 } 292 return copy; 293 } 294 295 function cloneControlMap(map: ControlMap): ControlMap { 296 return { 297 move_left: { ...map.move_left }, 298 move_right: { ...map.move_right }, 299 soft_drop: { ...map.soft_drop }, 300 hard_drop: { ...map.hard_drop }, 301 rotate_cw: { ...map.rotate_cw }, 302 rotate_ccw: { ...map.rotate_ccw }, 303 key_observations: { ...map.key_observations }, 304 }; 305 } 306 307 function gridBoundsSimilar(a: GridBounds, b: GridBounds): boolean { 308 // Tolerate rendering jitter but flag anything beyond ~10% size change. 309 const tol = Math.max(20, Math.min(a.width, b.width) * 0.15); 310 return ( 311 Math.abs(a.x - b.x) < tol && 312 Math.abs(a.y - b.y) < tol && 313 Math.abs(a.width - b.width) < tol && 314 Math.abs(a.height - b.height) < tol 315 ); 316 } 317 318 /** 319 * Returns a list of field names that differ between the baseline calibration 320 * and a fresh one. Empty list means no drift detected. 321 */ 322 function diffCalibrations(baseline: DriverCalibration, fresh: DriverCalibration): string[] { 323 const changes: string[] = []; 324 325 if (baseline.startMechanism !== fresh.startMechanism) { 326 changes.push("start_mechanism"); 327 } 328 const baseSel = baseline.startButton?.selector ?? null; 329 const freshSel = fresh.startButton?.selector ?? null; 330 if (baseSel !== freshSel) changes.push("start_button_selector"); 331 332 if (baseline.renderer !== fresh.renderer) changes.push("renderer"); 333 334 if (!!baseline.gridBounds !== !!fresh.gridBounds) { 335 changes.push("grid_bounds"); 336 } else if (baseline.gridBounds && fresh.gridBounds) { 337 if (!gridBoundsSimilar(baseline.gridBounds, fresh.gridBounds)) { 338 changes.push("grid_bounds"); 339 } 340 } 341 342 const bc = baseline.controls; 343 const fc = fresh.controls; 344 if (bc.left !== fc.left || bc.right !== fc.right || bc.down !== fc.down || 345 bc.rotate !== fc.rotate || bc.drop !== fc.drop) { 346 changes.push("controls"); 347 } 348 349 if (baseline.scoreElementSelector !== fresh.scoreElementSelector) { 350 changes.push("score_element"); 351 } 352 if (baseline.levelElementSelector !== fresh.levelElementSelector) { 353 changes.push("level_element"); 354 } 355 356 return changes; 357 } 358 359 // --------------------------------------------------------------------------- 360 // PlaywrightDriver 361 // --------------------------------------------------------------------------- 362 363 export class PlaywrightDriver implements TetrisDriver { 364 private page: Page; 365 private cal: DriverCalibration | null = null; 366 // First successful calibration, used as the cache baseline across reloads. 367 private firstCal: DriverCalibration | null = null; 368 // Candidate confirmed by the bot's verification bridge. When set, calibrate() 369 // replays this candidate instead of rediscovering the start mechanism. 370 private confirmedCandidate: StartCandidate | null = null; 371 // Set by the bot when bridge verification definitively failed -- the legacy 372 // detectStartMechanism() fallback must NOT run and override the bot's verdict. 373 private startRejected: boolean = false; 374 // Result of discoverControls(). Persists across calibration cache hits so 375 // downstream phases reuse the discovered mapping without re-running the 376 // expensive discovery loop. null = not yet discovered. 377 private discoveredControls: ControlMap | null = null; 378 // Cumulative drift info across the session. 379 private drift: CalibrationDrift = { 380 drifted: false, 381 changes: [], 382 recalibrations: 0, 383 cacheHits: 0, 384 cacheMisses: 0, 385 }; 386 private consoleErrors: string[] = []; 387 private log = (msg: string) => console.log(`[driver] ${msg}`); 388 389 // Inactivity watchdog: set by armInactivityWatchdog() once the game has 390 // started and we expect regular grid reads. If more than 391 // INACTIVITY_TIMEOUT_MS elapses without a successful grid read, readGrid 392 // and wait() throw InactivityAbortError so the bot can exit fast and 393 // still write whatever partial data it has. 394 private watchdogArmed = false; 395 private lastSuccessfulReadAt = 0; 396 private static readonly INACTIVITY_TIMEOUT_MS = 120_000; 397 398 constructor(page: Page) { 399 this.page = page; 400 } 401 402 armInactivityWatchdog(): void { 403 this.watchdogArmed = true; 404 this.lastSuccessfulReadAt = Date.now(); 405 } 406 407 private checkInactivity(): void { 408 if (!this.watchdogArmed) return; 409 const elapsed = Date.now() - this.lastSuccessfulReadAt; 410 if (elapsed > PlaywrightDriver.INACTIVITY_TIMEOUT_MS) { 411 throw new InactivityAbortError( 412 `no successful grid read in ${Math.round(elapsed / 1000)}s (watchdog)` 413 ); 414 } 415 } 416 417 // -- Lifecycle -- 418 419 async loadPage(url: string): Promise<{ loaded: boolean; detail: string; errorsOnLoad: number }> { 420 this.consoleErrors = []; 421 this.page.on("pageerror", (err) => { 422 this.consoleErrors.push(err.message); 423 }); 424 425 const errorsBefore = this.consoleErrors.length; 426 427 try { 428 const response = await this.page.goto(url, { 429 timeout: 15000, 430 waitUntil: "networkidle", 431 }); 432 if (!response || !response.ok()) { 433 return { 434 loaded: false, 435 detail: `page load failed: HTTP ${response?.status()}`, 436 errorsOnLoad: this.consoleErrors.length - errorsBefore, 437 }; 438 } 439 await this.page.waitForTimeout(3000); 440 } catch (err) { 441 return { 442 loaded: false, 443 detail: `page load failed: ${err instanceof Error ? err.message : String(err)}`, 444 errorsOnLoad: this.consoleErrors.length - errorsBefore, 445 }; 446 } 447 448 const newErrors = this.consoleErrors.slice(errorsBefore); 449 return { 450 loaded: true, 451 detail: newErrors.length === 0 452 ? "no console errors" 453 : `${newErrors.length} console error(s): ${newErrors[0]}`, 454 errorsOnLoad: newErrors.length, 455 }; 456 } 457 458 async surveyPage(): Promise<SurveyData> { 459 try { 460 return await this.page.evaluate(() => { 461 let hasOverlay = false; 462 const allEls = document.querySelectorAll("*"); 463 const vw = window.innerWidth; 464 const vh = window.innerHeight; 465 for (const el of allEls) { 466 const style = window.getComputedStyle(el); 467 const pos = style.position; 468 if (pos === "fixed" || pos === "absolute") { 469 const zIndex = parseInt(style.zIndex, 10); 470 if (zIndex > 0 || style.zIndex === "auto") { 471 const rect = (el as HTMLElement).getBoundingClientRect(); 472 if (rect.width > vw * 0.5 && rect.height > vh * 0.5) { 473 hasOverlay = true; 474 break; 475 } 476 } 477 } 478 } 479 480 const hasCanvas = document.querySelectorAll("canvas").length > 0; 481 482 let hasDomGrid = false; 483 const containers = document.querySelectorAll( 484 '[class*="board"], [class*="grid"], [class*="field"], [id*="board"], [id*="grid"], [id*="field"]' 485 ); 486 for (const container of containers) { 487 const ch = container.children; 488 if ( 489 (ch.length >= 180 && ch.length <= 230) || 490 (ch.length >= 18 && ch.length <= 22 && 491 ch[0]?.children.length >= 8 && ch[0]?.children.length <= 12) 492 ) { 493 hasDomGrid = true; 494 break; 495 } 496 } 497 if (!hasDomGrid) { 498 const tables = document.querySelectorAll("table"); 499 for (const table of tables) { 500 const rows = table.querySelectorAll("tr"); 501 if (rows.length >= 18 && (rows[0]?.querySelectorAll("td").length ?? 0) >= 8) { 502 hasDomGrid = true; 503 break; 504 } 505 } 506 } 507 // Heuristic scan for unlabeled grids 508 if (!hasDomGrid) { 509 const allElements = document.querySelectorAll("div, section, main, article"); 510 for (const el of allElements) { 511 const ch = el.children; 512 if (ch.length >= 180 && ch.length <= 230) { 513 const firstChild = ch[0] as HTMLElement; 514 if (!firstChild) continue; 515 const firstRect = firstChild.getBoundingClientRect(); 516 if (firstRect.width < 5 || firstRect.height < 5) continue; 517 let uniform = true; 518 for (let i = 1; i < Math.min(10, ch.length); i++) { 519 const r = (ch[i] as HTMLElement).getBoundingClientRect(); 520 if (Math.abs(r.width - firstRect.width) > 2 || Math.abs(r.height - firstRect.height) > 2) { 521 uniform = false; 522 break; 523 } 524 } 525 if (uniform) { hasDomGrid = true; break; } 526 } 527 if (ch.length >= 18 && ch.length <= 22) { 528 const firstRowCells = ch[0]?.children; 529 if (firstRowCells && firstRowCells.length >= 8 && firstRowCells.length <= 12) { 530 const rect = el.getBoundingClientRect(); 531 if (rect.width > 50 && rect.height > 100) { hasDomGrid = true; break; } 532 } 533 } 534 } 535 } 536 537 const bodyText = (document.body?.innerText || "").trim(); 538 const visibleText = bodyText 539 .split("\n") 540 .map((line: string) => line.trim()) 541 .filter((line: string) => line.length > 0) 542 .slice(0, 20); 543 544 const clickableSelector = 545 'button, a, [role="button"], [onclick], input[type="button"], input[type="submit"]'; 546 const clickableElements = document.querySelectorAll(clickableSelector).length; 547 548 return { has_overlay: hasOverlay, has_canvas: hasCanvas, has_dom_grid: hasDomGrid, visible_text: visibleText, clickable_elements: clickableElements }; 549 }); 550 } catch { 551 return { has_overlay: false, has_canvas: false, has_dom_grid: false, visible_text: [], clickable_elements: 0 }; 552 } 553 } 554 555 async detectGameLandmarks(): Promise<GameLandmarks> { 556 return await this.page.evaluate(() => { 557 const body = document.body; 558 const bodyText = body?.innerText?.trim() || ""; 559 const bodyHasContent = body !== null && 560 (bodyText.length > 0 || body.children.length > 0); 561 562 // Check for canvas 563 const canvases = document.querySelectorAll("canvas"); 564 let hasCanvas = false; 565 for (const c of canvases) { 566 const rect = (c as HTMLCanvasElement).getBoundingClientRect(); 567 if (rect.width >= 50 && rect.height >= 50) { 568 hasCanvas = true; 569 break; 570 } 571 } 572 573 // Check for named DOM grid containers 574 const namedContainers = document.querySelectorAll( 575 '[class*="board"], [class*="grid"], [class*="field"], [id*="board"], [id*="grid"]' 576 ); 577 let hasDomGrid = false; 578 for (const el of namedContainers) { 579 const rect = (el as HTMLElement).getBoundingClientRect(); 580 if (rect.width >= 50 && rect.height >= 50) { 581 hasDomGrid = true; 582 break; 583 } 584 } 585 586 // Check for tetris-grid-aspect element (2:1 height:width) 587 let hasTetrisRatioElement = false; 588 const allElements = document.querySelectorAll("div, section, main, article"); 589 for (const el of allElements) { 590 const rect = (el as HTMLElement).getBoundingClientRect(); 591 if (rect.width < 100 || rect.height < 200) continue; 592 const ratio = rect.height / rect.width; 593 if (ratio >= 1.5 && ratio <= 2.5) { 594 hasTetrisRatioElement = true; 595 break; 596 } 597 } 598 599 // Check for large container with many same-sized children 600 let hasManyCellsContainer = false; 601 for (const el of allElements) { 602 const children = el.children; 603 if (children.length < 50 || children.length > 400) continue; 604 const sizes = new Set<string>(); 605 const sampleCount = Math.min(10, children.length); 606 for (let i = 0; i < sampleCount; i++) { 607 const child = children[Math.floor(i * children.length / sampleCount)] as HTMLElement; 608 const r = child.getBoundingClientRect(); 609 if (r.width > 0 && r.height > 0) { 610 sizes.add(`${Math.round(r.width)}x${Math.round(r.height)}`); 611 } 612 } 613 if (sizes.size <= 3 && sizes.size > 0) { 614 hasManyCellsContainer = true; 615 break; 616 } 617 } 618 619 const landmarksFound: string[] = []; 620 if (bodyHasContent) landmarksFound.push("body_content"); 621 if (hasCanvas) landmarksFound.push("canvas"); 622 if (hasDomGrid) landmarksFound.push("dom_grid"); 623 if (hasTetrisRatioElement) landmarksFound.push("tetris_ratio"); 624 if (hasManyCellsContainer) landmarksFound.push("cells_container"); 625 626 return { 627 bodyHasContent, 628 hasCanvas, 629 hasDomGrid, 630 hasTetrisRatioElement, 631 hasManyCellsContainer, 632 landmarksFound, 633 }; 634 }); 635 } 636 637 async calibrate(): Promise<DriverCalibration> { 638 // Fast path: try applying the cached calibration from a prior run. 639 if (this.firstCal) { 640 this.drift.recalibrations++; 641 const cached = await this.applyCachedCalibration(); 642 if (cached) { 643 this.drift.cacheHits++; 644 this.cal = cached; 645 this.log( 646 `[cache] hit: replayed start="${cached.startMechanism}" renderer=${cached.renderer} ` + 647 `(hits=${this.drift.cacheHits}, misses=${this.drift.cacheMisses})` 648 ); 649 return cached; 650 } 651 this.drift.cacheMisses++; 652 this.log( 653 `[cache] miss: cached calibration no longer works, doing full recalibration ` + 654 `(hits=${this.drift.cacheHits}, misses=${this.drift.cacheMisses})` 655 ); 656 } 657 658 const fresh = await this.fullCalibrate(); 659 this.cal = fresh; 660 661 if (!this.firstCal) { 662 // First time -- freeze a copy as the baseline for drift detection. 663 this.firstCal = cloneCalibration(fresh); 664 } else { 665 // Not the first time: compute drift vs baseline. 666 const changes = diffCalibrations(this.firstCal, fresh); 667 if (changes.length > 0) { 668 this.drift.drifted = true; 669 for (const c of changes) { 670 if (!this.drift.changes.includes(c)) this.drift.changes.push(c); 671 } 672 this.log(`CONFLICT: calibration drifted: [${changes.join(", ")}]`); 673 } 674 } 675 676 return fresh; 677 } 678 679 // Runs the full (expensive) calibration flow. Does not touch firstCal/drift. 680 private async fullCalibrate(): Promise<DriverCalibration> { 681 await this.page.waitForTimeout(2000); 682 683 let startMechanism: StartMechanism; 684 let startButton: DriverCalibration["startButton"] | undefined; 685 686 if (this.confirmedCandidate) { 687 // Bot already verified the start. Replay it instead of rediscovering. 688 this.log( 689 `[bridge] replaying confirmed candidate: ${this.confirmedCandidate.label}` 690 ); 691 const applied = await this.applyCandidate(this.confirmedCandidate); 692 startMechanism = applied.ok ? this.confirmedCandidate.mechanism : "unknown"; 693 if (applied.ok && (this.confirmedCandidate.mechanism === "button" || this.confirmedCandidate.mechanism === "click_canvas")) { 694 startButton = { 695 selector: this.confirmedCandidate.selector ?? "canvas", 696 text: this.confirmedCandidate.text ?? this.confirmedCandidate.label, 697 disappeared: false, 698 position: this.confirmedCandidate.position ?? { x: 0, y: 0 }, 699 }; 700 } 701 await this.page.waitForTimeout(this.confirmedCandidate.waitMs ?? 400); 702 } else if (this.startRejected) { 703 // Bot's bridge verification rejected every candidate. Do NOT run the 704 // legacy fallback; it has historically produced false positives 705 // (e.g. clicking Pause) that the bridge was designed to prevent. 706 this.log(`[bridge] start rejected by bot; skipping legacy detection`); 707 startMechanism = "unknown"; 708 startButton = undefined; 709 } else { 710 const startResult = await this.detectStartMechanism(); 711 startMechanism = startResult.mechanism; 712 startButton = startResult.startButton; 713 } 714 715 let gridDetection = await this.detectGrid(); 716 let { renderer, gridBounds, cellWidth, cellHeight } = gridDetection; 717 let backgroundColor = 718 renderer === "canvas" && gridBounds 719 ? await this.sampleBackgroundColor(gridBounds, cellWidth, cellHeight) 720 : null; 721 722 // Re-calibration fallback (skipped when bot already confirmed or rejected the start). 723 if (!this.confirmedCandidate && !this.startRejected && (startMechanism === "unknown" || gridBounds === null)) { 724 const retry = await this.recalibrateWithRetry(startMechanism, gridBounds); 725 if (retry.startMechanism !== "unknown") startMechanism = retry.startMechanism; 726 if (retry.startButton) startButton = retry.startButton; 727 if (retry.gridBounds) { 728 renderer = retry.renderer; 729 gridBounds = retry.gridBounds; 730 cellWidth = retry.cellWidth; 731 cellHeight = retry.cellHeight; 732 backgroundColor = 733 renderer === "canvas" && gridBounds 734 ? await this.sampleBackgroundColor(gridBounds, cellWidth, cellHeight) 735 : null; 736 } 737 } 738 739 const controls = await this.detectControls(); 740 const scoreElementSelector = await this.detectScoreElement(); 741 const levelElementSelector = await this.detectLevelElement(); 742 743 const gridConfidence = await this.measureGridConfidence({ 744 renderer, gridDetected: gridBounds !== null, gridBounds, cellWidth, cellHeight, 745 controls, startMechanism, scoreElementSelector, levelElementSelector, 746 backgroundColor, consoleErrors: [...this.consoleErrors], gridConfidence: 0, 747 gridDetectedAt: "initial", 748 }); 749 750 const cal: DriverCalibration = { 751 renderer, 752 gridDetected: gridBounds !== null, 753 gridBounds, 754 cellWidth, 755 cellHeight, 756 controls, 757 startMechanism, 758 scoreElementSelector, 759 levelElementSelector, 760 backgroundColor, 761 consoleErrors: [...this.consoleErrors], 762 gridConfidence, 763 gridDetectedAt: "initial", 764 }; 765 766 if (startButton) cal.startButton = startButton; 767 return cal; 768 } 769 770 /** 771 * Attempt to replay the cached calibration on the current page. 772 * Returns a completed DriverCalibration on success, null on failure. 773 * On success, the game should be started and the grid detected. 774 */ 775 private async applyCachedCalibration(): Promise<DriverCalibration | null> { 776 const base = this.firstCal; 777 if (!base) return null; 778 779 try { 780 // Small settle delay -- a freshly-loaded page may still be booting. 781 await this.page.waitForTimeout(800); 782 783 // Step 1: re-apply the cached start mechanism. 784 const started = await this.replayStartMechanism(base); 785 if (!started) { 786 this.log( 787 `CONFLICT: cached start mechanism '${base.startMechanism}` + 788 (base.startButton ? ` ${base.startButton.selector}` : "") + 789 `' no longer works` 790 ); 791 return null; 792 } 793 794 // Step 2: verify the grid is back (same renderer, similar bounds). 795 await this.page.waitForTimeout(300); 796 const grid = await this.detectGrid(); 797 if (!grid.gridBounds) { 798 this.log("CONFLICT: cached start worked but no grid detected"); 799 return null; 800 } 801 if (base.gridBounds && !gridBoundsSimilar(base.gridBounds, grid.gridBounds)) { 802 this.log( 803 `CONFLICT: grid bounds changed significantly ` + 804 `(was ${JSON.stringify(base.gridBounds)}, now ${JSON.stringify(grid.gridBounds)})` 805 ); 806 return null; 807 } 808 if (base.renderer !== "unknown" && grid.renderer !== base.renderer) { 809 this.log(`CONFLICT: renderer changed from ${base.renderer} to ${grid.renderer}`); 810 return null; 811 } 812 813 const backgroundColor = 814 grid.renderer === "canvas" && grid.gridBounds 815 ? await this.sampleBackgroundColor(grid.gridBounds, grid.cellWidth, grid.cellHeight) 816 : base.backgroundColor; 817 818 const cal: DriverCalibration = { 819 renderer: grid.renderer, 820 gridDetected: true, 821 gridBounds: grid.gridBounds, 822 cellWidth: grid.cellWidth, 823 cellHeight: grid.cellHeight, 824 controls: { ...base.controls }, 825 controlMap: base.controlMap ? cloneControlMap(base.controlMap) : null, 826 startMechanism: base.startMechanism, 827 scoreElementSelector: base.scoreElementSelector, 828 levelElementSelector: base.levelElementSelector, 829 backgroundColor, 830 consoleErrors: [...this.consoleErrors], 831 gridConfidence: base.gridConfidence, 832 gridDetectedAt: "initial", 833 fromCache: true, 834 }; 835 if (base.startButton) cal.startButton = { ...base.startButton }; 836 return cal; 837 } catch (err) { 838 this.log( 839 `[cache] replay threw: ${err instanceof Error ? err.message : String(err)}` 840 ); 841 return null; 842 } 843 } 844 845 /** 846 * Perform the cached start action. Returns true if a visual change occurred. 847 */ 848 private async replayStartMechanism(base: DriverCalibration): Promise<boolean> { 849 try { 850 const before = await this.page.screenshot(); 851 852 switch (base.startMechanism) { 853 case "auto": 854 // Nothing to replay -- game should already be running. 855 await this.page.waitForTimeout(400); 856 break; 857 case "enter": 858 await this.page.keyboard.press("Enter"); 859 break; 860 case "space": 861 await this.page.keyboard.press("Space"); 862 break; 863 case "anykey": 864 await this.page.keyboard.press("ArrowDown"); 865 break; 866 case "click_canvas": { 867 const pos = base.startButton?.position; 868 if (pos) { 869 await this.page.mouse.click(pos.x, pos.y); 870 } else { 871 const canvas = this.page.locator("canvas").first(); 872 if ((await canvas.count()) > 0) await canvas.click(); 873 else return false; 874 } 875 break; 876 } 877 case "button": { 878 // Prefer the cached selector; fall back to coordinate click. 879 let clicked = false; 880 const sel = base.startButton?.selector; 881 if (sel) { 882 try { 883 const locator = this.page.locator(sel).first(); 884 const count = await locator.count(); 885 if (count > 0) { 886 await locator.click({ timeout: 2000 }); 887 clicked = true; 888 } 889 } catch { /* fall through to coordinate click */ } 890 } 891 if (!clicked && base.startButton?.position) { 892 const pos = base.startButton.position; 893 await this.page.mouse.click(pos.x, pos.y); 894 clicked = true; 895 } 896 if (!clicked) return false; 897 break; 898 } 899 default: 900 return false; 901 } 902 903 await this.page.waitForTimeout(500); 904 905 // For auto-start, we already have no input -- just verify something changed 906 // relative to the blank/initial page state. 907 if (base.startMechanism === "auto") { 908 const after = await this.page.screenshot(); 909 return !before.equals(after); 910 } 911 912 // For the other mechanisms, a visual change after the action is the signal. 913 const result = await this.detectVisualChange({ frames: 3, intervalMs: 100, before }); 914 return result.changed; 915 } catch { 916 return false; 917 } 918 } 919 920 async recalibrate(): Promise<DriverCalibration> { 921 const prev = this.cal; 922 await this.page.waitForTimeout(500); 923 924 const gridDetection = await this.detectGrid(); 925 if (gridDetection.gridBounds && prev) { 926 const backgroundColor = 927 gridDetection.renderer === "canvas" && gridDetection.gridBounds 928 ? await this.sampleBackgroundColor(gridDetection.gridBounds, gridDetection.cellWidth, gridDetection.cellHeight) 929 : null; 930 931 this.cal = { 932 ...prev, 933 renderer: gridDetection.renderer, 934 gridDetected: true, 935 gridBounds: gridDetection.gridBounds, 936 cellWidth: gridDetection.cellWidth, 937 cellHeight: gridDetection.cellHeight, 938 backgroundColor, 939 gridDetectedAt: "after_start", 940 }; 941 } else if (!prev) { 942 return this.calibrate(); 943 } 944 945 return this.cal!; 946 } 947 948 /** 949 * Lightweight grid re-detection without any side effects. Unlike 950 * recalibrate() / calibrate(), this never clicks, presses keys, or 951 * runs detectStartMechanism(). Safe to call from verifyGameStarted() 952 * mid-start-discovery -- if the page has since spawned its grid (e.g. 953 * a DOM game that builds cells inside requestAnimationFrame after a 954 * start button click), the cached calibration gets updated; otherwise 955 * this.cal is left untouched. 956 */ 957 async refreshGridDetection(): Promise<void> { 958 // Short settle delay: some games build their grid inside the first few 959 // animation frames after startGame() runs, so the initial detectGrid() 960 // inside tryStartMechanism() may have fired before the DOM was ready. 961 await this.page.waitForTimeout(200); 962 const gridDetection = await this.detectGrid(); 963 if (!gridDetection.gridBounds) return; 964 965 const backgroundColor = 966 gridDetection.renderer === "canvas" 967 ? await this.sampleBackgroundColor( 968 gridDetection.gridBounds, 969 gridDetection.cellWidth, 970 gridDetection.cellHeight 971 ) 972 : null; 973 974 if (this.cal) { 975 this.cal = { 976 ...this.cal, 977 renderer: gridDetection.renderer, 978 gridDetected: true, 979 gridBounds: gridDetection.gridBounds, 980 cellWidth: gridDetection.cellWidth, 981 cellHeight: gridDetection.cellHeight, 982 backgroundColor, 983 gridDetectedAt: "after_start", 984 }; 985 } else { 986 this.cal = { 987 renderer: gridDetection.renderer, 988 gridDetected: true, 989 gridBounds: gridDetection.gridBounds, 990 cellWidth: gridDetection.cellWidth, 991 cellHeight: gridDetection.cellHeight, 992 controls: { ...DEFAULT_CONTROLS }, 993 startMechanism: "unknown", 994 scoreElementSelector: null, 995 levelElementSelector: null, 996 backgroundColor, 997 consoleErrors: [...this.consoleErrors], 998 gridConfidence: 0, 999 gridDetectedAt: "after_start", 1000 }; 1001 } 1002 } 1003 1004 getCalibration(): DriverCalibration { 1005 if (!this.cal) throw new Error("calibrate() must be called before getCalibration()"); 1006 return this.cal; 1007 } 1008 1009 getCalibrationDrift(): CalibrationDrift { 1010 return { 1011 drifted: this.drift.drifted, 1012 changes: [...this.drift.changes], 1013 recalibrations: this.drift.recalibrations, 1014 cacheHits: this.drift.cacheHits, 1015 cacheMisses: this.drift.cacheMisses, 1016 }; 1017 } 1018 1019 // -- Start-mechanism verification bridge -- 1020 // 1021 // The bot drives start detection explicitly via this trio: 1022 // 1. discoverStartCandidates() -- returns ordered list 1023 // 2. tryStartMechanism(candidate) -- applies one, reports deltas 1024 // 3. confirmStartMechanism(candidate) -- commits after bot verification 1025 // 1026 // Unlike detectStartMechanism(), tryStartMechanism() does NOT judge the 1027 // outcome. It only reports observable deltas so the bot can run its own 1028 // gameplay-based checks before committing. 1029 1030 async discoverStartCandidates(): Promise<StartCandidate[]> { 1031 const candidates: StartCandidate[] = []; 1032 1033 // 1. Auto-start: no action, just wait briefly. 1034 candidates.push({ 1035 mechanism: "auto", 1036 label: "auto-start (wait 1.2s)", 1037 waitMs: 1200, 1038 }); 1039 1040 // 2. DOM buttons, sorted by prominence (start-ish first, disabled/pause-ish last). 1041 try { 1042 const buttons = await this.collectButtonCandidates(); 1043 for (const b of buttons) { 1044 // Skip disabled buttons -- they cannot start a game. 1045 if (b.disabled) continue; 1046 candidates.push({ 1047 mechanism: "button", 1048 label: `button "${b.text || b.selector}"`, 1049 selector: b.selector, 1050 text: b.text, 1051 position: { x: b.x, y: b.y }, 1052 }); 1053 } 1054 } catch { /* no buttons */ } 1055 1056 // 3. Keyboard triggers. 1057 candidates.push({ mechanism: "enter", label: "key Enter", key: "Enter" }); 1058 candidates.push({ mechanism: "space", label: "key Space", key: "Space" }); 1059 candidates.push({ mechanism: "anykey", label: "key ArrowDown", key: "ArrowDown" }); 1060 1061 // 4. Canvas clicks (if a canvas exists). 1062 try { 1063 const canvas = this.page.locator("canvas").first(); 1064 if ((await canvas.count()) > 0) { 1065 const box = await canvas.boundingBox(); 1066 if (box) { 1067 const cx = box.x + box.width / 2; 1068 const cy = box.y + box.height / 2; 1069 candidates.push({ 1070 mechanism: "click_canvas", 1071 label: "canvas click center", 1072 position: { x: Math.round(cx), y: Math.round(cy) }, 1073 }); 1074 candidates.push({ 1075 mechanism: "click_canvas", 1076 label: "canvas click upper", 1077 position: { x: Math.round(cx), y: Math.round(box.y + box.height * 0.25) }, 1078 }); 1079 candidates.push({ 1080 mechanism: "click_canvas", 1081 label: "canvas click lower", 1082 position: { x: Math.round(cx), y: Math.round(box.y + box.height * 0.75) }, 1083 }); 1084 } 1085 } 1086 } catch { /* no canvas */ } 1087 1088 return candidates; 1089 } 1090 1091 async tryStartMechanism(candidate: StartCandidate): Promise<TryStartResult> { 1092 const errorsBefore = this.consoleErrors.length; 1093 1094 let before: Buffer | null = null; 1095 let domBefore = ""; 1096 let clickableBefore = 0; 1097 try { 1098 before = await this.page.screenshot(); 1099 const snap = await this.snapshotDomState(); 1100 domBefore = snap.domKey; 1101 clickableBefore = snap.clickableCount; 1102 } catch { /* screenshot can fail on teardown */ } 1103 1104 let applied = { ok: false }; 1105 try { 1106 applied = await this.applyCandidate(candidate); 1107 } catch { /* treat as not applied */ } 1108 1109 if (!applied.ok) { 1110 return { 1111 visualChanged: false, 1112 domChanged: false, 1113 errorOccurred: this.consoleErrors.length > errorsBefore, 1114 newClickableElements: 0, 1115 removedElements: 0, 1116 }; 1117 } 1118 1119 // Give the game a moment to react. 1120 await this.page.waitForTimeout(candidate.waitMs ?? 300); 1121 1122 let visualChanged = false; 1123 let domChanged = false; 1124 let newClickableElements = 0; 1125 let removedElements = 0; 1126 1127 try { 1128 if (before) { 1129 const after = await this.page.screenshot(); 1130 visualChanged = !before.equals(after); 1131 } 1132 const snap = await this.snapshotDomState(); 1133 domChanged = snap.domKey !== domBefore; 1134 const delta = snap.clickableCount - clickableBefore; 1135 if (delta > 0) newClickableElements = delta; 1136 else if (delta < 0) removedElements = -delta; 1137 } catch { /* report what we have */ } 1138 1139 // Populate a minimal calibration so verifyGameStarted can call readGrid(). 1140 // The bot may reject this candidate, in which case clearConfirmedStartMechanism() 1141 // will wipe this.cal along with the rest of the bridge state. 1142 try { 1143 const gridDetection = await this.detectGrid(); 1144 if (gridDetection.gridBounds) { 1145 const backgroundColor = 1146 gridDetection.renderer === "canvas" 1147 ? await this.sampleBackgroundColor( 1148 gridDetection.gridBounds, 1149 gridDetection.cellWidth, 1150 gridDetection.cellHeight 1151 ) 1152 : null; 1153 this.cal = { 1154 renderer: gridDetection.renderer, 1155 gridDetected: true, 1156 gridBounds: gridDetection.gridBounds, 1157 cellWidth: gridDetection.cellWidth, 1158 cellHeight: gridDetection.cellHeight, 1159 controls: { ...DEFAULT_CONTROLS }, 1160 startMechanism: candidate.mechanism, 1161 scoreElementSelector: null, 1162 levelElementSelector: null, 1163 backgroundColor, 1164 consoleErrors: [...this.consoleErrors], 1165 gridConfidence: 0, 1166 gridDetectedAt: "after_start", 1167 }; 1168 } 1169 } catch { /* no grid detected yet */ } 1170 1171 return { 1172 visualChanged, 1173 domChanged, 1174 errorOccurred: this.consoleErrors.length > errorsBefore, 1175 newClickableElements, 1176 removedElements, 1177 }; 1178 } 1179 1180 confirmStartMechanism(candidate: StartCandidate): void { 1181 this.confirmedCandidate = candidate; 1182 this.startRejected = false; 1183 this.log(`[bridge] confirmed start candidate: ${candidate.label}`); 1184 } 1185 1186 clearConfirmedStartMechanism(): void { 1187 if (this.confirmedCandidate) { 1188 this.log(`[bridge] cleared confirmed start candidate`); 1189 } 1190 this.confirmedCandidate = null; 1191 // Drop cached calibrations so a reload starts fresh. 1192 this.firstCal = null; 1193 this.cal = null; 1194 } 1195 1196 rejectStartMechanism(): void { 1197 this.startRejected = true; 1198 this.confirmedCandidate = null; 1199 // Drop cached calibrations; subsequent calibrate() calls must run fresh 1200 // but MUST NOT attempt any start detection. 1201 this.firstCal = null; 1202 this.cal = null; 1203 this.log(`[bridge] start mechanism rejected by bot`); 1204 } 1205 1206 /** Shared helper: apply a candidate without judging the outcome. */ 1207 private async applyCandidate(candidate: StartCandidate): Promise<{ ok: boolean }> { 1208 try { 1209 switch (candidate.mechanism) { 1210 case "auto": 1211 // Nothing to click/press; the wait happens in tryStartMechanism. 1212 return { ok: true }; 1213 case "enter": 1214 case "space": 1215 case "anykey": { 1216 const key = candidate.key 1217 ?? (candidate.mechanism === "enter" ? "Enter" 1218 : candidate.mechanism === "space" ? "Space" 1219 : "ArrowDown"); 1220 await this.page.keyboard.press(key); 1221 return { ok: true }; 1222 } 1223 case "button": { 1224 const sel = candidate.selector; 1225 let clicked = false; 1226 if (sel) { 1227 try { 1228 const locator = this.page.locator(sel).first(); 1229 if ((await locator.count()) > 0) { 1230 await locator.click({ timeout: 2000 }); 1231 clicked = true; 1232 } 1233 } catch { /* fall through */ } 1234 } 1235 if (!clicked && candidate.position) { 1236 await this.page.mouse.click(candidate.position.x, candidate.position.y); 1237 clicked = true; 1238 } 1239 return { ok: clicked }; 1240 } 1241 case "click_canvas": { 1242 if (candidate.position) { 1243 await this.page.mouse.click(candidate.position.x, candidate.position.y); 1244 return { ok: true }; 1245 } 1246 const canvas = this.page.locator("canvas").first(); 1247 if ((await canvas.count()) > 0) { 1248 await canvas.click(); 1249 return { ok: true }; 1250 } 1251 return { ok: false }; 1252 } 1253 default: 1254 return { ok: false }; 1255 } 1256 } catch { 1257 return { ok: false }; 1258 } 1259 } 1260 1261 /** 1262 * DOM snapshot used to cheaply detect whether tryStartMechanism() caused 1263 * meaningful structural changes on the page. 1264 */ 1265 private async snapshotDomState(): Promise<{ domKey: string; clickableCount: number }> { 1266 try { 1267 return await this.page.evaluate(() => { 1268 const clickableSelector = 1269 'button, a, [role="button"], [onclick], input[type="button"], input[type="submit"]'; 1270 const clickable = document.querySelectorAll(clickableSelector); 1271 const clickableCount = clickable.length; 1272 1273 // Compact key describing the interactive skeleton. 1274 const parts: string[] = []; 1275 clickable.forEach((el, i) => { 1276 if (i > 40) return; 1277 const rect = (el as HTMLElement).getBoundingClientRect(); 1278 parts.push( 1279 `${el.tagName.toLowerCase()}:${(el.textContent || "").trim().slice(0, 20)}:${Math.round(rect.width)}x${Math.round(rect.height)}` 1280 ); 1281 }); 1282 const canvasCount = document.querySelectorAll("canvas").length; 1283 parts.push(`canvas=${canvasCount}`); 1284 1285 // Also include a short excerpt of body text so things like "Paused" 1286 // toggling to "Game Over" register as changes. 1287 const bodyText = (document.body?.innerText || "") 1288 .replace(/\s+/g, " ") 1289 .trim() 1290 .slice(0, 300); 1291 parts.push(`body=${bodyText}`); 1292 1293 return { domKey: parts.join("|"), clickableCount }; 1294 }); 1295 } catch { 1296 return { domKey: "", clickableCount: 0 }; 1297 } 1298 } 1299 1300 /** 1301 * Return clickable elements sorted by prominence. Used by 1302 * discoverStartCandidates(). Boosts "start"-like labels and demotes 1303 * "pause"-like labels. 1304 */ 1305 private async collectButtonCandidates(): Promise<Array<{ 1306 text: string; selector: string; x: number; y: number; disabled: boolean; 1307 }>> { 1308 return await this.page.evaluate(() => { 1309 const seen = new Set<Element>(); 1310 const results: Array<{ 1311 index: number; text: string; x: number; y: number; 1312 width: number; height: number; area: number; centerDist: number; 1313 selector: string; hasBackground: boolean; disabled: boolean; 1314 }> = []; 1315 1316 const clickableSelector = 1317 'button, a, [role="button"], [onclick], input[type="button"], input[type="submit"]'; 1318 for (const el of document.querySelectorAll(clickableSelector)) { 1319 if (!seen.has(el)) seen.add(el); 1320 } 1321 1322 const allEls = document.querySelectorAll("*"); 1323 for (const el of allEls) { 1324 if (seen.has(el)) continue; 1325 try { 1326 const style = window.getComputedStyle(el); 1327 if (style.cursor === "pointer") seen.add(el); 1328 } catch { /* skip */ } 1329 } 1330 1331 const pageW = window.innerWidth; 1332 const pageH = window.innerHeight; 1333 const pageCenterX = pageW / 2; 1334 const pageCenterY = pageH / 2; 1335 1336 let idx = 0; 1337 for (const el of seen) { 1338 const rect = el.getBoundingClientRect(); 1339 if (rect.width < 5 || rect.height < 5) continue; 1340 // Skip elements that are far outside the document (negative coords or 1341 // > 3x the viewport) but allow buttons that are below the fold -- we 1342 // use locator.click() which scrolls them into view. 1343 if (rect.top < -200 || rect.left < -200) continue; 1344 if (rect.top > pageH * 3 || rect.left > pageW * 3) continue; 1345 if (rect.width > pageW * 0.8 && rect.height > pageH * 0.8) continue; 1346 1347 const cx = rect.left + rect.width / 2; 1348 const cy = rect.top + rect.height / 2; 1349 const centerDist = Math.sqrt((cx - pageCenterX) ** 2 + (cy - pageCenterY) ** 2); 1350 1351 let hasBackground = false; 1352 try { 1353 const style = window.getComputedStyle(el as HTMLElement); 1354 const bg = style.backgroundColor; 1355 if (bg && bg !== "transparent" && bg !== "rgba(0, 0, 0, 0)") hasBackground = true; 1356 } catch { /* skip */ } 1357 1358 // Check for a stable id or class selector; disabled buttons are still 1359 // surfaced so the bot can verify that clicking them doesn't start 1360 // the game (fail fast on false positives). 1361 const disabled = (el as HTMLInputElement).disabled === true 1362 || el.getAttribute("aria-disabled") === "true"; 1363 1364 let selector = ""; 1365 if (el.id) selector = `#${el.id}`; 1366 else if ((el as HTMLElement).className) { 1367 const cls = (el as HTMLElement).className.toString().split(" ")[0]; 1368 if (cls) selector = `${el.tagName.toLowerCase()}.${cls}`; 1369 } 1370 if (!selector) selector = `${el.tagName.toLowerCase()}:nth-of-type(${idx + 1})`; 1371 1372 results.push({ 1373 index: idx, text: (el.textContent || "").trim().slice(0, 50), 1374 x: Math.round(cx), y: Math.round(cy), 1375 width: rect.width, height: rect.height, 1376 area: rect.width * rect.height, centerDist, selector, hasBackground, 1377 disabled, 1378 }); 1379 idx++; 1380 } 1381 1382 // Sort: prefer "start"-like labels first, "pause"/"restart"-like last, 1383 // disabled elements demoted, then prominence. 1384 const isStartish = (text: string): number => { 1385 const t = text.toLowerCase(); 1386 if (/\bstart\b|\bplay\b|\bbegin\b|\bgo\b|\binicio\b|\bjugar\b|\bempezar\b|\bcomenzar\b|\bnueva\b|new game/.test(t)) return 0; 1387 if (/\brestart\b|\breset\b|\bplay again\b/.test(t)) return 2; 1388 if (/\bpause\b|\bstop\b|\bquit\b|\bexit\b|\bpausa\b|\bsalir\b/.test(t)) return 3; 1389 return 1; 1390 }; 1391 1392 results.sort((a, b) => { 1393 const ai = isStartish(a.text); 1394 const bi = isStartish(b.text); 1395 if (ai !== bi) return ai - bi; 1396 if (a.disabled !== b.disabled) return a.disabled ? 1 : -1; 1397 if (a.hasBackground !== b.hasBackground) return a.hasBackground ? -1 : 1; 1398 if (Math.abs(b.area - a.area) > 100) return b.area - a.area; 1399 return a.centerDist - b.centerDist; 1400 }); 1401 1402 return results.map(r => ({ 1403 text: r.text, selector: r.selector, x: r.x, y: r.y, disabled: r.disabled, 1404 })); 1405 }); 1406 } 1407 1408 // -- Grid Reading -- 1409 1410 async readGrid(settledGrid?: Grid | null): Promise<GridSnapshot> { 1411 this.checkInactivity(); 1412 try { 1413 const cal = this.cal; 1414 if (!cal) return makeSnapshot(null); 1415 1416 let grid: Grid | null = null; 1417 1418 if (cal.renderer === "canvas" && cal.gridBounds) { 1419 grid = await this.readCanvasGrid(cal.gridBounds, cal.cellWidth, cal.cellHeight, cal.backgroundColor); 1420 } 1421 if (!grid && cal.renderer === "dom") { 1422 grid = await this.readDomGrid(); 1423 } 1424 if (!grid && cal.gridBounds) { 1425 grid = await this.readCanvasGrid(cal.gridBounds, cal.cellWidth, cal.cellHeight, cal.backgroundColor); 1426 } 1427 if (!grid) { 1428 grid = await this.readDomGrid(); 1429 } 1430 1431 if (grid) this.lastSuccessfulReadAt = Date.now(); 1432 return makeSnapshot(grid, settledGrid); 1433 } catch (err) { 1434 if (err instanceof InactivityAbortError) throw err; 1435 return makeSnapshot(null); 1436 } 1437 } 1438 1439 gridsAreDifferent(a: Grid | null, b: Grid | null): boolean { 1440 if (a === null || b === null) return a !== b; 1441 if (a.length !== b.length) return true; 1442 for (let r = 0; r < a.length; r++) { 1443 if (a[r].length !== b[r].length) return true; 1444 for (let c = 0; c < a[r].length; c++) { 1445 if (a[r][c] !== b[r][c]) return true; 1446 } 1447 } 1448 return false; 1449 } 1450 1451 // -- Input -- 1452 1453 async pressKey(action: "left" | "right" | "down" | "rotate" | "drop"): Promise<void> { 1454 // Prefer the discovered control map when available, falling back to the 1455 // legacy controls field. If the discovered soft_drop is null (no soft 1456 // drop on this game), pressing "down" becomes a no-op -- callers that 1457 // care about distinguishing the two should check getControl("soft_drop") 1458 // first. 1459 const discovered = this.discoveredControls; 1460 if (discovered) { 1461 const mapped = this.mapLegacyActionToDiscovered(action, discovered); 1462 if (mapped === null) { 1463 // Explicit no-op: the game doesn't have this action. We still swallow 1464 // the call silently rather than throwing so the bot's play loops that 1465 // sprinkle in optional soft_drop presses don't crash. 1466 return; 1467 } 1468 if (mapped !== undefined) { 1469 await this.page.keyboard.press(mapped); 1470 return; 1471 } 1472 } 1473 const cal = this.cal; 1474 const key = cal ? cal.controls[action] : DEFAULT_CONTROLS[action]; 1475 await this.page.keyboard.press(key); 1476 } 1477 1478 /** 1479 * Translate the legacy pressKey action names to the discovered control map. 1480 * Returns: 1481 * - a string key if discovered and valid 1482 * - null if the action maps to soft_drop and soft_drop is explicitly 1483 * not_found (no-op signal) 1484 * - undefined if no discovery info for this action (caller falls back) 1485 */ 1486 private mapLegacyActionToDiscovered( 1487 action: "left" | "right" | "down" | "rotate" | "drop", 1488 map: ControlMap 1489 ): string | null | undefined { 1490 switch (action) { 1491 case "left": 1492 return map.move_left.key ?? undefined; 1493 case "right": 1494 return map.move_right.key ?? undefined; 1495 case "down": 1496 // Distinguish "discovery ran and found no soft drop" from "discovery 1497 // never ran for this action". The first returns null (no-op), the 1498 // second returns undefined (fall through to defaults). 1499 if (map.soft_drop.confidence === "not_found") return null; 1500 return map.soft_drop.key ?? undefined; 1501 case "rotate": 1502 return map.rotate_cw.key ?? undefined; 1503 case "drop": 1504 return map.hard_drop.key ?? undefined; 1505 } 1506 } 1507 1508 async pressRawKey(key: string): Promise<void> { 1509 await this.page.keyboard.press(key); 1510 } 1511 1512 // -- Control discovery -- 1513 1514 getControl(action: GameAction): string | null { 1515 const map = this.discoveredControls; 1516 if (map) { 1517 switch (action) { 1518 case "move_left": return map.move_left.key; 1519 case "move_right": return map.move_right.key; 1520 case "soft_drop": return map.soft_drop.key; 1521 case "hard_drop": return map.hard_drop.key; 1522 case "rotate_cw": return map.rotate_cw.key; 1523 case "rotate_ccw": return map.rotate_ccw.key; 1524 case "pause": return null; 1525 case "hold": return null; 1526 } 1527 } 1528 // Fallback to legacy controls field. 1529 const cal = this.cal; 1530 const controls = cal ? cal.controls : DEFAULT_CONTROLS; 1531 switch (action) { 1532 case "move_left": return controls.left; 1533 case "move_right": return controls.right; 1534 case "soft_drop": return controls.down; 1535 case "hard_drop": return controls.drop; 1536 case "rotate_cw": return controls.rotate; 1537 case "rotate_ccw": return null; 1538 case "pause": return null; 1539 case "hold": return null; 1540 } 1541 } 1542 1543 /** 1544 * Discover the control mapping by pressing candidate keys and watching 1545 * grid deltas. This is expensive (can reload the page several times) so 1546 * callers should only invoke it once per session. 1547 * 1548 * The result is cached on the driver and flows through getControl() and 1549 * pressKey() from the moment it returns. 1550 */ 1551 async discoverControls(serverUrl: string): Promise<ControlMap> { 1552 const log = (msg: string) => console.log(`[discover] ${msg}`); 1553 // 50s hard budget. Discovery happens between bridge verification and the 1554 // first test phase, so we want to be quick. Each reload costs ~2s, so 1555 // this is ~25 reloads max. 1556 const deadline = Date.now() + 50_000; 1557 const budgetExceeded = () => Date.now() >= deadline; 1558 1559 // Start from an empty map. If any key probe fails, we simply leave that 1560 // slot as "not_found" with no observation. 1561 const map = emptyControlMap(); 1562 1563 // IMPORTANT: tests run after discovery expect the driver state to 1564 // resemble "game freshly started". Discovery is destructive (it presses 1565 // keys, stacks pieces, may even trigger game_over), so we always 1566 // RELOAD between discovery trials and after discovery finishes. 1567 1568 // Helper: reload the page, re-apply the confirmed start, and wait for 1569 // a piece to become observable. 1570 const freshStart = async (): Promise<GridSnapshot | null> => { 1571 try { 1572 await this.loadPage(serverUrl); 1573 } catch { 1574 return null; 1575 } 1576 try { 1577 // calibrate() will replay the confirmed candidate (if one is set) 1578 // and populate this.cal. Discovery runs AFTER bridge verification, 1579 // so confirmedCandidate is already committed at this point. 1580 await this.calibrate(); 1581 } catch { 1582 return null; 1583 } 1584 // Grid might still be spawning; give it a brief window. 1585 await this.wait(300); 1586 // Fall back to refresh in case the grid wasn't detected at the first 1587 // calibrate() pass (DOM games that build cells post-click). 1588 if (!this.cal?.gridDetected) { 1589 try { 1590 await this.refreshGridDetection(); 1591 } catch { /* ignore */ } 1592 } 1593 // Poll for an active piece so the delta classification works. 1594 const emptyGrid: Grid = Array.from({ length: GRID_ROWS }, () => 1595 Array.from({ length: GRID_COLS }, () => false) 1596 ); 1597 let snap = await this.readGrid(emptyGrid); 1598 let tries = 0; 1599 while ((!snap.grid || snap.filledCount === 0) && tries < 20) { 1600 await this.wait(100); 1601 snap = await this.readGrid(emptyGrid); 1602 tries++; 1603 } 1604 if (!snap.grid) return null; 1605 return snap; 1606 }; 1607 1608 // Helper: classify the delta between two grids. 1609 // Returns a label indicating what probably happened, or "no_change". 1610 const classifyDelta = ( 1611 before: Grid, 1612 after: Grid 1613 ): 1614 | { kind: "no_change" } 1615 | { kind: "move_left"; distance: number } 1616 | { kind: "move_right"; distance: number } 1617 | { kind: "move_down"; distance: number } 1618 | { kind: "hard_drop"; distance: number } 1619 | { kind: "rotate" } 1620 | { kind: "other"; detail: string } => { 1621 // Extract fill cells from each grid. 1622 const cellsA: [number, number][] = []; 1623 const cellsB: [number, number][] = []; 1624 for (let r = 0; r < before.length; r++) { 1625 for (let c = 0; c < before[r].length; c++) { 1626 if (before[r][c]) cellsA.push([r, c]); 1627 if (after[r][c]) cellsB.push([r, c]); 1628 } 1629 } 1630 // Same cells? No change. 1631 const keyA = cellsA.map(([r, c]) => `${r},${c}`).sort().join("|"); 1632 const keyB = cellsB.map(([r, c]) => `${r},${c}`).sort().join("|"); 1633 if (keyA === keyB) return { kind: "no_change" }; 1634 1635 // If cell counts are similar (+/- 1), try to detect a rigid translation 1636 // of the active piece. We do this by looking at the symmetric 1637 // differences: cells that disappeared and cells that appeared. 1638 const setA = new Set(keyA.split("|")); 1639 const setB = new Set(keyB.split("|")); 1640 const disappeared: [number, number][] = []; 1641 const appeared: [number, number][] = []; 1642 for (const [r, c] of cellsA) { 1643 if (!setB.has(`${r},${c}`)) disappeared.push([r, c]); 1644 } 1645 for (const [r, c] of cellsB) { 1646 if (!setA.has(`${r},${c}`)) appeared.push([r, c]); 1647 } 1648 if (disappeared.length === 0 && appeared.length === 0) { 1649 return { kind: "no_change" }; 1650 } 1651 1652 // Compute centroids + shapes of the disappeared/appeared sets. 1653 const avg = (arr: [number, number][]) => { 1654 const sumR = arr.reduce((s, [r]) => s + r, 0); 1655 const sumC = arr.reduce((s, [, c]) => s + c, 0); 1656 return [sumR / arr.length, sumC / arr.length] as [number, number]; 1657 }; 1658 const normalize = (cells: [number, number][]): string => { 1659 if (cells.length === 0) return ""; 1660 const minR = Math.min(...cells.map(([r]) => r)); 1661 const minC = Math.min(...cells.map(([, c]) => c)); 1662 return cells 1663 .map(([r, c]) => `${r - minR},${c - minC}`) 1664 .sort() 1665 .join("|"); 1666 }; 1667 1668 // Case 1: equal-size symmetric diff -> pure translation or rotation 1669 // of the active piece. When a 4-cell piece moves 1 column, some cells 1670 // overlap between old/new position (e.g. O-piece has 2 overlapping 1671 // cells), so the symmetric diff can be as small as 2 cells. We allow 1672 // 1-4 cells here. 1673 if ( 1674 disappeared.length === appeared.length && 1675 disappeared.length >= 1 && 1676 disappeared.length <= 4 1677 ) { 1678 const [avgRA, avgCA] = avg(disappeared); 1679 const [avgRB, avgCB] = avg(appeared); 1680 const dRow = avgRB - avgRA; 1681 const dCol = avgCB - avgCA; 1682 const shapeA = normalize(disappeared); 1683 const shapeB = normalize(appeared); 1684 // Compare full-piece footprints: if a 4-cell piece rotates, the 1685 // set of cells in the "disappeared" union with the settled grid 1686 // can be very different from before. A cleaner rotation signal is 1687 // the total cell count in before vs after. For a pure translation, 1688 // the piece has the same bounding-box footprint (same shape) but 1689 // shifted. For a rotation, the bounding box dimensions usually 1690 // change (tall->wide or wide->tall). 1691 const fullBB = (cells: [number, number][]) => { 1692 if (cells.length === 0) return { w: 0, h: 0 }; 1693 return { 1694 w: Math.max(...cells.map(([, c]) => c)) - Math.min(...cells.map(([, c]) => c)) + 1, 1695 h: Math.max(...cells.map(([r]) => r)) - Math.min(...cells.map(([r]) => r)) + 1, 1696 }; 1697 }; 1698 const bbA = fullBB(cellsA); 1699 const bbB = fullBB(cellsB); 1700 // Total-cells test: if the piece count didn't change and the 1701 // bounding box aspect flipped (e.g. 4x1 -> 1x4), that's rotation. 1702 // This catches I-piece rotation reliably. 1703 if ( 1704 cellsA.length === cellsB.length && 1705 ((bbA.w !== bbB.w) || (bbA.h !== bbB.h)) && 1706 Math.abs(bbA.w - bbB.h) <= 1 && Math.abs(bbA.h - bbB.w) <= 1 1707 ) { 1708 return { kind: "rotate" }; 1709 } 1710 1711 // Rotation by shape change: if the disappeared/appeared shapes 1712 // differ, and the centroid drift doesn't look like a clean 1-col 1713 // horizontal translation, classify as rotation. Checked BEFORE the 1714 // horizontal/vertical translation tests so rotations that coincide 1715 // with auto-drop are still tagged as rotations, not translations. 1716 if (shapeA !== shapeB) { 1717 // A clean horizontal translation has dCol >= 1 and dRow small. 1718 // A clean vertical translation has dRow >= 1 and dCol small. 1719 // Anything else is rotation when the shape differs. 1720 const looksLikeClearHorizontal = 1721 Math.abs(dCol) >= 0.9 && Math.abs(dRow) < 0.6; 1722 const looksLikeClearVertical = 1723 Math.abs(dCol) < 0.6 && dRow >= 0.9; 1724 if (!looksLikeClearHorizontal && !looksLikeClearVertical) { 1725 return { kind: "rotate" }; 1726 } 1727 } 1728 1729 // Horizontal translation: large dCol dominates, shape is stable. 1730 if (shapeA === shapeB && Math.abs(dCol) >= 0.9 && Math.abs(dRow) <= 1.2) { 1731 if (dCol <= -0.9) return { kind: "move_left", distance: Math.max(1, Math.round(-dCol)) }; 1732 if (dCol >= 0.9) return { kind: "move_right", distance: Math.max(1, Math.round(dCol)) }; 1733 } 1734 // Vertical translation: dRow dominates, small dCol drift allowed. 1735 if (shapeA === shapeB && Math.abs(dCol) < 0.7 && dRow >= 0.5) { 1736 const distance = Math.max(1, Math.round(dRow)); 1737 if (distance >= 5) return { kind: "hard_drop", distance }; 1738 return { kind: "move_down", distance }; 1739 } 1740 // Shape same but neither clearly horizontal nor vertical -> fall 1741 // through to "other". This avoids misclassifying noise as motion. 1742 } 1743 1744 // Case 2: 4 cells appeared near the bottom and 4 (or fewer) disappeared 1745 // from higher up -> hard drop that teleported the piece. 1746 if (appeared.length >= 3 && disappeared.length >= 3) { 1747 const avgAppearedRow = avg(appeared)[0]; 1748 const avgDisappearedRow = avg(disappeared)[0]; 1749 const dRow = avgAppearedRow - avgDisappearedRow; 1750 if (dRow >= 4) { 1751 return { kind: "hard_drop", distance: Math.round(dRow) }; 1752 } 1753 } 1754 1755 // Case 3: more appeared than disappeared (a new piece spawned while 1756 // the old one dropped and locked). Treat as hard drop if the old 1757 // piece ended up near the bottom. 1758 if (appeared.length > disappeared.length && appeared.length >= 4) { 1759 // Find the set of appeared cells in the bottom half. 1760 const bottomAppeared = appeared.filter(([r]) => r >= GRID_ROWS / 2); 1761 if (bottomAppeared.length >= 3 && disappeared.length <= 4) { 1762 const avgDisappearedRow = disappeared.length > 0 1763 ? avg(disappeared)[0] 1764 : 0; 1765 const avgBottomRow = avg(bottomAppeared)[0]; 1766 const dRow = avgBottomRow - avgDisappearedRow; 1767 if (dRow >= 4) { 1768 return { kind: "hard_drop", distance: Math.round(dRow) }; 1769 } 1770 } 1771 } 1772 1773 return { kind: "other", detail: `disappeared=${disappeared.length}, appeared=${appeared.length}` }; 1774 }; 1775 1776 // Helper: try a single candidate key for an action, return whether it 1777 // matched the expected classification. 1778 const tryCandidateKey = async ( 1779 action: GameAction, 1780 key: string, 1781 expected: ( 1782 delta: ReturnType<typeof classifyDelta> 1783 ) => { matched: boolean; observation: string } 1784 ): Promise<boolean> => { 1785 // Snapshot before. 1786 const before = await this.readGrid(); 1787 if (!before.grid) { 1788 map.key_observations[key] = "grid read failed before press"; 1789 return false; 1790 } 1791 // Ignore keys on a fully-empty grid -- the candidate might do nothing 1792 // simply because there's no active piece yet. 1793 if (before.filledCount === 0) { 1794 map.key_observations[key] = "grid empty before press (no piece)"; 1795 return false; 1796 } 1797 try { 1798 await this.pressRawKey(key); 1799 } catch { 1800 map.key_observations[key] = "keyboard press threw"; 1801 return false; 1802 } 1803 await this.wait(120); 1804 const after = await this.readGrid(); 1805 if (!after.grid) { 1806 map.key_observations[key] = "grid read failed after press"; 1807 return false; 1808 } 1809 const delta = classifyDelta(before.grid, after.grid); 1810 const result = expected(delta); 1811 // Only record this observation if we don't already have one for the key 1812 // (first observation wins -- keeps the report meaningful). 1813 if (!map.key_observations[key]) { 1814 map.key_observations[key] = result.observation; 1815 } 1816 if (result.matched) { 1817 const slot: ControlMapping = { 1818 key, 1819 confidence: "suspected", 1820 observation: result.observation, 1821 }; 1822 switch (action) { 1823 case "move_left": map.move_left = slot; break; 1824 case "move_right": map.move_right = slot; break; 1825 case "soft_drop": map.soft_drop = slot; break; 1826 case "hard_drop": map.hard_drop = slot; break; 1827 case "rotate_cw": map.rotate_cw = slot; break; 1828 case "rotate_ccw": map.rotate_ccw = slot; break; 1829 default: break; 1830 } 1831 return true; 1832 } 1833 return false; 1834 }; 1835 1836 // ---- Movement discovery (order: least disruptive first) ---- 1837 // Try each action in priority order, reloading between actions to get 1838 // a fresh piece that hasn't already moved from the previous probe. 1839 1840 // move_left 1841 if (!budgetExceeded()) { 1842 log("phase: move_left"); 1843 await freshStart(); 1844 for (const key of CONTROL_CANDIDATES.move_left) { 1845 if (budgetExceeded()) break; 1846 const matched = await tryCandidateKey("move_left", key, (delta) => { 1847 if (delta.kind === "move_left") { 1848 return { matched: true, observation: `moved ${delta.distance} col(s) left` }; 1849 } 1850 if (delta.kind === "move_right") { 1851 return { matched: false, observation: `moved ${delta.distance} col(s) right (wrong direction)` }; 1852 } 1853 if (delta.kind === "hard_drop") { 1854 return { matched: false, observation: `hard_drop (${delta.distance} rows)` }; 1855 } 1856 if (delta.kind === "move_down") { 1857 return { matched: false, observation: `moved ${delta.distance} row(s) down` }; 1858 } 1859 if (delta.kind === "rotate") { 1860 return { matched: false, observation: "rotation" }; 1861 } 1862 if (delta.kind === "no_change") { 1863 return { matched: false, observation: "no change" }; 1864 } 1865 return { matched: false, observation: `other change (${delta.detail})` }; 1866 }); 1867 if (matched) { log(` move_left: ${key}`); break; } 1868 } 1869 } 1870 1871 // move_right -- no need to reload if move_left probe succeeded without 1872 // disrupting the board meaningfully. 1873 if (!budgetExceeded()) { 1874 log("phase: move_right"); 1875 await freshStart(); 1876 for (const key of CONTROL_CANDIDATES.move_right) { 1877 if (budgetExceeded()) break; 1878 const matched = await tryCandidateKey("move_right", key, (delta) => { 1879 if (delta.kind === "move_right") { 1880 return { matched: true, observation: `moved ${delta.distance} col(s) right` }; 1881 } 1882 if (delta.kind === "move_left") { 1883 return { matched: false, observation: `moved ${delta.distance} col(s) left (wrong direction)` }; 1884 } 1885 if (delta.kind === "hard_drop") { 1886 return { matched: false, observation: `hard_drop (${delta.distance} rows)` }; 1887 } 1888 if (delta.kind === "move_down") { 1889 return { matched: false, observation: `moved ${delta.distance} row(s) down` }; 1890 } 1891 if (delta.kind === "rotate") { 1892 return { matched: false, observation: "rotation" }; 1893 } 1894 if (delta.kind === "no_change") { 1895 return { matched: false, observation: "no change" }; 1896 } 1897 return { matched: false, observation: `other change (${delta.detail})` }; 1898 }); 1899 if (matched) { log(` move_right: ${key}`); break; } 1900 } 1901 } 1902 1903 // rotate_cw. Fast path: if an earlier phase observed this key producing 1904 // a rotation, promote it without re-testing. 1905 if (!budgetExceeded()) { 1906 log("phase: rotate_cw"); 1907 // Check if a prior phase already saw one of the rotate candidates as 1908 // a rotation. 1909 let promotedEarly = false; 1910 for (const key of CONTROL_CANDIDATES.rotate_cw) { 1911 const obs = map.key_observations[key]; 1912 if (obs === "rotation" || obs === "shape changed (rotation)") { 1913 map.rotate_cw = { 1914 key, 1915 confidence: "suspected", 1916 observation: "rotation (promoted from earlier phase)", 1917 }; 1918 log(` rotate_cw: ${key} (promoted from observation)`); 1919 promotedEarly = true; 1920 break; 1921 } 1922 } 1923 if (!promotedEarly) { 1924 await freshStart(); 1925 for (const key of CONTROL_CANDIDATES.rotate_cw) { 1926 if (budgetExceeded()) break; 1927 const matched = await tryCandidateKey("rotate_cw", key, (delta) => { 1928 if (delta.kind === "rotate") { 1929 return { matched: true, observation: "shape changed (rotation)" }; 1930 } 1931 if (delta.kind === "hard_drop") { 1932 return { matched: false, observation: `hard_drop (${delta.distance} rows)` }; 1933 } 1934 if (delta.kind === "move_down") { 1935 return { matched: false, observation: `moved ${delta.distance} row(s) down` }; 1936 } 1937 if (delta.kind === "move_left") { 1938 return { matched: false, observation: `moved ${delta.distance} col(s) left` }; 1939 } 1940 if (delta.kind === "move_right") { 1941 return { matched: false, observation: `moved ${delta.distance} col(s) right` }; 1942 } 1943 if (delta.kind === "no_change") { 1944 return { matched: false, observation: "no change" }; 1945 } 1946 return { matched: false, observation: `other change (${delta.detail})` }; 1947 }); 1948 if (matched) { log(` rotate_cw: ${key}`); break; } 1949 } 1950 } 1951 } 1952 1953 // hard_drop (do this BEFORE soft_drop so we know which key teleports). 1954 // IMPORTANT: hard drop candidates include keys like Space that may be 1955 // bound to other functions (pause, confirm, etc). Pressing Space on a 1956 // game that uses it for pause will freeze all subsequent probes. So we 1957 // RELOAD before every hard_drop attempt to guarantee a clean state. 1958 if (!budgetExceeded()) { 1959 log("phase: hard_drop"); 1960 for (const key of CONTROL_CANDIDATES.hard_drop) { 1961 if (budgetExceeded()) break; 1962 if (map.hard_drop.confidence !== "not_found") break; 1963 // Prior observation fast-path: if we already saw this key act like 1964 // a rotation/left/right/etc. in an earlier phase, skip retesting. 1965 const priorObs = map.key_observations[key]; 1966 if (priorObs && !priorObs.includes("teleported") && !priorObs.includes("hard_drop")) { 1967 continue; 1968 } 1969 // Always reload before a hard_drop probe. 1970 await freshStart(); 1971 const matched = await tryCandidateKey("hard_drop", key, (delta) => { 1972 if (delta.kind === "hard_drop") { 1973 return { matched: true, observation: `teleported ${delta.distance} rows to bottom` }; 1974 } 1975 if (delta.kind === "move_down") { 1976 return { 1977 matched: false, 1978 observation: `moved ${delta.distance} row(s) down (soft drop, not hard drop)`, 1979 }; 1980 } 1981 if (delta.kind === "rotate") { 1982 return { matched: false, observation: "rotation" }; 1983 } 1984 if (delta.kind === "move_left") { 1985 return { matched: false, observation: `moved ${delta.distance} col(s) left` }; 1986 } 1987 if (delta.kind === "move_right") { 1988 return { matched: false, observation: `moved ${delta.distance} col(s) right` }; 1989 } 1990 if (delta.kind === "no_change") { 1991 return { matched: false, observation: "no change" }; 1992 } 1993 return { matched: false, observation: `other change (${delta.detail})` }; 1994 }); 1995 if (matched) { log(` hard_drop: ${key}`); break; } 1996 } 1997 } 1998 1999 // soft_drop -- this is where the main bug lives. Try alternatives FIRST. 2000 // ArrowDown is tried only if it wasn't already claimed as hard_drop. 2001 if (!budgetExceeded()) { 2002 log("phase: soft_drop"); 2003 await freshStart(); 2004 // Skip ArrowDown if we already discovered it maps to hard_drop on 2005 // this game. Same for any key already claimed by another action. 2006 const claimedKeys = new Set<string>(); 2007 for (const slot of [map.move_left, map.move_right, map.hard_drop, map.rotate_cw]) { 2008 if (slot.key) claimedKeys.add(slot.key); 2009 } 2010 for (const key of CONTROL_CANDIDATES.soft_drop) { 2011 if (budgetExceeded()) break; 2012 if (claimedKeys.has(key)) { 2013 // ArrowDown was already claimed as hard_drop etc. 2014 map.key_observations[key] = 2015 map.key_observations[key] || "already claimed by another action"; 2016 continue; 2017 } 2018 const matched = await tryCandidateKey("soft_drop", key, (delta) => { 2019 if (delta.kind === "move_down") { 2020 return { 2021 matched: delta.distance >= 1 && delta.distance <= 3, 2022 observation: `moved ${delta.distance} row(s) down`, 2023 }; 2024 } 2025 if (delta.kind === "hard_drop") { 2026 return { 2027 matched: false, 2028 observation: `teleported ${delta.distance} rows (hard_drop, not soft_drop)`, 2029 }; 2030 } 2031 if (delta.kind === "rotate") { 2032 return { matched: false, observation: "rotation" }; 2033 } 2034 if (delta.kind === "move_left") { 2035 return { matched: false, observation: `moved ${delta.distance} col(s) left` }; 2036 } 2037 if (delta.kind === "move_right") { 2038 return { matched: false, observation: `moved ${delta.distance} col(s) right` }; 2039 } 2040 if (delta.kind === "no_change") { 2041 return { matched: false, observation: "no change" }; 2042 } 2043 return { matched: false, observation: `other change (${delta.detail})` }; 2044 }); 2045 if (matched) { log(` soft_drop: ${key}`); break; } 2046 } 2047 if (map.soft_drop.confidence === "not_found") { 2048 log(" soft_drop: NOT FOUND"); 2049 } 2050 } 2051 2052 // rotate_ccw (best effort; cheap skip if budget exceeded) 2053 if (!budgetExceeded()) { 2054 log("phase: rotate_ccw"); 2055 await freshStart(); 2056 for (const key of CONTROL_CANDIDATES.rotate_ccw) { 2057 if (budgetExceeded()) break; 2058 const matched = await tryCandidateKey("rotate_ccw", key, (delta) => { 2059 if (delta.kind === "rotate") { 2060 return { matched: true, observation: "shape changed (rotation)" }; 2061 } 2062 if (delta.kind === "no_change") { 2063 return { matched: false, observation: "no change" }; 2064 } 2065 return { matched: false, observation: `other change` }; 2066 }); 2067 if (matched) { log(` rotate_ccw: ${key}`); break; } 2068 } 2069 } 2070 2071 // Final reload so tests start from a clean state. 2072 await freshStart(); 2073 2074 this.discoveredControls = map; 2075 // Mirror into the cached calibration so bot-side code that reads 2076 // cal.controlMap sees the discovery result. 2077 if (this.cal) { 2078 this.cal.controlMap = map; 2079 // Also back-port the discovered keys into the legacy controls field 2080 // so any code that still reads cal.controls.<x> keeps working. 2081 if (map.move_left.key) this.cal.controls.left = map.move_left.key; 2082 if (map.move_right.key) this.cal.controls.right = map.move_right.key; 2083 if (map.rotate_cw.key) this.cal.controls.rotate = map.rotate_cw.key; 2084 if (map.hard_drop.key) this.cal.controls.drop = map.hard_drop.key; 2085 // Only override the legacy `down` if we found a distinct soft drop key. 2086 // If soft_drop is not_found, leave `down` alone so legacy callers still 2087 // have SOMETHING to press -- pressKey() will intercept and no-op anyway. 2088 if (map.soft_drop.key) this.cal.controls.down = map.soft_drop.key; 2089 } 2090 // Persist into the first-cal baseline too so cache replays preserve the 2091 // discovered map. 2092 if (this.firstCal) { 2093 this.firstCal.controlMap = cloneControlMap(map); 2094 if (map.move_left.key) this.firstCal.controls.left = map.move_left.key; 2095 if (map.move_right.key) this.firstCal.controls.right = map.move_right.key; 2096 if (map.rotate_cw.key) this.firstCal.controls.rotate = map.rotate_cw.key; 2097 if (map.hard_drop.key) this.firstCal.controls.drop = map.hard_drop.key; 2098 if (map.soft_drop.key) this.firstCal.controls.down = map.soft_drop.key; 2099 } 2100 return map; 2101 } 2102 2103 async wait(ms: number): Promise<void> { 2104 this.checkInactivity(); 2105 await this.page.waitForTimeout(ms); 2106 } 2107 2108 // -- Score/Level -- 2109 2110 async readScore(): Promise<number | null> { 2111 const cal = this.cal; 2112 if (!cal?.scoreElementSelector) return null; 2113 try { 2114 const scoreText = await this.page.textContent(cal.scoreElementSelector); 2115 const nums = this.extractScoreFromText(scoreText); 2116 return nums.length > 0 ? Math.max(...nums) : null; 2117 } catch { 2118 return null; 2119 } 2120 } 2121 2122 async readLevel(): Promise<number | null> { 2123 try { 2124 return await this.page.evaluate(() => { 2125 const allElements = document.querySelectorAll("*"); 2126 for (const el of allElements) { 2127 const text = ((el as HTMLElement).innerText || "").toLowerCase(); 2128 if (text.includes("level") && el.children.length < 5) { 2129 const match = text.match(/level\s*[:\-=]?\s*(\d+)/i); 2130 if (match) return parseInt(match[1], 10); 2131 2132 const children = el.querySelectorAll("span, div, p, td, strong, em, b"); 2133 for (const child of children) { 2134 const childText = (child.textContent || "").trim(); 2135 if (/^\d+$/.test(childText)) return parseInt(childText, 10); 2136 } 2137 2138 const next = el.nextElementSibling; 2139 if (next) { 2140 const nextText = (next.textContent || "").trim(); 2141 if (/^\d+$/.test(nextText)) return parseInt(nextText, 10); 2142 } 2143 } 2144 } 2145 // Also try nivel (Spanish) 2146 for (const el of allElements) { 2147 const text = ((el as HTMLElement).innerText || "").toLowerCase(); 2148 if (text.includes("nivel") && el.children.length < 5) { 2149 const match = text.match(/nivel\s*[:\-=]?\s*(\d+)/i); 2150 if (match) return parseInt(match[1], 10); 2151 } 2152 } 2153 return null; 2154 }); 2155 } catch { 2156 return null; 2157 } 2158 } 2159 2160 // -- Page State Queries -- 2161 2162 /** 2163 * Detect if a game-over modal/overlay is visible on the page. 2164 * Language-agnostic: looks for a NEW visible element that wasn't there during 2165 * gameplay, covering a significant portion of the viewport, with position 2166 * fixed/absolute and meaningful z-index. Returns a description like "modal" 2167 * or "overlay" rather than extracted text. 2168 */ 2169 async detectGameOverText(): Promise<string | null> { 2170 try { 2171 return await this.page.evaluate(() => { 2172 const vw = window.innerWidth; 2173 const vh = window.innerHeight; 2174 const viewportArea = vw * vh; 2175 2176 // Find elements that look like modals/overlays 2177 const all = document.querySelectorAll<HTMLElement>("*"); 2178 for (const el of all) { 2179 const style = window.getComputedStyle(el); 2180 const pos = style.position; 2181 if (pos !== "fixed" && pos !== "absolute") continue; 2182 2183 // Must be visible 2184 if (style.display === "none" || style.visibility === "hidden" || 2185 parseFloat(style.opacity) === 0) continue; 2186 2187 const rect = el.getBoundingClientRect(); 2188 if (rect.width === 0 || rect.height === 0) continue; 2189 2190 const area = rect.width * rect.height; 2191 // Significant coverage: 15% or more of viewport 2192 if (area < viewportArea * 0.15) continue; 2193 2194 // Must have visible background or children (not a transparent wrapper) 2195 const hasBg = style.backgroundColor !== "rgba(0, 0, 0, 0)" && 2196 style.backgroundColor !== "transparent"; 2197 const hasContent = el.children.length > 0 || (el.textContent || "").trim().length > 0; 2198 if (!hasBg && !hasContent) continue; 2199 2200 // High z-index relative to the page 2201 const z = parseInt(style.zIndex, 10); 2202 if (!isNaN(z) && z < 1) continue; 2203 2204 return "modal"; 2205 } 2206 2207 return null; 2208 }); 2209 } catch { 2210 return null; 2211 } 2212 } 2213 2214 /** 2215 * Detect if a restart/new-game option is visible. 2216 * Language-agnostic: looks for a clickable element that appears NOW and 2217 * wasn't there during initial play. We approximate this by finding any 2218 * visible clickable element (button, [role=button], cursor:pointer) inside 2219 * a visible modal/overlay, or a clickable element that wasn't present at 2220 * calibration time. This avoids text matching. 2221 */ 2222 async detectRestartOption(): Promise<boolean> { 2223 try { 2224 return await this.page.evaluate(() => { 2225 const vw = window.innerWidth; 2226 const vh = window.innerHeight; 2227 const viewportArea = vw * vh; 2228 2229 // Find modals/overlays 2230 const all = document.querySelectorAll<HTMLElement>("*"); 2231 for (const el of all) { 2232 const style = window.getComputedStyle(el); 2233 const pos = style.position; 2234 if (pos !== "fixed" && pos !== "absolute") continue; 2235 if (style.display === "none" || style.visibility === "hidden" || 2236 parseFloat(style.opacity) === 0) continue; 2237 const rect = el.getBoundingClientRect(); 2238 if (rect.width === 0 || rect.height === 0) continue; 2239 if (rect.width * rect.height < viewportArea * 0.15) continue; 2240 2241 // Look for clickable elements inside this modal 2242 const clickables = el.querySelectorAll<HTMLElement>( 2243 "button, [role=button], a, input[type=button], input[type=submit]" 2244 ); 2245 for (const c of clickables) { 2246 const cRect = c.getBoundingClientRect(); 2247 if (cRect.width > 0 && cRect.height > 0) return true; 2248 } 2249 2250 // Also check for elements with cursor: pointer 2251 const all2 = el.querySelectorAll<HTMLElement>("*"); 2252 for (const c of all2) { 2253 if (window.getComputedStyle(c).cursor === "pointer") { 2254 const cRect = c.getBoundingClientRect(); 2255 if (cRect.width > 0 && cRect.height > 0) return true; 2256 } 2257 } 2258 } 2259 2260 return false; 2261 }); 2262 } catch { 2263 return false; 2264 } 2265 } 2266 2267 async detectNextPiecePreview(): Promise<boolean> { 2268 try { 2269 return await this.page.evaluate(() => { 2270 const allElements = document.querySelectorAll("*"); 2271 for (const el of allElements) { 2272 const text = ((el as HTMLElement).innerText || "").toLowerCase(); 2273 if ((text.includes("next") || text.includes("siguiente")) && el.children.length < 10) { 2274 const rect = (el as HTMLElement).getBoundingClientRect(); 2275 if (rect.width > 20 && rect.height > 20) return true; 2276 } 2277 } 2278 2279 const canvases = document.querySelectorAll("canvas"); 2280 if (canvases.length >= 2) { 2281 const mainRect = canvases[0].getBoundingClientRect(); 2282 for (let i = 1; i < canvases.length; i++) { 2283 const rect = canvases[i].getBoundingClientRect(); 2284 if (rect.width < mainRect.width * 0.5 && rect.height < mainRect.height * 0.5 && 2285 rect.width > 20 && rect.height > 20) return true; 2286 } 2287 } 2288 2289 const nextContainers = document.querySelectorAll( 2290 '[class*="next"], [id*="next"], [class*="preview"], [id*="preview"], [class*="siguiente"], [id*="siguiente"]' 2291 ); 2292 for (const container of nextContainers) { 2293 const rect = (container as HTMLElement).getBoundingClientRect(); 2294 if (rect.width > 20 && rect.height > 20) return true; 2295 } 2296 return false; 2297 }); 2298 } catch { 2299 return false; 2300 } 2301 } 2302 2303 getConsoleErrors(): string[] { 2304 return [...this.consoleErrors]; 2305 } 2306 2307 // -- Screenshots -- 2308 2309 async screenshot(): Promise<Buffer> { 2310 return await this.page.screenshot(); 2311 } 2312 2313 async screenshotGridArea(): Promise<Buffer | null> { 2314 const cal = this.cal; 2315 if (!cal || !cal.gridBounds) return null; 2316 const b = cal.gridBounds; 2317 // For DOM renderers, gridBounds are viewport coordinates and can be clipped 2318 // directly. For canvas renderers they are internal canvas coordinates, so 2319 // re-derive the on-page bounds from the canvas location to stay accurate. 2320 try { 2321 if (cal.renderer === "canvas") { 2322 const boundingBox = await this.page.locator("canvas").first().boundingBox(); 2323 if (!boundingBox) return null; 2324 return await this.page.screenshot({ 2325 clip: { 2326 x: Math.max(0, Math.round(boundingBox.x)), 2327 y: Math.max(0, Math.round(boundingBox.y)), 2328 width: Math.max(1, Math.round(boundingBox.width)), 2329 height: Math.max(1, Math.round(boundingBox.height)), 2330 }, 2331 }); 2332 } 2333 return await this.page.screenshot({ 2334 clip: { 2335 x: Math.max(0, Math.round(b.x)), 2336 y: Math.max(0, Math.round(b.y)), 2337 width: Math.max(1, Math.round(b.width)), 2338 height: Math.max(1, Math.round(b.height)), 2339 }, 2340 }); 2341 } catch { 2342 return null; 2343 } 2344 } 2345 2346 async captureGridDomFingerprint(): Promise<string> { 2347 try { 2348 return await this.page.evaluate(() => { 2349 // Locate the most plausible grid container. Mirrors the detection in 2350 // detectGrid() but runs standalone so the fingerprint works even when 2351 // the calibration has not committed to a specific grid yet. 2352 const findContainer = (): Element | null => { 2353 const tables = document.querySelectorAll("table"); 2354 for (const table of tables) { 2355 const rows = table.querySelectorAll("tr"); 2356 if (rows.length >= 18) { 2357 const firstRow = rows[0].querySelectorAll("td"); 2358 if (firstRow.length >= 8 && firstRow.length <= 12) return table; 2359 } 2360 } 2361 const namedCandidates = document.querySelectorAll( 2362 '[class*="board"], [class*="grid"], [class*="field"], ' + 2363 '[id*="board"], [id*="grid"], [id*="field"]' 2364 ); 2365 for (const c of namedCandidates) { 2366 const ch = c.children; 2367 if (ch.length >= 180 && ch.length <= 230) return c; 2368 if ( 2369 ch.length >= 18 && ch.length <= 22 && 2370 ch[0] && ch[0].children.length >= 8 && ch[0].children.length <= 12 2371 ) { 2372 return c; 2373 } 2374 } 2375 // Heuristic scan for any container with ~200 uniform children. 2376 const allElements = document.querySelectorAll("div, section, main, article"); 2377 for (const el of allElements) { 2378 const ch = el.children; 2379 if (ch.length >= 180 && ch.length <= 230) return el; 2380 } 2381 return null; 2382 }; 2383 2384 const container = findContainer(); 2385 if (!container) return ""; 2386 2387 const parts: string[] = []; 2388 parts.push(`count=${container.children.length}`); 2389 2390 // Serialize each child: class, inline position, inline background. 2391 // Only inline styles (not computed) -- avoids paying for 2392 // getComputedStyle() on 200+ elements per fingerprint, and in practice 2393 // absolute-positioned piece overlays always use inline top/left/bg. 2394 let i = 0; 2395 for (const child of container.children) { 2396 if (i >= 260) break; // hard cap to bound work 2397 const el = child as HTMLElement; 2398 const cls = el.className || ""; 2399 const style = el.style; 2400 const left = style.left || ""; 2401 const top = style.top || ""; 2402 const bg = style.backgroundColor || ""; 2403 const color = style.getPropertyValue("--color") || ""; 2404 parts.push(`${i}:${cls}:${left}:${top}:${bg}:${color}`); 2405 i++; 2406 } 2407 2408 return parts.join("|"); 2409 }); 2410 } catch { 2411 return ""; 2412 } 2413 } 2414 2415 async measureDropInterval(): Promise<number> { 2416 try { 2417 const intervals: number[] = []; 2418 let lastChangeTime = Date.now(); 2419 let prevSnap = await this.readGrid(); 2420 2421 for (let i = 0; i < 10; i++) { 2422 await this.page.waitForTimeout(100); 2423 const snap = await this.readGrid(); 2424 if (snap.grid && prevSnap.grid && this.gridsAreDifferent(snap.grid, prevSnap.grid)) { 2425 const now = Date.now(); 2426 const interval = now - lastChangeTime; 2427 if (interval > 50 && interval < 3000) intervals.push(interval); 2428 lastChangeTime = now; 2429 prevSnap = snap; 2430 } 2431 } 2432 2433 if (intervals.length >= 2) { 2434 return intervals.reduce((a, b) => a + b, 0) / intervals.length; 2435 } 2436 } catch { /* ignore */ } 2437 return 0; 2438 } 2439 2440 // ========================================================================= 2441 // PRIVATE METHODS 2442 // ========================================================================= 2443 2444 // -- Grid reading internals -- 2445 2446 private async readCanvasGrid( 2447 bounds: GridBounds, cellW: number, cellH: number, 2448 bgColor: [number, number, number] | null 2449 ): Promise<Grid | null> { 2450 const bgR = bgColor ? bgColor[0] : 0; 2451 const bgG = bgColor ? bgColor[1] : 0; 2452 const bgB = bgColor ? bgColor[2] : 0; 2453 const threshold = 50; 2454 2455 const grid = await this.page.evaluate( 2456 ({ x, y, cellW, cellH, rows, cols, bgR, bgG, bgB, threshold }) => { 2457 const canvas = document.querySelector("canvas") as HTMLCanvasElement | null; 2458 if (!canvas) return null; 2459 const ctx = canvas.getContext("2d"); 2460 if (!ctx) return null; 2461 2462 const offsets = [ 2463 [0, 0], 2464 [-Math.floor(cellW / 4), 0], 2465 [Math.floor(cellW / 4), 0], 2466 [0, -Math.floor(cellH / 4)], 2467 [0, Math.floor(cellH / 4)], 2468 ]; 2469 2470 const result: boolean[][] = []; 2471 for (let row = 0; row < rows; row++) { 2472 const rowData: boolean[] = []; 2473 for (let col = 0; col < cols; col++) { 2474 const cx = Math.floor(x + col * cellW + cellW / 2); 2475 const cy = Math.floor(y + row * cellH + cellH / 2); 2476 2477 let filledCount = 0; 2478 for (const [ox, oy] of offsets) { 2479 const px = Math.min(Math.max(cx + ox, 0), canvas.width - 1); 2480 const py = Math.min(Math.max(cy + oy, 0), canvas.height - 1); 2481 const pixel = ctx.getImageData(px, py, 1, 1).data; 2482 const dr = pixel[0] - bgR; 2483 const dg = pixel[1] - bgG; 2484 const db = pixel[2] - bgB; 2485 const dist = Math.sqrt(dr * dr + dg * dg + db * db); 2486 if (dist > threshold) filledCount++; 2487 } 2488 rowData.push(filledCount >= 3); 2489 } 2490 result.push(rowData); 2491 } 2492 return result; 2493 }, 2494 { x: bounds.x, y: bounds.y, cellW, cellH, rows: GRID_ROWS, cols: GRID_COLS, bgR, bgG, bgB, threshold } 2495 ); 2496 2497 if (grid) { 2498 const totalCells = GRID_ROWS * GRID_COLS; 2499 const filledCells = grid.reduce((sum, row) => sum + row.filter(Boolean).length, 0); 2500 if (filledCells / totalCells > 0.60) return null; 2501 } 2502 2503 return grid; 2504 } 2505 2506 private async readDomGrid(): Promise<Grid | null> { 2507 const grid = await this.page.evaluate(({ rows, cols }) => { 2508 function isCellFilled(cell: HTMLElement, emptyBg?: string): boolean { 2509 const style = window.getComputedStyle(cell); 2510 const bg = style.backgroundColor; 2511 const cls = cell.className.toLowerCase(); 2512 2513 if ( 2514 cls.includes("filled") || cls.includes("active") || 2515 cls.includes("block") || cls.includes("piece") || 2516 cls.includes("occupied") || cls.includes("locked") || 2517 cell.dataset.filled === "true" || cell.dataset.type !== undefined 2518 ) return true; 2519 2520 if (emptyBg && bg === emptyBg) return false; 2521 2522 return ( 2523 bg !== "" && bg !== "rgba(0, 0, 0, 0)" && 2524 bg !== "transparent" && bg !== "rgb(0, 0, 0)" 2525 ); 2526 } 2527 2528 function detectEmptyBg(cells: HTMLElement[]): string | undefined { 2529 const colorCounts = new Map<string, number>(); 2530 for (const cell of cells) { 2531 const bg = window.getComputedStyle(cell).backgroundColor; 2532 colorCounts.set(bg, (colorCounts.get(bg) || 0) + 1); 2533 } 2534 let maxCount = 0; 2535 let emptyBg: string | undefined; 2536 for (const [color, count] of colorCounts) { 2537 if (count > maxCount) { maxCount = count; emptyBg = color; } 2538 } 2539 if (emptyBg && maxCount > cells.length * 0.6) return emptyBg; 2540 return undefined; 2541 } 2542 2543 // Strategy 1: table-based grid 2544 const tables = document.querySelectorAll("table"); 2545 for (const table of tables) { 2546 const trs = table.querySelectorAll("tr"); 2547 if (trs.length >= rows) { 2548 const allCells: HTMLElement[] = []; 2549 for (let r = 0; r < rows; r++) { 2550 const tds = trs[r].querySelectorAll("td"); 2551 for (let c = 0; c < Math.min(cols, tds.length); c++) allCells.push(tds[c] as HTMLElement); 2552 } 2553 const emptyBg = detectEmptyBg(allCells); 2554 const result: boolean[][] = []; 2555 for (let r = 0; r < rows; r++) { 2556 const tds = trs[r].querySelectorAll("td"); 2557 const rowData: boolean[] = []; 2558 for (let c = 0; c < cols; c++) { 2559 rowData.push(c < tds.length ? isCellFilled(tds[c] as HTMLElement, emptyBg) : false); 2560 } 2561 result.push(rowData); 2562 } 2563 return result; 2564 } 2565 } 2566 2567 // Overlay detection: some games render the active piece as absolute- 2568 // positioned sibling divs inside the grid container (so they float 2569 // over the static cell grid). These are NOT part of the cell loop but 2570 // their position tells us which grid cell they occupy. Called after 2571 // building the cell-based grid; only overwrites empty cells. 2572 function applyOverlayPieces( 2573 container: Element, cellGrid: boolean[][], cellsConsumed: number, 2574 actualRows: number, actualCols: number 2575 ): void { 2576 const containerRect = container.getBoundingClientRect(); 2577 if (containerRect.width <= 0 || containerRect.height <= 0) return; 2578 const cellW = containerRect.width / actualCols; 2579 const cellH = containerRect.height / actualRows; 2580 if (cellW < 5 || cellH < 5) return; 2581 2582 const allChildren = container.children; 2583 for (let i = cellsConsumed; i < allChildren.length; i++) { 2584 const el = allChildren[i] as HTMLElement; 2585 const style = window.getComputedStyle(el); 2586 // Skip statically-positioned siblings -- we only want pieces 2587 // that float over the grid. 2588 if (style.position !== "absolute" && style.position !== "fixed") continue; 2589 const rect = el.getBoundingClientRect(); 2590 if (rect.width <= 0 || rect.height <= 0) continue; 2591 // Center of the overlay element, relative to the container 2592 const cx = rect.left + rect.width / 2 - containerRect.left; 2593 const cy = rect.top + rect.height / 2 - containerRect.top; 2594 const col = Math.floor(cx / cellW); 2595 const row = Math.floor(cy / cellH); 2596 if (row < 0 || row >= actualRows || col < 0 || col >= actualCols) continue; 2597 cellGrid[row][col] = true; 2598 } 2599 } 2600 2601 // Strategy 2: named grid containers 2602 const containers = document.querySelectorAll( 2603 '[class*="board"], [class*="grid"], [class*="field"], [id*="board"], [id*="grid"], [id*="field"]' 2604 ); 2605 for (const container of containers) { 2606 const children = container.children; 2607 // Container has rows*cols (+/- 10) static cells, optionally followed 2608 // by a handful of absolute-positioned children that act as piece 2609 // overlays (e.g. the active piece rendered on top of the static grid). 2610 // Accept up to 30 extra children beyond the cell count. 2611 if (children.length >= rows * cols - 10 && children.length <= rows * cols + 30) { 2612 const actualCols = cols; 2613 // If we're at the short end of the range, fall back to the old 2614 // behaviour and derive actualRows from the child count. Otherwise 2615 // assume the extras are overlays and use the full grid dimensions. 2616 const isShortGrid = children.length <= rows * cols + 4; 2617 const actualRows = isShortGrid 2618 ? Math.round(children.length / actualCols) 2619 : rows; 2620 const cellsConsumed = actualRows * actualCols; 2621 const allCells = Array.from(children).slice(0, cellsConsumed) as HTMLElement[]; 2622 const emptyBg = detectEmptyBg(allCells); 2623 const result: boolean[][] = []; 2624 for (let r = 0; r < actualRows; r++) { 2625 const rowData: boolean[] = []; 2626 for (let c = 0; c < actualCols; c++) { 2627 const cell = allCells[r * actualCols + c]; 2628 rowData.push(cell ? isCellFilled(cell, emptyBg) : false); 2629 } 2630 result.push(rowData); 2631 } 2632 // Overlay detection is a no-op when there are no extra children 2633 // past the static cell grid, so it's safe to always call. 2634 applyOverlayPieces(container, result, cellsConsumed, actualRows, actualCols); 2635 return result; 2636 } 2637 if (children.length >= rows - 2 && children.length <= rows + 2) { 2638 const firstRowCells = children[0]?.children; 2639 if (firstRowCells && firstRowCells.length >= cols - 2 && firstRowCells.length <= cols + 2) { 2640 const actualRows = children.length; 2641 const actualCols = firstRowCells.length; 2642 const allCells: HTMLElement[] = []; 2643 for (let r = 0; r < actualRows; r++) { 2644 const cells = children[r].children; 2645 for (let c = 0; c < Math.min(actualCols, cells.length); c++) allCells.push(cells[c] as HTMLElement); 2646 } 2647 const emptyBg = detectEmptyBg(allCells); 2648 let valid = true; 2649 const result: boolean[][] = []; 2650 for (let r = 0; r < actualRows; r++) { 2651 const cells = children[r].children; 2652 if (cells.length < actualCols) { valid = false; break; } 2653 const rowData: boolean[] = []; 2654 for (let c = 0; c < actualCols; c++) rowData.push(isCellFilled(cells[c] as HTMLElement, emptyBg)); 2655 result.push(rowData); 2656 } 2657 if (valid) return result; 2658 } 2659 } 2660 } 2661 2662 // Strategy 3: heuristic scan for ANY container with many same-sized children 2663 const allElements = document.querySelectorAll("div, section, main, article"); 2664 for (const el of allElements) { 2665 const ch = el.children; 2666 if (ch.length >= 180 && ch.length <= 230) { 2667 const firstChild = ch[0] as HTMLElement; 2668 if (!firstChild) continue; 2669 const firstRect = firstChild.getBoundingClientRect(); 2670 if (firstRect.width < 5 || firstRect.height < 5) continue; 2671 let uniform = true; 2672 for (let i = 1; i < Math.min(10, ch.length); i++) { 2673 const r = (ch[i] as HTMLElement).getBoundingClientRect(); 2674 if (Math.abs(r.width - firstRect.width) > 2 || Math.abs(r.height - firstRect.height) > 2) { uniform = false; break; } 2675 } 2676 if (uniform) { 2677 const actualCols = cols; 2678 const actualRows = Math.round(ch.length / actualCols); 2679 const allCells = Array.from(ch).slice(0, actualRows * actualCols) as HTMLElement[]; 2680 const emptyBg = detectEmptyBg(allCells); 2681 const result: boolean[][] = []; 2682 for (let r = 0; r < actualRows; r++) { 2683 const rowData: boolean[] = []; 2684 for (let c = 0; c < actualCols; c++) { 2685 const cell = allCells[r * actualCols + c]; 2686 rowData.push(cell ? isCellFilled(cell, emptyBg) : false); 2687 } 2688 result.push(rowData); 2689 } 2690 return result; 2691 } 2692 } 2693 if (ch.length >= 18 && ch.length <= 22) { 2694 const firstRowCells = ch[0]?.children; 2695 if (firstRowCells && firstRowCells.length >= 8 && firstRowCells.length <= 12) { 2696 const rect = el.getBoundingClientRect(); 2697 if (rect.width > 50 && rect.height > 100) { 2698 const actualRows = ch.length; 2699 const actualCols = firstRowCells.length; 2700 const allCells: HTMLElement[] = []; 2701 for (let r = 0; r < actualRows; r++) { 2702 const cells = ch[r].children; 2703 for (let c = 0; c < Math.min(actualCols, cells.length); c++) allCells.push(cells[c] as HTMLElement); 2704 } 2705 const emptyBg = detectEmptyBg(allCells); 2706 let valid = true; 2707 const result: boolean[][] = []; 2708 for (let r = 0; r < actualRows; r++) { 2709 const cells = ch[r].children; 2710 if (cells.length < actualCols) { valid = false; break; } 2711 const rowData: boolean[] = []; 2712 for (let c = 0; c < actualCols; c++) rowData.push(isCellFilled(cells[c] as HTMLElement, emptyBg)); 2713 result.push(rowData); 2714 } 2715 if (valid) return result; 2716 } 2717 } 2718 } 2719 } 2720 2721 return null; 2722 }, { rows: GRID_ROWS, cols: GRID_COLS }); 2723 2724 return grid; 2725 } 2726 2727 private async sampleBackgroundColor( 2728 bounds: GridBounds, cellW: number, cellH: number 2729 ): Promise<[number, number, number] | null> { 2730 try { 2731 return await this.page.evaluate( 2732 ({ x, y, cellW, cellH }) => { 2733 const canvas = document.querySelector("canvas") as HTMLCanvasElement | null; 2734 if (!canvas) return null; 2735 const ctx = canvas.getContext("2d"); 2736 if (!ctx) return null; 2737 const px = Math.floor(x + cellW / 2); 2738 const py = Math.floor(y + cellH / 2); 2739 const pixel = ctx.getImageData(px, py, 1, 1).data; 2740 return [pixel[0], pixel[1], pixel[2]] as [number, number, number]; 2741 }, 2742 { x: bounds.x, y: bounds.y, cellW, cellH } 2743 ); 2744 } catch { 2745 return null; 2746 } 2747 } 2748 2749 // -- Start detection -- 2750 2751 private async detectStartMechanism(): Promise<{ 2752 mechanism: StartMechanism; 2753 startButton?: DriverCalibration["startButton"]; 2754 }> { 2755 const deadline = Date.now() + 30000; 2756 const budgetExceeded = () => Date.now() >= deadline; 2757 2758 try { 2759 const diag = await this.page.evaluate(() => ({ 2760 title: document.title, 2761 buttons: Array.from(document.querySelectorAll("button")).map(b => b.textContent?.trim()), 2762 canvases: Array.from(document.querySelectorAll("canvas")).length, 2763 bodySize: document.body?.innerHTML?.length ?? 0, 2764 })); 2765 this.log(`Page: "${diag.title}", ${diag.buttons.length} buttons, ${diag.canvases} canvases`); 2766 } catch { /* continue */ } 2767 2768 // Phase 1: Auto-start check 2769 { 2770 this.log("Phase 1: checking auto-start..."); 2771 const result = await this.detectVisualChange({ frames: 6, intervalMs: 200 }); 2772 if (result.changed) { 2773 const interactive = await this.verifyInteractivity(); 2774 if (interactive) { 2775 this.log("Auto-start detected and interactive"); 2776 return { mechanism: "auto" }; 2777 } 2778 this.log("Visual change but not interactive (animation?)"); 2779 } 2780 } 2781 2782 // Phase 2: DOM buttons FIRST (more reliable for DOM games, language-agnostic) 2783 if (!budgetExceeded()) { 2784 this.log("Phase 2: trying DOM buttons..."); 2785 const buttonResult = await this.tryDomButtons(budgetExceeded); 2786 if (buttonResult) return buttonResult; 2787 } 2788 2789 // Phase 3: Keyboard triggers 2790 if (!budgetExceeded()) { 2791 this.log("Phase 3: trying keyboard triggers..."); 2792 const keyResult = await this.tryKeyboardTriggers(budgetExceeded); 2793 if (keyResult) return keyResult; 2794 } 2795 2796 // Phase 4: Canvas clicks 2797 if (!budgetExceeded()) { 2798 this.log("Phase 4: trying canvas clicks..."); 2799 const canvasResult = await this.tryCanvasClicks(budgetExceeded); 2800 if (canvasResult) return canvasResult; 2801 } 2802 2803 // Phase 5: Retry all 2804 if (!budgetExceeded()) { 2805 const r2 = await this.tryDomButtons(budgetExceeded); 2806 if (r2) return r2; 2807 } 2808 if (!budgetExceeded()) { 2809 const r3 = await this.tryKeyboardTriggers(budgetExceeded); 2810 if (r3) return r3; 2811 } 2812 if (!budgetExceeded()) { 2813 const r4 = await this.tryCanvasClicks(budgetExceeded); 2814 if (r4) return r4; 2815 } 2816 2817 return { mechanism: "unknown" }; 2818 } 2819 2820 private async detectVisualChange( 2821 options?: { frames?: number; intervalMs?: number; before?: Buffer } 2822 ): Promise<{ changed: boolean }> { 2823 const FRAMES = options?.frames ?? 6; 2824 const INTERVAL = options?.intervalMs ?? 200; 2825 2826 const screenshots: Buffer[] = []; 2827 for (let i = 0; i < FRAMES; i++) { 2828 screenshots.push(await this.page.screenshot()); 2829 if (i < FRAMES - 1) await this.page.waitForTimeout(INTERVAL); 2830 } 2831 2832 let changed = false; 2833 2834 if (options?.before) { 2835 for (const shot of screenshots) { 2836 if (!options.before.equals(shot)) { changed = true; break; } 2837 } 2838 } else { 2839 // Compare consecutive frames 2840 for (let f = 0; f < screenshots.length - 1; f++) { 2841 if (!screenshots[f].equals(screenshots[f + 1])) { changed = true; break; } 2842 } 2843 // Also check with a late frame 2844 if (!changed) { 2845 await this.page.waitForTimeout(1200); 2846 const lateFrame = await this.page.screenshot(); 2847 if (!screenshots[0].equals(lateFrame)) changed = true; 2848 } 2849 } 2850 2851 return { changed }; 2852 } 2853 2854 private async verifyInteractivity(): Promise<boolean> { 2855 try { 2856 await this.page.waitForTimeout(200); 2857 2858 const baseline = await this.page.screenshot(); 2859 const domBefore = await this.page.evaluate(() => { 2860 const candidates = document.querySelectorAll( 2861 '[class*="board"], [class*="grid"], [class*="field"], [id*="board"], [id*="grid"], table' 2862 ); 2863 let best = ""; 2864 for (const el of candidates) { 2865 const snap = Array.from(el.children).map(c => 2866 (c as HTMLElement).className + (c as HTMLElement).style.cssText 2867 ).join("|"); 2868 if (snap.length > best.length) best = snap; 2869 } 2870 if (!best) best = document.body.innerHTML.substring(0, 5000); 2871 return best; 2872 }); 2873 2874 for (const key of ["ArrowLeft", "ArrowRight", "ArrowDown"]) { 2875 await this.page.keyboard.press(key); 2876 await this.page.waitForTimeout(100); 2877 2878 const after = await this.page.screenshot(); 2879 if (!baseline.equals(after)) return true; 2880 2881 const domAfter = await this.page.evaluate(() => { 2882 const candidates = document.querySelectorAll( 2883 '[class*="board"], [class*="grid"], [class*="field"], [id*="board"], [id*="grid"], table' 2884 ); 2885 let best = ""; 2886 for (const el of candidates) { 2887 const snap = Array.from(el.children).map(c => 2888 (c as HTMLElement).className + (c as HTMLElement).style.cssText 2889 ).join("|"); 2890 if (snap.length > best.length) best = snap; 2891 } 2892 if (!best) best = document.body.innerHTML.substring(0, 5000); 2893 return best; 2894 }); 2895 if (domAfter !== domBefore) return true; 2896 } 2897 return false; 2898 } catch { 2899 return false; 2900 } 2901 } 2902 2903 private async tryDomButtons( 2904 budgetExceeded: () => boolean 2905 ): Promise<{ mechanism: StartMechanism; startButton?: DriverCalibration["startButton"] } | null> { 2906 try { 2907 const elementInfos = await this.page.evaluate(() => { 2908 const seen = new Set<Element>(); 2909 const results: Array<{ 2910 index: number; text: string; x: number; y: number; 2911 width: number; height: number; area: number; centerDist: number; 2912 selector: string; hasBackground: boolean; 2913 }> = []; 2914 2915 const clickableSelector = 2916 'button, a, [role="button"], [onclick], input[type="button"], input[type="submit"]'; 2917 for (const el of document.querySelectorAll(clickableSelector)) { 2918 if (!seen.has(el)) seen.add(el); 2919 } 2920 2921 const allEls = document.querySelectorAll("*"); 2922 for (const el of allEls) { 2923 if (seen.has(el)) continue; 2924 try { 2925 const style = window.getComputedStyle(el); 2926 if (style.cursor === "pointer") seen.add(el); 2927 } catch { /* skip */ } 2928 } 2929 2930 const pageW = window.innerWidth; 2931 const pageH = window.innerHeight; 2932 const pageCenterX = pageW / 2; 2933 const pageCenterY = pageH / 2; 2934 2935 let idx = 0; 2936 for (const el of seen) { 2937 const rect = el.getBoundingClientRect(); 2938 if (rect.width < 5 || rect.height < 5) continue; 2939 if (rect.top > pageH || rect.left > pageW) continue; 2940 if (rect.width > pageW * 0.8 && rect.height > pageH * 0.8) continue; 2941 2942 const cx = rect.left + rect.width / 2; 2943 const cy = rect.top + rect.height / 2; 2944 const centerDist = Math.sqrt((cx - pageCenterX) ** 2 + (cy - pageCenterY) ** 2); 2945 2946 let hasBackground = false; 2947 try { 2948 const style = window.getComputedStyle(el as HTMLElement); 2949 const bg = style.backgroundColor; 2950 if (bg && bg !== "transparent" && bg !== "rgba(0, 0, 0, 0)") hasBackground = true; 2951 } catch { /* skip */ } 2952 2953 let selector = ""; 2954 if (el.id) selector = `#${el.id}`; 2955 else if ((el as HTMLElement).className) { 2956 const cls = (el as HTMLElement).className.toString().split(" ")[0]; 2957 if (cls) selector = `${el.tagName.toLowerCase()}.${cls}`; 2958 } 2959 if (!selector) selector = `${el.tagName.toLowerCase()}:nth-of-type(${idx + 1})`; 2960 2961 results.push({ 2962 index: idx, text: (el.textContent || "").trim().slice(0, 50), 2963 x: Math.round(cx), y: Math.round(cy), 2964 width: rect.width, height: rect.height, 2965 area: rect.width * rect.height, centerDist, selector, hasBackground, 2966 }); 2967 idx++; 2968 } 2969 2970 results.sort((a, b) => { 2971 if (a.hasBackground !== b.hasBackground) return a.hasBackground ? -1 : 1; 2972 if (Math.abs(b.area - a.area) > 100) return b.area - a.area; 2973 return a.centerDist - b.centerDist; 2974 }); 2975 2976 return results; 2977 }); 2978 2979 for (const info of elementInfos) { 2980 if (budgetExceeded()) break; 2981 try { 2982 const wasVisible = await this.page.evaluate( 2983 ({ x, y }) => document.elementFromPoint(x, y) !== null, 2984 { x: info.x, y: info.y } 2985 ); 2986 if (!wasVisible) continue; 2987 2988 const before = await this.page.screenshot(); 2989 this.log(`Clicking "${info.text}" (${info.selector}) at (${info.x},${info.y})`); 2990 await this.page.mouse.click(info.x, info.y); 2991 // Wait 300ms for JS to initialize after button click 2992 await this.page.waitForTimeout(300); 2993 2994 const result = await this.detectVisualChange({ frames: 3, intervalMs: 100, before }); 2995 if (result.changed) { 2996 // Check for immediate game over text (false start rejection) 2997 await this.page.waitForTimeout(200); 2998 const gameOverText = await this.detectGameOverText(); 2999 if (gameOverText) { 3000 this.log(`Button "${info.text}" triggered game over immediately, rejecting`); 3001 continue; 3002 } 3003 3004 const interactive = await this.verifyInteractivity(); 3005 if (!interactive) { 3006 this.log(`Button "${info.text}" not interactive, continuing`); 3007 try { await this.page.keyboard.press("Escape"); await this.page.waitForTimeout(50); } catch {} 3008 continue; 3009 } 3010 3011 const disappeared = await this.page.evaluate( 3012 ({ selector }) => { 3013 if (!selector) return false; 3014 try { 3015 const el = document.querySelector(selector); 3016 if (!el) return true; 3017 const rect = el.getBoundingClientRect(); 3018 return rect.width === 0 || rect.height === 0; 3019 } catch { return false; } 3020 }, 3021 { selector: info.selector } 3022 ); 3023 3024 return { 3025 mechanism: "button", 3026 startButton: { 3027 selector: info.selector, text: info.text, 3028 disappeared, position: { x: info.x, y: info.y }, 3029 }, 3030 }; 3031 } 3032 3033 try { await this.page.keyboard.press("Escape"); await this.page.waitForTimeout(50); } catch {} 3034 } catch { /* continue */ } 3035 } 3036 } catch { /* phase failed */ } 3037 return null; 3038 } 3039 3040 private async tryKeyboardTriggers( 3041 budgetExceeded: () => boolean 3042 ): Promise<{ mechanism: StartMechanism } | null> { 3043 const mechanismMap: Record<string, StartMechanism> = { 3044 Enter: "enter", 3045 Space: "space", 3046 ArrowDown: "anykey", 3047 z: "anykey", 3048 }; 3049 3050 for (const key of ["Enter", "Space", "ArrowDown", "z"]) { 3051 if (budgetExceeded()) break; 3052 try { 3053 const before = await this.page.screenshot(); 3054 await this.page.keyboard.press(key); 3055 await this.page.waitForTimeout(100); 3056 3057 const result = await this.detectVisualChange({ frames: 3, intervalMs: 100, before }); 3058 if (result.changed) { 3059 const interactive = await this.verifyInteractivity(); 3060 if (interactive) return { mechanism: mechanismMap[key] }; 3061 } 3062 } catch { /* continue */ } 3063 } 3064 3065 // Combo: click then key 3066 for (const key of ["Enter", "Space"]) { 3067 if (budgetExceeded()) break; 3068 try { 3069 const before = await this.page.screenshot(); 3070 const canvas = this.page.locator("canvas").first(); 3071 if ((await canvas.count()) > 0) await canvas.click(); 3072 else { 3073 const vp = this.page.viewportSize(); 3074 if (vp) await this.page.mouse.click(vp.width / 2, vp.height / 2); 3075 } 3076 await this.page.waitForTimeout(100); 3077 await this.page.keyboard.press(key); 3078 await this.page.waitForTimeout(100); 3079 3080 const result = await this.detectVisualChange({ frames: 3, intervalMs: 100, before }); 3081 if (result.changed) { 3082 const interactive = await this.verifyInteractivity(); 3083 if (interactive) return { mechanism: mechanismMap[key] }; 3084 } 3085 } catch { /* continue */ } 3086 } 3087 3088 return null; 3089 } 3090 3091 private async tryCanvasClicks( 3092 budgetExceeded: () => boolean 3093 ): Promise<{ mechanism: StartMechanism; startButton?: DriverCalibration["startButton"] } | null> { 3094 let targetBox: { x: number; y: number; width: number; height: number } | null = null; 3095 3096 try { 3097 const canvas = this.page.locator("canvas").first(); 3098 if ((await canvas.count()) > 0) targetBox = await canvas.boundingBox(); 3099 } catch { /* no canvas */ } 3100 3101 if (!targetBox) { 3102 const vp = this.page.viewportSize(); 3103 if (vp) targetBox = { x: 0, y: 0, width: vp.width, height: vp.height }; 3104 } 3105 if (!targetBox) return null; 3106 3107 const cx = targetBox.x + targetBox.width / 2; 3108 const cy = targetBox.y + targetBox.height / 2; 3109 3110 const positions = [ 3111 { x: cx, y: cy, label: "center" }, 3112 { x: cx, y: targetBox.y + targetBox.height * 0.25, label: "upper" }, 3113 { x: cx, y: targetBox.y + targetBox.height * 0.75, label: "lower" }, 3114 ]; 3115 3116 for (let row = 0; row < 3; row++) { 3117 for (let col = 0; col < 3; col++) { 3118 if (row === 1 && col === 1) continue; 3119 positions.push({ 3120 x: targetBox.x + targetBox.width * (col + 0.5) / 3, 3121 y: targetBox.y + targetBox.height * (row + 0.5) / 3, 3122 label: `grid_${row}_${col}`, 3123 }); 3124 } 3125 } 3126 3127 for (const pos of positions) { 3128 if (budgetExceeded()) break; 3129 try { 3130 const before = await this.page.screenshot(); 3131 await this.page.mouse.click(pos.x, pos.y); 3132 await this.page.waitForTimeout(300); 3133 3134 const result = await this.detectVisualChange({ frames: 3, intervalMs: 100, before }); 3135 if (result.changed) { 3136 const interactive = await this.verifyInteractivity(); 3137 if (interactive) { 3138 return { 3139 mechanism: "click_canvas", 3140 startButton: { 3141 selector: "canvas", 3142 text: `canvas click at ${pos.label}`, 3143 disappeared: false, 3144 position: { x: Math.round(pos.x), y: Math.round(pos.y) }, 3145 }, 3146 }; 3147 } 3148 } 3149 } catch { /* continue */ } 3150 } 3151 3152 return null; 3153 } 3154 3155 private async recalibrateWithRetry( 3156 currentStart: StartMechanism, 3157 currentGrid: GridBounds | null 3158 ): Promise<{ 3159 renderer: RendererType; gridBounds: GridBounds | null; 3160 cellWidth: number; cellHeight: number; startMechanism: StartMechanism; 3161 startButton?: DriverCalibration["startButton"]; 3162 }> { 3163 let startMechanism: StartMechanism = currentStart; 3164 let gridResult = { renderer: "unknown" as RendererType, gridBounds: currentGrid, cellWidth: 0, cellHeight: 0 }; 3165 let startButton: DriverCalibration["startButton"] | undefined; 3166 3167 const attempts: Array<{ name: StartMechanism; action: () => Promise<void> }> = [ 3168 { name: "button", action: async () => { 3169 const btn = this.page.locator("button").first(); 3170 if ((await btn.count()) > 0) await btn.click(); 3171 }}, 3172 { name: "click_canvas", action: async () => { 3173 const canvas = this.page.locator("canvas").first(); 3174 if ((await canvas.count()) > 0) await canvas.click(); 3175 }}, 3176 { name: "click_canvas", action: async () => { 3177 await this.page.locator("body").click({ position: { x: 200, y: 200 } }); 3178 }}, 3179 { name: "enter", action: async () => { await this.page.keyboard.press("Enter"); } }, 3180 { name: "space", action: async () => { await this.page.keyboard.press("Space"); } }, 3181 { name: "anykey", action: async () => { await this.page.keyboard.press("ArrowDown"); } }, 3182 ]; 3183 3184 for (const attempt of attempts) { 3185 try { 3186 const before = await this.page.screenshot(); 3187 await attempt.action(); 3188 await this.page.waitForTimeout(300); 3189 3190 if (startMechanism === "unknown") { 3191 const result = await this.detectVisualChange({ frames: 3, intervalMs: 100, before }); 3192 if (result.changed) { 3193 const interactive = await this.verifyInteractivity(); 3194 if (interactive) { 3195 startMechanism = attempt.name; 3196 } 3197 } 3198 } 3199 3200 if (!gridResult.gridBounds) { 3201 const detected = await this.detectGrid(); 3202 if (detected.gridBounds) gridResult = detected; 3203 } 3204 3205 if (startMechanism !== "unknown" && gridResult.gridBounds) break; 3206 } catch { /* continue */ } 3207 } 3208 3209 return { ...gridResult, startMechanism, startButton }; 3210 } 3211 3212 // -- Grid detection -- 3213 3214 private async detectGrid(): Promise<{ 3215 renderer: RendererType; gridBounds: GridBounds | null; 3216 cellWidth: number; cellHeight: number; 3217 }> { 3218 // Check for canvas 3219 try { 3220 const canvasCount = await this.page.locator("canvas").count(); 3221 if (canvasCount > 0) { 3222 const bounds = await this.page.locator("canvas").first().boundingBox(); 3223 if (bounds) { 3224 const canvasDims = await this.page.evaluate(() => { 3225 const c = document.querySelector("canvas"); 3226 return c ? { width: c.width, height: c.height } : null; 3227 }); 3228 3229 const internalW = canvasDims ? canvasDims.width : bounds.width; 3230 const internalH = canvasDims ? canvasDims.height : bounds.height; 3231 const ratio = internalH / internalW; 3232 3233 let gridX = 0, gridY = 0, gridW = internalW, gridH = internalH; 3234 3235 if (ratio >= 1.5 && ratio <= 2.5) { 3236 // Whole canvas is the grid 3237 } else if (ratio < 1.5) { 3238 gridW = internalH / 2; 3239 gridH = internalH; 3240 gridX = 0; 3241 gridY = 0; 3242 } 3243 3244 return { 3245 renderer: "canvas" as RendererType, 3246 gridBounds: { x: gridX, y: gridY, width: gridW, height: gridH }, 3247 cellWidth: gridW / 10, 3248 cellHeight: gridH / 20, 3249 }; 3250 } 3251 } 3252 } catch { /* continue */ } 3253 3254 // Check for DOM-based grid 3255 try { 3256 const domResult = await this.page.evaluate(() => { 3257 const tables = document.querySelectorAll("table"); 3258 for (const table of tables) { 3259 const rows = table.querySelectorAll("tr"); 3260 if (rows.length >= 18) { 3261 const firstRow = rows[0].querySelectorAll("td"); 3262 if (firstRow.length >= 8 && firstRow.length <= 12) { 3263 const rect = table.getBoundingClientRect(); 3264 return { type: "dom" as const, bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, rows: rows.length, cols: firstRow.length }; 3265 } 3266 } 3267 } 3268 3269 const containers = document.querySelectorAll( 3270 '[class*="board"], [class*="grid"], [class*="field"], [id*="board"], [id*="grid"], [id*="field"]' 3271 ); 3272 for (const container of containers) { 3273 const children = container.children; 3274 if (children.length >= 180 && children.length <= 230) { 3275 const rect = container.getBoundingClientRect(); 3276 return { type: "dom" as const, bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, rows: Math.round(children.length / 10), cols: 10 }; 3277 } 3278 if (children.length >= 18 && children.length <= 22) { 3279 const firstRowCells = children[0].children; 3280 if (firstRowCells.length >= 8 && firstRowCells.length <= 12) { 3281 const rect = container.getBoundingClientRect(); 3282 return { type: "dom" as const, bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, rows: children.length, cols: firstRowCells.length }; 3283 } 3284 } 3285 } 3286 3287 // Heuristic scan 3288 const allElements = document.querySelectorAll("div, section, main, article"); 3289 for (const el of allElements) { 3290 const ch = el.children; 3291 if (ch.length >= 180 && ch.length <= 230) { 3292 const firstChild = ch[0] as HTMLElement; 3293 if (!firstChild) continue; 3294 const firstRect = firstChild.getBoundingClientRect(); 3295 if (firstRect.width < 5 || firstRect.height < 5) continue; 3296 let uniform = true; 3297 for (let i = 1; i < Math.min(10, ch.length); i++) { 3298 const r = (ch[i] as HTMLElement).getBoundingClientRect(); 3299 if (Math.abs(r.width - firstRect.width) > 2 || Math.abs(r.height - firstRect.height) > 2) { uniform = false; break; } 3300 } 3301 if (uniform) { 3302 const rect = el.getBoundingClientRect(); 3303 return { type: "dom" as const, bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, rows: Math.round(ch.length / 10), cols: 10 }; 3304 } 3305 } 3306 if (ch.length >= 18 && ch.length <= 22) { 3307 const firstRowCells = ch[0]?.children; 3308 if (firstRowCells && firstRowCells.length >= 8 && firstRowCells.length <= 12) { 3309 const rect = el.getBoundingClientRect(); 3310 if (rect.width > 50 && rect.height > 100) { 3311 return { type: "dom" as const, bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, rows: ch.length, cols: firstRowCells.length }; 3312 } 3313 } 3314 } 3315 } 3316 3317 return null; 3318 }); 3319 3320 if (domResult) { 3321 return { 3322 renderer: "dom", 3323 gridBounds: domResult.bounds, 3324 cellWidth: domResult.bounds.width / domResult.cols, 3325 cellHeight: domResult.bounds.height / domResult.rows, 3326 }; 3327 } 3328 } catch { /* continue */ } 3329 3330 // Check for SVG 3331 try { 3332 const svgCount = await this.page.locator("svg").count(); 3333 if (svgCount > 0) { 3334 const bounds = await this.page.locator("svg").first().boundingBox(); 3335 if (bounds) { 3336 return { 3337 renderer: "svg", 3338 gridBounds: { x: bounds.x, y: bounds.y, width: bounds.width, height: bounds.height }, 3339 cellWidth: bounds.width / 10, 3340 cellHeight: bounds.height / 20, 3341 }; 3342 } 3343 } 3344 } catch { /* continue */ } 3345 3346 return { renderer: "unknown", gridBounds: null, cellWidth: 0, cellHeight: 0 }; 3347 } 3348 3349 // -- Control detection -- 3350 3351 private async detectControls(): Promise<Controls> { 3352 const controls: Controls = { ...DEFAULT_CONTROLS }; 3353 3354 try { 3355 const pageText = await this.page.evaluate(() => document.body.innerText.toLowerCase()); 3356 if (pageText.includes("wasd") || pageText.includes("w,a,s,d")) { 3357 controls.left = "a"; controls.right = "d"; controls.down = "s"; controls.rotate = "w"; 3358 } 3359 if (/z\s*(=|:)?\s*rotate/i.test(pageText)) controls.rotate = "z"; 3360 if (/x\s*(=|:)?\s*rotate/i.test(pageText)) controls.rotate = "x"; 3361 } catch { /* use defaults */ } 3362 3363 try { 3364 const before = await this.page.screenshot(); 3365 await this.page.keyboard.press(controls.left); 3366 await this.page.waitForTimeout(200); 3367 const after = await this.page.screenshot(); 3368 if (Buffer.from(before).equals(Buffer.from(after))) { 3369 await this.page.keyboard.press("a"); 3370 await this.page.waitForTimeout(200); 3371 const afterA = await this.page.screenshot(); 3372 if (!Buffer.from(before).equals(Buffer.from(afterA))) { 3373 controls.left = "a"; controls.right = "d"; controls.down = "s"; controls.rotate = "w"; 3374 } 3375 } 3376 } catch { /* use defaults */ } 3377 3378 try { 3379 const before = await this.page.screenshot(); 3380 await this.page.keyboard.press(controls.rotate); 3381 await this.page.waitForTimeout(200); 3382 const after = await this.page.screenshot(); 3383 if (Buffer.from(before).equals(Buffer.from(after))) { 3384 for (const alt of ["z", "x", "ArrowUp"]) { 3385 if (alt === controls.rotate) continue; 3386 await this.page.keyboard.press(alt); 3387 await this.page.waitForTimeout(200); 3388 const afterAlt = await this.page.screenshot(); 3389 if (!Buffer.from(before).equals(Buffer.from(afterAlt))) { 3390 controls.rotate = alt; 3391 break; 3392 } 3393 } 3394 } 3395 } catch { /* use defaults */ } 3396 3397 return controls; 3398 } 3399 3400 // -- Score/Level element detection -- 3401 3402 private async detectScoreElement(): Promise<string | null> { 3403 try { 3404 return await this.page.evaluate(() => { 3405 function _buildSelector(el: Element): string | null { 3406 if (el.id) return `#${el.id}`; 3407 if ((el as HTMLElement).className) { 3408 const cls = (el as HTMLElement).className.split(" ")[0]; 3409 if (cls) return `.${cls}`; 3410 } 3411 return null; 3412 } 3413 3414 const allElements = document.querySelectorAll("*"); 3415 3416 // Strategy 1: child near "score"/"puntuacion"/"puntaje" text with only a number 3417 for (const el of allElements) { 3418 const text = ((el as HTMLElement).innerText || "").toLowerCase(); 3419 if ((text.includes("score") || text.includes("puntuacion") || text.includes("puntaje") || text.includes("puntos")) && el.children.length < 10) { 3420 const descendants = el.querySelectorAll("span, div, p, td, strong, em, b"); 3421 for (const desc of descendants) { 3422 const descText = desc.textContent?.trim() || ""; 3423 if (/^\d+$/.test(descText) && desc.children.length === 0) { 3424 const sel = _buildSelector(desc); 3425 if (sel) return sel; 3426 } 3427 } 3428 const next = el.nextElementSibling; 3429 if (next) { 3430 const nextText = next.textContent?.trim() || ""; 3431 if (/^\d+$/.test(nextText)) { 3432 const sel = _buildSelector(next); 3433 if (sel) return sel; 3434 } 3435 } 3436 const sel = _buildSelector(el); 3437 if (sel) return sel; 3438 } 3439 } 3440 3441 // Strategy 2: labeled text like "Score: 123" 3442 for (const el of allElements) { 3443 if (el.children.length > 3) continue; 3444 const text = (el as HTMLElement).textContent?.trim() || ""; 3445 const scoreMatch = text.match(/(?:score|puntuacion|puntaje|puntos)\s*[:\-=]?\s*(\d+)/i); 3446 if (scoreMatch) { 3447 const sel = _buildSelector(el); 3448 if (sel) return sel; 3449 } 3450 } 3451 3452 // Strategy 3: leaf elements with just a number 3453 const candidates: HTMLElement[] = []; 3454 for (const el of allElements) { 3455 const text = (el as HTMLElement).textContent?.trim() || ""; 3456 if (/^\d+$/.test(text) && el.children.length === 0) candidates.push(el as HTMLElement); 3457 } 3458 if (candidates.length > 0) { 3459 const sel = _buildSelector(candidates[0]); 3460 if (sel) return sel; 3461 } 3462 3463 return null; 3464 }); 3465 } catch { 3466 return null; 3467 } 3468 } 3469 3470 private async detectLevelElement(): Promise<string | null> { 3471 try { 3472 return await this.page.evaluate(() => { 3473 function _buildSelector(el: Element): string | null { 3474 if (el.id) return `#${el.id}`; 3475 if ((el as HTMLElement).className) { 3476 const cls = (el as HTMLElement).className.split(" ")[0]; 3477 if (cls) return `.${cls}`; 3478 } 3479 return null; 3480 } 3481 3482 const allElements = document.querySelectorAll("*"); 3483 for (const el of allElements) { 3484 const text = ((el as HTMLElement).innerText || "").toLowerCase(); 3485 if ((text.includes("level") || text.includes("nivel")) && el.children.length < 5) { 3486 const sel = _buildSelector(el); 3487 if (sel) return sel; 3488 } 3489 } 3490 return null; 3491 }); 3492 } catch { 3493 return null; 3494 } 3495 } 3496 3497 // -- Grid confidence -- 3498 3499 private async measureGridConfidence(cal: DriverCalibration): Promise<number> { 3500 if (!cal.gridBounds) return 0; 3501 3502 let successes = 0; 3503 let attempts = 0; 3504 const pollCount = 6; 3505 let lastGrid: Grid | null = null; 3506 let gridChanged = false; 3507 3508 for (let i = 0; i < pollCount; i++) { 3509 attempts++; 3510 try { 3511 const snap = await this.readGrid(); 3512 if (snap.grid) { 3513 successes++; 3514 if (lastGrid) { 3515 if (this.gridsAreDifferent(snap.grid, lastGrid)) gridChanged = true; 3516 } 3517 lastGrid = snap.grid; 3518 } 3519 } catch { /* read failed */ } 3520 await this.page.waitForTimeout(500); 3521 } 3522 3523 if (successes > 0 && !gridChanged && cal.startMechanism !== "unknown") { 3524 const additionalStarts: Array<{ name: string; action: () => Promise<void> }> = [ 3525 { name: "space", action: async () => { await this.page.keyboard.press("Space"); } }, 3526 { name: "enter", action: async () => { await this.page.keyboard.press("Enter"); } }, 3527 { name: "click", action: async () => { 3528 const canvas = this.page.locator("canvas").first(); 3529 if ((await canvas.count()) > 0) await canvas.click(); 3530 else await this.page.locator("body").click({ position: { x: 200, y: 200 } }); 3531 }}, 3532 ]; 3533 3534 for (const start of additionalStarts) { 3535 try { 3536 await start.action(); 3537 await this.page.waitForTimeout(1500); 3538 const snap = await this.readGrid(); 3539 if (snap.grid && lastGrid) { 3540 if (this.gridsAreDifferent(snap.grid, lastGrid)) { gridChanged = true; break; } 3541 lastGrid = snap.grid; 3542 } 3543 } catch { /* continue */ } 3544 } 3545 } 3546 3547 return attempts > 0 ? successes / attempts : 0; 3548 } 3549 3550 // -- Helpers -- 3551 3552 private extractScoreFromText(text: string | null): number[] { 3553 if (!text) return [0]; 3554 const labeledMatch = text.match(/(?:score|puntuacion|puntaje|puntos)\s*[:\-=]?\s*(\d+)/i); 3555 if (labeledMatch) return [parseInt(labeledMatch[1], 10)]; 3556 const allNumbers = (text.match(/\d+/g) || []).map(Number); 3557 return allNumbers.length > 0 ? allNumbers : [0]; 3558 } 3559 }