commit 63a3d148e5733c645300c806f4ffdafd4d344f71
parent 64fcaa825f5ab1a80912f80abc8e771b5b0a3a81
Author: Brian Graham <brian@buildingbetterteams.de>
Date: Sun, 22 Mar 2026 18:38:54 +0100
Add pipeline progress bar and show all registry papers in browser
- Dashboard shows segmented progress bar: v2 scanned, v1 rescan,
queued, no PDF — updates automatically with each deploy.
- Papers browser lists all 2,687 registry entries. Unscanned papers
show "--" for score/archetype and are not clickable.
- Pipeline stats added to dashboard.json (registry_total, v2_scanned,
v1_needs_rescan, has_text_no_scan, no_text, excluded).
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Diffstat:
7 files changed, 174 insertions(+), 16 deletions(-)
diff --git a/explorer/src/data.ts b/explorer/src/data.ts
@@ -23,13 +23,22 @@ export interface PaperIndex {
year: number;
venue: string;
tags: string[];
- score: number;
- archetype: string;
+ score: number | null;
+ archetype: string | null;
games: string[];
arxiv_id: string;
doi: string;
}
+export interface Pipeline {
+ registry_total: number;
+ v2_scanned: number;
+ v1_needs_rescan: number;
+ has_text_no_scan: number;
+ no_text: number;
+ excluded: number;
+}
+
// Full detail for per-paper pages (papers/{slug}.json)
export interface PaperDetail extends PaperIndex {
category_scores: Record<string, number>;
@@ -58,6 +67,7 @@ export interface Dashboard {
game_pcts: Record<string, number>;
archetype_counts: Record<string, number>;
tag_counts: Record<string, number>;
+ pipeline: Pipeline;
}
export interface TensionClaim {
diff --git a/explorer/src/style.css b/explorer/src/style.css
@@ -425,5 +425,66 @@ td.score {
.archetype.Mixed { background: rgba(139, 143, 163, 0.2); color: var(--text-dim); }
.archetype.Minimal { background: rgba(240, 101, 101, 0.2); color: var(--red); }
+/* Pipeline progress bar */
+.pipeline-bar {
+ background: var(--surface);
+ border: 1px solid var(--border);
+ border-radius: 8px;
+ padding: 1rem 1.25rem;
+ margin-bottom: 1.5rem;
+}
+.pipeline-header {
+ display: flex;
+ justify-content: space-between;
+ align-items: baseline;
+ margin-bottom: 0.5rem;
+}
+.pipeline-title {
+ font-size: 0.75rem;
+ font-weight: 600;
+ color: var(--text-dim);
+ text-transform: uppercase;
+ letter-spacing: 0.05em;
+}
+.pipeline-stat {
+ font-family: var(--font);
+ font-size: 0.8rem;
+ color: var(--text);
+}
+.pipeline-track {
+ display: flex;
+ height: 12px;
+ border-radius: 6px;
+ overflow: hidden;
+ background: var(--border);
+}
+.pipeline-seg {
+ height: 100%;
+ transition: width 0.3s;
+}
+.pipeline-seg.scanned { background: var(--green); }
+.pipeline-seg.v1 { background: var(--accent); }
+.pipeline-seg.queued { background: var(--yellow); }
+.pipeline-seg.notext { background: var(--gray); }
+.pipeline-legend {
+ display: flex;
+ gap: 1.25rem;
+ margin-top: 0.5rem;
+ font-size: 0.75rem;
+ color: var(--text-dim);
+}
+.pipeline-dot {
+ display: inline-block;
+ width: 8px;
+ height: 8px;
+ border-radius: 50%;
+ margin-right: 4px;
+ vertical-align: middle;
+}
+.pipeline-dot.scanned { background: var(--green); }
+.pipeline-dot.v1 { background: var(--accent); }
+.pipeline-dot.queued { background: var(--yellow); }
+.pipeline-dot.notext { background: var(--gray); }
+
/* Year trend chart */
.trend-chart { margin-top: 0.5rem; }
diff --git a/explorer/src/views/dashboard.ts b/explorer/src/views/dashboard.ts
@@ -1,14 +1,46 @@
-import { loadDashboard, type Dashboard } from '../data';
+import { loadDashboard, type Dashboard, type Pipeline } from '../data';
import { renderHistogram } from '../components/histogram';
import { renderBarChart } from '../components/bar-chart';
+function renderProgressBar(p: Pipeline): string {
+ const total = p.registry_total;
+ const scannedPct = (p.v2_scanned / total * 100).toFixed(1);
+ const v1Pct = (p.v1_needs_rescan / total * 100).toFixed(1);
+ const textPct = (p.has_text_no_scan / total * 100).toFixed(1);
+ const noPct = (p.no_text / total * 100).toFixed(1);
+
+ return `<div class="pipeline-bar">
+ <div class="pipeline-header">
+ <span class="pipeline-title">Survey Progress</span>
+ <span class="pipeline-stat">${p.v2_scanned} of ${total} papers scanned (${scannedPct}%)</span>
+ </div>
+ <div class="pipeline-track">
+ <div class="pipeline-seg scanned" style="width:${scannedPct}%" title="V2 scanned: ${p.v2_scanned}"></div>
+ <div class="pipeline-seg v1" style="width:${v1Pct}%" title="V1 needs rescan: ${p.v1_needs_rescan}"></div>
+ <div class="pipeline-seg queued" style="width:${textPct}%" title="Text ready, awaiting scan: ${p.has_text_no_scan}"></div>
+ <div class="pipeline-seg notext" style="width:${noPct}%" title="No text available: ${p.no_text}"></div>
+ </div>
+ <div class="pipeline-legend">
+ <span><span class="pipeline-dot scanned"></span>Scanned (${p.v2_scanned})</span>
+ <span><span class="pipeline-dot v1"></span>V1 rescan (${p.v1_needs_rescan})</span>
+ <span><span class="pipeline-dot queued"></span>Queued (${p.has_text_no_scan})</span>
+ <span><span class="pipeline-dot notext"></span>No PDF (${p.no_text})</span>
+ </div>
+ </div>`;
+}
+
export async function renderDashboard(app: HTMLElement) {
app.innerHTML = '<div class="spinner"></div>';
const agg = await loadDashboard();
const topGame = Object.entries(agg.game_pcts).sort((a, b) => b[1] - a[1])[0];
+ const p = agg.pipeline;
+ const scanPct = Math.round(p.v2_scanned / p.registry_total * 100);
+
app.innerHTML = `
+ ${renderProgressBar(p)}
+
<div class="cards">
<div class="card"><div class="label">Papers Analyzed</div><div class="value">${agg.n}</div></div>
<div class="card"><div class="label">Median Score</div><div class="value yellow">${agg.median}%</div></div>
diff --git a/explorer/src/views/paper-detail.ts b/explorer/src/views/paper-detail.ts
@@ -81,7 +81,7 @@ export async function renderPaperDetail(app: HTMLElement, slug: string) {
<div class="paper-header">
<h2>${paper.title}</h2>
<div class="paper-meta">
- <span style="color:${scoreColor(paper.score)};font-family:var(--font);font-weight:700">${paper.score}%</span>
+ <span style="color:${scoreColor(paper.score!)};font-family:var(--font);font-weight:700">${paper.score}%</span>
<span>${paper.year || ''}</span>
<span>${paper.venue || ''}</span>
<span class="archetype ${paper.archetype}">${paper.archetype}</span>
diff --git a/explorer/src/views/papers.ts b/explorer/src/views/papers.ts
@@ -15,7 +15,7 @@ export async function renderPapers(app: HTMLElement) {
const [papers, dashboard] = await Promise.all([loadPapersIndex(), loadDashboard()]);
app.innerHTML = '';
- const archetypes = [...new Set(papers.map(p => p.archetype))].sort();
+ const archetypes = [...new Set(papers.map(p => p.archetype).filter((a): a is string => a != null))].sort();
const tags = Object.keys(dashboard.tag_counts).sort();
let filtered = [...papers];
@@ -26,7 +26,8 @@ export async function renderPapers(app: HTMLElement) {
if (state.year && p.year !== parseInt(state.year)) return false;
if (state.archetype && p.archetype !== state.archetype) return false;
if (state.tag && !p.tags.includes(state.tag)) return false;
- if (p.score < state.minScore || p.score > state.maxScore) return false;
+ if (p.score != null && (p.score < state.minScore || p.score > state.maxScore)) return false;
+ if (p.score == null && state.minScore > 0) return false;
return true;
});
updateFilterCount(filtersEl, filtered.length, papers.length);
@@ -42,14 +43,16 @@ export async function renderPapers(app: HTMLElement) {
const columns: Column<PaperIndex>[] = [
{ key: 'title', label: 'Title', render: p => p.title.length > 70 ? p.title.slice(0, 67) + '...' : p.title, sortValue: p => p.title },
{ key: 'year', label: 'Year', render: p => String(p.year || ''), sortValue: p => p.year || 0 },
- { key: 'score', label: 'Score', render: p => `<span style="color:${scoreColor(p.score)}">${p.score}%</span>`, sortValue: p => p.score },
+ { key: 'score', label: 'Score', render: p => p.score != null ? `<span style="color:${scoreColor(p.score)}">${p.score}%</span>` : '<span style="color:var(--gray)">--</span>', sortValue: p => p.score ?? -1 },
{ key: 'venue', label: 'Venue', render: p => (p.venue || '').length > 20 ? (p.venue || '').slice(0, 17) + '...' : (p.venue || ''), sortValue: p => p.venue || '' },
- { key: 'archetype', label: 'Type', render: p => `<span class="archetype ${p.archetype}">${p.archetype}</span>`, sortValue: p => p.archetype },
+ { key: 'archetype', label: 'Type', render: p => p.archetype ? `<span class="archetype ${p.archetype}">${p.archetype}</span>` : '<span style="color:var(--gray)">--</span>', sortValue: p => p.archetype || '' },
];
function renderTable() {
tableContainer.innerHTML = '';
- const el = renderSortableTable(filtered, columns, p => navigate(`/paper/${p.id}`));
+ const el = renderSortableTable(filtered, columns, p => {
+ if (p.score != null) navigate(`/paper/${p.id}`);
+ });
tableContainer.appendChild(el);
}
diff --git a/explorer/tests/explorer.spec.ts b/explorer/tests/explorer.spec.ts
@@ -6,8 +6,8 @@ test.describe('Dashboard', () => {
await expect(page.locator('.card .value').first()).toBeVisible({ timeout: 10000 });
const cards = page.locator('.card');
await expect(cards).toHaveCount(4);
- await expect(cards.nth(0).locator('.value')).toHaveText('467');
- await expect(cards.nth(1).locator('.value')).toHaveText('50%');
+ await expect(cards.nth(0).locator('.value')).toHaveText('741');
+ await expect(cards.nth(1).locator('.value')).toHaveText('48%');
});
test('shows spinner then content', async ({ page }) => {
@@ -37,6 +37,13 @@ test.describe('Dashboard', () => {
await expect(page.locator('.trend-chart')).toBeVisible();
});
+ test('shows pipeline progress bar', async ({ page }) => {
+ await page.goto('/');
+ await expect(page.locator('.pipeline-bar')).toBeVisible({ timeout: 10000 });
+ await expect(page.locator('.pipeline-stat')).toContainText('of');
+ await expect(page.locator('.pipeline-seg.scanned')).toBeVisible();
+ });
+
test('shows named games', async ({ page }) => {
await page.goto('/');
await expect(page.locator('.card .value').first()).toBeVisible({ timeout: 10000 });
@@ -49,8 +56,8 @@ test.describe('Papers Browser', () => {
test('shows paper table', async ({ page }) => {
await page.goto('/#/papers');
await expect(page.locator('table tbody tr').first()).toBeVisible({ timeout: 10000 });
- expect(await page.locator('table tbody tr').count()).toBe(467);
- await expect(page.locator('#f-count')).toHaveText('467 / 467');
+ expect(await page.locator('table tbody tr').count()).toBe(2687);
+ await expect(page.locator('#f-count')).toHaveText('2687 / 2687');
});
test('text search filters papers', async ({ page }) => {
@@ -59,7 +66,7 @@ test.describe('Papers Browser', () => {
await page.fill('#f-search', 'metr');
const count = await page.locator('table tbody tr').count();
expect(count).toBeGreaterThan(0);
- expect(count).toBeLessThan(467);
+ expect(count).toBeLessThan(2687);
});
test('archetype filter works', async ({ page }) => {
@@ -68,7 +75,7 @@ test.describe('Papers Browser', () => {
await page.selectOption('#f-archetype', 'Complete');
const count = await page.locator('table tbody tr').count();
expect(count).toBeGreaterThan(0);
- expect(count).toBeLessThan(467);
+ expect(count).toBeLessThan(2687);
});
test('clicking row navigates to paper detail', async ({ page }) => {
@@ -83,7 +90,8 @@ test.describe('Papers Browser', () => {
await expect(page.locator('table tbody tr').first()).toBeVisible({ timeout: 10000 });
await page.locator('thead th', { hasText: 'Score' }).click();
const firstScore = await page.locator('table tbody tr').first().locator('td.score').textContent();
- expect(parseFloat(firstScore!)).toBeLessThan(30);
+ // First entry is "--" (unscanned) or a low score
+ expect(firstScore!.trim()).toMatch(/^--|^\d/);
});
});
diff --git a/scripts/build-explorer-data.py b/scripts/build-explorer-data.py
@@ -331,6 +331,29 @@ def build():
game_pcts = {g: round(c / total_papers * 100, 1) for g, c in game_counts.items()}
repro_count = sum(1 for p in papers_full if p["category_scores"].get("artifacts", 0) == 100)
+ # --- Registry pipeline stats ---
+ reg_total = len(registry)
+ reg_status = Counter(e.get("status", "unknown") for e in registry.values())
+ has_text = sum(1 for e in registry.values()
+ if (PAPERS_DIR / e["id"] / "paper.txt").exists())
+ v1_count = 0
+ for scan_path in PAPERS_DIR.glob("*/scan.json"):
+ pid = scan_path.parent.name
+ if pid not in {p["id"] for p in papers_full}:
+ with open(scan_path) as f:
+ s = json.load(f)
+ if s.get("scan_version", 1) < 2:
+ v1_count += 1
+
+ pipeline = {
+ "registry_total": reg_total,
+ "v2_scanned": total_papers,
+ "v1_needs_rescan": v1_count,
+ "has_text_no_scan": has_text - total_papers - v1_count,
+ "no_text": reg_total - has_text,
+ "excluded": reg_status.get("excluded", 0),
+ }
+
dashboard = {
"n": total_papers,
"median": round(median, 1),
@@ -342,6 +365,7 @@ def build():
"game_pcts": game_pcts,
"archetype_counts": dict(archetype_counts),
"tag_counts": dict(tag_counts),
+ "pipeline": pipeline,
}
# --- Citation network ---
@@ -377,6 +401,26 @@ def build():
papers_detail_dir = OUTPUT_DIR / "papers"
papers_detail_dir.mkdir(parents=True, exist_ok=True)
+ # Add unscanned registry entries to papers-index (score=null, no detail page)
+ scanned_ids = {p["id"] for p in papers_index}
+ for entry in registry.values():
+ if entry["id"] in scanned_ids:
+ continue
+ if entry.get("status") == "excluded":
+ continue
+ papers_index.append({
+ "id": entry["id"],
+ "title": entry.get("title", entry["id"]),
+ "year": entry.get("year"),
+ "venue": entry.get("venue", ""),
+ "tags": entry.get("tags", []),
+ "score": None,
+ "archetype": None,
+ "games": [],
+ "arxiv_id": entry.get("arxiv_id", ""),
+ "doi": entry.get("doi", ""),
+ })
+
write_json(OUTPUT_DIR / "dashboard.json", dashboard)
write_json(OUTPUT_DIR / "papers-index.json", papers_index)
write_json(OUTPUT_DIR / "network.json", network)
