/** * Fester — dedicated DAG build orchestration service. * * Port of the standalone fester repo (backend/) to a bun mini-service: * - PipelineEngine → engine.ts (DAG execution, pause/step, retries, cache, BTC stamps) * - scheduler/ → scheduler.ts (weighted node scoring: cpu/temp/policy/instability) * - nodes/ → nodes.ts (NodeStateRegistry + synthetic drift + real local node) * - events/ → events.ts (EventBus + FesterEvent schema) * - storage/ → store.ts (SQLite: builds / sessions / events / node_states) * - analysis/ → autopsy.ts, cause.ts (failure autopsy + causal graph) * - api/ → api.ts (REST mirroring the repo's route surface) * - seed → seed.ts (first-boot synthetic history for replay views) * * Transport: Bun.serve with native WebSocket at path '/' — the same * transport the original fester ui/app.js uses (raw WebSocket with * auto-reconnect). The gateway forwards /?XTransformPort=3010 upgrades * here; REST is served on the same port for the SysDeck server-side proxy. * * v0.3.1 session gate: every request (REST and WS upgrade) must carry a * valid SysDeck web session cookie ('sd_session'), verified against the * SAME HMAC secret the Next.js app uses — read straight out of its * SQLite SdKv table (web.session.secret). Until the web console has * booted once (no secret in the DB yet), the service stays in its * documented standalone mode: unauthenticated, loopback-only. * * WebSocket client protocol (all optional, default = full broadcast): * → {type:'debugger', buildId, command:'pause'|'resume'|'step'} * → {type:'subscribe', buildId} — only receive that build's events * (+ global node_update / hello / heartbeat) * → {type:'unsubscribe'} — back to full broadcast * ← {type:'hello'|'debugger-ack'|'subscribed'|'unsubscribed'|} * A {type:'node_update', heartbeat:true, nodes:N} frame is broadcast every * 10s so clients can detect a stale connection. */ import { FesterEngine } from './engine' import { EventBus, type FesterEvent } from './events' import { Store } from './store' import { NodeRegistry } from './nodes' import { seedHistory } from './seed' import { bindLoop, ensureLoop } from './clock' import { handleApi } from './api' import { createHmac, timingSafeEqual } from 'node:crypto' import { fileURLToPath } from 'node:url' import { Database } from 'bun:sqlite' import type { ServerWebSocket } from 'bun' const PORT = 3010 const JOURNAL_CAP = 5000 // max journaled events per build (oldest dropped) // ── session gate (shared secret with the web console) ──────────────── // The token format mirrors src/lib/sysdeck/session.ts exactly: // v0.4.0: 'v2...' — // user-bound, minted after a unix-account (PAM) login // 0.3.1 : 'v1..")>' — still // accepted so upgrades don't drop live streams // The secret is the web app's SdKv row 'web.session.secret' — opened // read-only, cached 60s so the prisma writer never sees lock churn. const SYSDECK_DB = fileURLToPath(new URL('../../db/custom.db', import.meta.url)) let cachedSecret: string | null = null let secretReadAt = 0 function readSharedSecret(): string | null { const now = Date.now() if (cachedSecret && now - secretReadAt < 60_000) return cachedSecret try { const sqlite = new Database(SYSDECK_DB, { readonly: true }) try { const row = sqlite.query("SELECT value FROM SdKv WHERE key = 'web.session.secret'").get() as | { value: string } | null if (row?.value) { cachedSecret = row.value secretReadAt = now return row.value } } finally { sqlite.close() } } catch { // db absent (web console never booted) or transiently busy — a // stale cached secret still enforces; no secret at all = standalone } return cachedSecret } /** Verify the sd_session cookie against the shared secret. * v0.4.0: the web console's tokens are user-bound — 'v2...' * (HMAC over the whole body). v1 (0.3.1 shared-password sessions) still * verifies so an upgrade across 0.3.1 → 0.4.0 does not drop live streams. */ function sessionOk(req: Request): boolean { const secret = readSharedSecret() if (!secret) return true // web console never booted → standalone mode const cookie = req.headers.get('cookie') ?? '' const m = /(?:^|;\s*)sd_session=([^;]+)/.exec(cookie) const token = m?.[1] if (!token) return false const parts = token.split('.') // v2 — 'v2...' if (parts.length === 4 && parts[0] === 'v2') { const expiresAt = Number(parts[1]) if (!Number.isFinite(expiresAt) || String(expiresAt) !== parts[1]) return false if (expiresAt < Date.now()) return false const user = Buffer.from(parts[2], 'base64url').toString('utf8') if (!user || user.length > 64) return false const expected = createHmac('sha256', secret).update(`v2.${parts[1]}.${parts[2]}`).digest('hex') if (parts[3].length !== expected.length || !/^[0-9a-f]+$/.test(parts[3])) return false return timingSafeEqual(Buffer.from(expected), Buffer.from(parts[3])) } // v1 — 'v1..' (legacy 0.3.1 shared session) if (parts.length === 3 && parts[0] === 'v1') { const expiresAt = Number(parts[1]) if (!Number.isFinite(expiresAt) || String(expiresAt) !== parts[1]) return false if (expiresAt < Date.now()) return false const expected = createHmac('sha256', secret).update(`v1.${parts[1]}`).digest('hex') if (parts[2].length !== expected.length || !/^[0-9a-f]+$/.test(parts[2])) return false return timingSafeEqual(Buffer.from(expected), Buffer.from(parts[2])) } return false } const bus = new EventBus() const store = new Store() const nodes = new NodeRegistry() const engine = new FesterEngine(bus, store, nodes) // ── boot: recover state from a previous process ────────────────────── // 1. Any build left 'running' in the store by a previous process is marked // failed with a "service restarted" note (never show phantom builds). // 2. A fresh store is seeded with a realistic history (see seed.ts). const staleBuilds = store.markStaleRunningFailed() if (staleBuilds.length > 0) { console.log(`[fester] marked ${staleBuilds.length} stale running build(s) failed: ${staleBuilds.map((b) => b.build_id).join(', ')}`) for (const b of staleBuilds) { // journal the reason so the timeline explains the state transition void store .appendEvent({ type: 'debug', timestamp: Date.now() / 1000, build_id: b.build_id, state: 'failed', reason: 'service restarted', meta: { note: 'service restarted while this build was running' }, }) .catch(() => undefined) } } const seededCount = seedHistory(store) if (seededCount > 0) console.log(`[fester] seeded ${seededCount} synthetic historical builds + replay sessions`) // ── WebSocket hub ──────────────────────────────────────────────────── // Every fester event is broadcast to all sockets as JSON — except sockets // that subscribed to a single build: those only receive that build's // events plus global node_update frames. Clients send // {type:'hello'} | {type:'debugger',...} | {type:'subscribe'|'unsubscribe'}. const clients = new Set>() const subscriptions = new Map, string>() function broadcast(event: FesterEvent | { type: string; [k: string]: unknown }): void { const payload = JSON.stringify(event) for (const ws of clients) { if (ws.readyState !== WebSocket.OPEN) continue const sub = subscriptions.get(ws) if (!sub || event.type === 'node_update' || event.build_id === sub) ws.send(payload) } } const server = Bun.serve({ // v0.3.0 security (audit follow-up): bind loopback only. This service // is unauthenticated by design (a local build orchestrator); the web // console reaches it server-side via 127.0.0.1 and the browser reaches // the live event stream through the local port gateway — never // directly. Bun's default (no hostname) is 0.0.0.0, which put every // REST mutation + the WS event stream on the LAN. hostname: '127.0.0.1', port: PORT, fetch(req, srv) { // v0.3.1 session gate: REST and WS upgrades alike require the web // console's session cookie (standalone mode until a secret exists). if (!sessionOk(req)) { return Response.json( { ok: false, error: 'authentication required (sign in to the SysDeck web console first)' }, { status: 401 }, ) } const url = new URL(req.url) // live request context: (re)start the periodic loops if needed (see clock.ts) ensureLoop('fester_ambient') ensureLoop('fester_heartbeat') // WebSocket endpoint: the gateway sends upgrades at path '/' with the // XTransformPort query (io-style contract: path always '/'). if (url.pathname === '/') { if (req.headers.get('upgrade') === 'websocket' || url.searchParams.get('EIO')) { if (srv.upgrade(req)) return undefined } // plain GET / → service banner (also used as a WS health probe) return Response.json({ ok: true, service: 'fester', version: '0.2.1', ws: '/', nodes: nodes.names() }) } if (url.pathname.startsWith('/api/')) { return handleApi({ req, url, bus, store, nodes, engine }) } return Response.json({ ok: false, error: 'not found' }, { status: 404 }) }, websocket: { open(ws) { clients.add(ws) ensureLoop('fester_ambient') ensureLoop('fester_heartbeat') ws.send(JSON.stringify({ type: 'hello', service: 'fester', version: '0.2.1', nodes: nodes.names(), clients: clients.size })) }, message(ws, message) { try { const msg = JSON.parse(String(message)) as { type?: string; buildId?: string; command?: string } if (msg?.type === 'debugger' && msg.buildId && msg.command) { const out = engine.debuggerCommand(msg.buildId, msg.command) ws.send(JSON.stringify({ type: 'debugger-ack', ...out })) broadcast({ type: 'debug', timestamp: Date.now() / 1000, build_id: msg.buildId, state: out.state ?? out.command, meta: { command: msg.command } }) } else if (msg?.type === 'subscribe' && msg.buildId) { subscriptions.set(ws, msg.buildId) ws.send(JSON.stringify({ type: 'subscribed', buildId: msg.buildId, clients: clients.size })) } else if (msg?.type === 'unsubscribe') { const had = subscriptions.delete(ws) ws.send(JSON.stringify({ type: 'unsubscribed', buildId: msg.buildId ?? null, wasSubscribed: had })) } } catch { /* ignore malformed client messages */ } }, close(ws) { clients.delete(ws) subscriptions.delete(ws) }, }, }) // ── event fan-out: bus → WS broadcast + capped timeline journal ────── const journalCounts = new Map() const capNoted = new Set() bus.subscribe((event) => { broadcast(event) void journal(event) }) async function journal(event: FesterEvent): Promise { try { const bid = event.build_id if (!bid) { await store.appendEvent(event) return } let n = journalCounts.get(bid) if (n === undefined) { n = store.countEvents(bid) journalCounts.set(bid, n) } await store.appendEvent(event) n += 1 if (n > JOURNAL_CAP) { // drop the oldest overflow, note the cap once per build await store.trimEvents(bid, n - JOURNAL_CAP) journalCounts.set(bid, JOURNAL_CAP) if (!capNoted.has(bid)) { capNoted.add(bid) const note: FesterEvent = { type: 'debug', timestamp: Date.now() / 1000, build_id: bid, state: 'journal-cap', reason: `journal capped at ${JOURNAL_CAP} events per build — oldest entries dropped`, meta: { cap: JOURNAL_CAP }, } await store.appendEvent(note) broadcast(note) } } else { journalCounts.set(bid, n) } } catch (err) { console.error('[fester:journal] failed to persist event', err) } } // ── heartbeat: a 10s liveness frame so clients can detect stale links ─ // Loop is created/supervised from live request contexts (see clock.ts). bindLoop('fester_heartbeat', 10_000, () => { broadcast({ type: 'node_update', timestamp: Date.now() / 1000, heartbeat: true, nodes: nodes.names().length }) }) // Ambient node drift loop — synthetic probe results for cluster nodes, // real /proc metrics for the local node. nodes.startAmbient(2000, (snapshot) => { broadcast({ type: 'node_update', timestamp: Date.now() / 1000, ...snapshot }) store.upsertNodeState(snapshot) }) // start both periodic loops immediately at boot (see clock.ts: loops are // re-created/supervised from live request contexts after hot reloads, but on // a fresh boot starting them here avoids depending on a first request). ensureLoop('fester_ambient') ensureLoop('fester_heartbeat') console.log(`[fester] DAG orchestration service on :${PORT}`) console.log(`[fester] nodes: ${nodes.names().join(', ')}`) export { server }