dcosnet
/
fester
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
fester/backend/main.py

946 lines
33 KiB
Python
Raw Blame History

"""
backend/main.py — the real Fester FastAPI application.
Wires all routers, the WebSocket stream, and the singleton bus + registry.
Run with:
python -m backend.main
or:
uvicorn backend.main:app --host 0.0.0.0 --port 8080
"""
from __future__ import annotations
import asyncio
import json
import shutil
import time
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Body
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
# Singletons (must be created BEFORE routers that use them)
from backend.api.api import bus, registry, observability, cause_graph, attach_ws_stream
from backend.api.ws import WebSocketStream
# Routers
from backend.api.replay import router as replay_router
from backend.api.autopsy import router as autopsy_router
from backend.api.timeline import router as timeline_router
from backend.api.debugger import router as debugger_router, set_engine as dbg_set_engine
from backend.api.pipeline_control import router as pipeline_router, set_engine as pc_set_engine
from backend.api.nodes import router as nodes_router
from backend.api.metrics import router as metrics_router
from backend.api.cause import router as cause_router
# Other deps
from backend.events.schema import EventType
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
from backend.analysis.failure_propagation import FailurePropagator
from backend.config import CONFIG
# ----------------------------------------------------------------
# App
# ----------------------------------------------------------------
ROOT = Path(__file__).resolve().parent.parent # /home/z/my-project/fester
app = FastAPI(
title="Fester",
description="Distributed build + scheduler + observability system",
version="0.2.0",
)
# Attach WS stream to bus
ws_stream = WebSocketStream()
attach_ws_stream(ws_stream)
# Wire a timeline store subscriber so every bus event lands in the journal
def _timeline_subscriber(event: Dict[str, Any]):
TIMELINE_STORE.append(event)
bus.subscribe(_timeline_subscriber)
# Wire a failure propagator so we can compute blast radius later
_propagator = FailurePropagator()
bus.subscribe(_propagator.ingest_event)
# Wire metrics counters so /metrics reflects real activity
from backend.api.metrics import record_pipeline_action, record_cache_hit
def _metrics_subscriber(event: Dict[str, Any]):
"""Increment counters for task_update + cache_update events."""
etype = event.get("type")
if etype == "task_update":
state = event.get("state")
if state:
record_pipeline_action(state)
elif etype == "cache_update":
# Cache hit: try meta.target first, then top-level target, then "unknown"
target = event.get("target") or (event.get("meta", {}) or {}).get("target") or "unknown"
if event.get("state") == "hit":
record_cache_hit(target)
bus.subscribe(_metrics_subscriber)
# ----------------------------------------------------------------
# Build history (in-memory; replace with DB later)
# ----------------------------------------------------------------
# Build history lives in build_runner (defined below).
# ----------------------------------------------------------------
# Static UI (mount LAST so it doesn't shadow API routes)
# ----------------------------------------------------------------
@app.get("/")
@app.get("/index.html")
async def index():
return FileResponse(ROOT / "index.html")
@app.get("/style.css")
async def style_css():
return FileResponse(ROOT / "style.css", media_type="text/css")
@app.get("/ui/app.js")
async def app_js():
return FileResponse(ROOT / "ui" / "app.js", media_type="application/javascript")
@app.get("/ui/{name}.html")
async def ui_page(name: str):
p = ROOT / "ui" / f"{name}.html"
if not p.exists() or not p.is_file():
raise HTTPException(404)
return FileResponse(p, media_type="text/html")
# Mount cockpit folder (for the cockpit module)
app.mount("/cockpit", StaticFiles(directory=str(ROOT / "cockpit")), name="cockpit")
# ----------------------------------------------------------------
# Include API routers
# ----------------------------------------------------------------
app.include_router(replay_router)
app.include_router(autopsy_router)
app.include_router(timeline_router)
app.include_router(debugger_router)
app.include_router(pipeline_router)
app.include_router(nodes_router)
app.include_router(metrics_router)
app.include_router(cause_router)
# ----------------------------------------------------------------
# Build runner — real engine + synthetic fallback
# ----------------------------------------------------------------
from backend.pipeline.runner import BuildRunner
# Nodes for the runner come from CONFIG; the registry is the singleton.
_runner_nodes = [{"name": n["name"], "host": n.get("host"), "max_jobs": n.get("max_jobs", 8),
"state": "online"} for n in CONFIG.get("nodes", [])]
build_runner = BuildRunner(bus, registry, _runner_nodes)
# ----------------------------------------------------------------
# Top-level endpoints that don't fit a router
# ----------------------------------------------------------------
class BuildBody(BaseModel):
cmd: str = "make -j$(nproc)"
dir: str = "/home/user/linux"
project: str | None = None
arch: str = "x86_64"
target: str = "linux-gnu"
toolchain: str = "gcc"
@app.post("/api/build")
async def api_build(body: BuildBody):
"""Kick off a build. Returns a build_id immediately; events stream via /ws."""
build_id = await build_runner.start(body)
return {"status": "started", "build_id": build_id}
@app.get("/api/builds")
async def api_builds():
"""Build history."""
return {"builds": build_runner.list_builds()}
@app.get("/api/builds/{build_id}")
async def api_build_detail(build_id: str):
state = build_runner.state(build_id)
if state:
return state
raise HTTPException(404, "build not found")
@app.post("/api/builds/{build_id}/cancel")
async def api_build_cancel(build_id: str):
if build_runner.cancel(build_id):
return {"status": "cancelled", "build_id": build_id}
raise HTTPException(404, "build not found or not running")
@app.get("/api/sessions")
async def api_sessions():
"""List replay sessions known to the server (from SessionDB)."""
from backend.api.replay import SESSION_DB
return {"sessions": SESSION_DB.list_sessions()}
# ----------------------------------------------------------------
# Targets API — backed by backend/targets/catalog.py + ARCHES/RUNTIMES
# ----------------------------------------------------------------
from backend.targets.catalog import TARGET_SYSTEMS, ARCHES, RUNTIMES
def _build_targets_catalog():
"""Combine catalog target systems + arches into a unified toggle list."""
out = []
# Per-arch cross-compile targets (always present)
arch_defaults = {
"x86_64": True, # most common, default-on
"aarch64": True,
"riscv64": False,
"arm64": True,
}
for arch in ARCHES:
out.append({
"name": f"{arch}-linux-gnu",
"enabled": arch_defaults.get(arch, False),
"arch": arch,
"type": "cross-compile",
"source": "arch",
})
# Per-target-system entries (Gentoo, Buildroot, OpenWrt, etc.)
for tname, tinfo in TARGET_SYSTEMS.items():
out.append({
"name": tname,
"enabled": False, # opt-in
"type": tinfo.get("type", "unknown"),
"toolchain": tinfo.get("toolchain"),
"supports_cross": tinfo.get("supports_cross", False),
"source": "catalog",
})
return out
TARGETS_CATALOG = _build_targets_catalog()
@app.get("/api/targets")
async def api_targets():
"""List all known build targets (arches + catalog systems)."""
return TARGETS_CATALOG
@app.get("/api/targets/arches")
async def api_targets_arches():
"""List supported architectures."""
return {"arches": ARCHES}
@app.get("/api/targets/runtimes")
async def api_targets_runtimes():
"""List supported execution runtimes."""
return {"runtimes": RUNTIMES}
@app.post("/api/targets/{name}")
async def api_toggle_target(name: str, body: Dict[str, Any] = Body(default={})):
for t in TARGETS_CATALOG:
if t["name"] == name:
t["enabled"] = bool(body.get("enabled", False))
# Broadcast to ws-targets subscribers
await _broadcast_targets({
"type": "target-update",
"data": {"name": name, "enabled": t["enabled"]},
})
return {"status": "ok", "name": name, "enabled": t["enabled"]}
raise HTTPException(404, "target not found")
# ----------------------------------------------------------------
# Release API
# ----------------------------------------------------------------
class ReleaseBody(BaseModel):
repo: str = "https://forgejo.local/project.git"
project: str | None = None
target: str = "linux-gnu"
@app.post("/api/release")
async def api_release(body: ReleaseBody):
"""Kick off a release build. Mirrors /api/build but tags the build_id
with a 'release-' prefix and uses a longer release-shaped pipeline."""
release_id = str(uuid.uuid4())[:8]
build_id = f"release-{release_id}"
# Construct a release-shaped BuildBody and dispatch through BuildRunner
release_body = BuildBody(
cmd=f"make release REPO={body.repo}",
dir="/home/user/release",
project=body.project or body.repo,
arch="x86_64",
target=body.target,
toolchain="gcc",
)
actual_id = await build_runner.start(release_body)
return {
"status": "started",
"repo": body.repo,
"release_id": release_id,
"build_id": actual_id,
"pipeline": ["fetch_source", "configure", "build_kernel", "build_modules",
"package", "release"],
}
# ----------------------------------------------------------------
# Policy API — real implementation backed by an in-memory policy store
# ----------------------------------------------------------------
from backend.policy.engine import PolicyEngine
from backend.api.nodes import PolicyBody as NodePolicyBody # reuse for top-level policy too
# Singleton policy engine (no DB learning for now — just key/value overrides)
POLICY_ENGINE = PolicyEngine(db=None)
_POLICY_STORE: Dict[str, Any] = {} # simple key→value overrides
# Pydantic model for /api/policy/set
class PolicyBody(BaseModel):
key: str
value: Any
@app.post("/api/policy/set")
async def api_policy_set(body: PolicyBody):
"""Set a policy key=value. Currently understood keys:
- scheduler.mode (unified|weighted-thermal-aware|cache-first|target-isolated|experimental-intelligence)
- scheduler.thermal_cap (float, 0..1)
- scheduler.cache_first (bool)
- build.default_arch (string)
Other keys are stored but not yet acted upon.
"""
_POLICY_STORE[body.key] = body.value
bus.emit(EventType.PIPELINE_UPDATE, state="policy",
meta={"key": body.key, "value": body.value, "action": "set"})
return {"status": "ok", "key": body.key, "value": body.value,
"stored_keys": list(_POLICY_STORE.keys())}
@app.post("/api/policy/clear")
async def api_policy_clear(body: Dict[str, Any] = Body(default={})):
key = body.get("key")
if key and key in _POLICY_STORE:
del _POLICY_STORE[key]
bus.emit(EventType.PIPELINE_UPDATE, state="policy",
meta={"key": key, "action": "clear"})
return {"status": "cleared", "key": key, "remaining_keys": list(_POLICY_STORE.keys())}
@app.get("/api/policy")
async def api_policy_list():
"""List all current policy overrides."""
return {"policies": _POLICY_STORE}
# ----------------------------------------------------------------
# Failure propagation (blast radius)
# ----------------------------------------------------------------
@app.get("/api/propagation/{action}")
async def api_propagation(action: str):
"""Compute the downstream blast radius if `action` fails."""
return _propagator.impact_report(action)
# ----------------------------------------------------------------
# Sessions attach/detach (for the CLI / IDE feature)
# ----------------------------------------------------------------
@app.post("/api/session/attach")
async def api_session_attach(body: Dict[str, Any] = Body(default={})):
return {"status": "attached", "action": body.get("action")}
@app.post("/api/session/detach")
async def api_session_detach(body: Dict[str, Any] = Body(default={})):
return {"status": "detached", "action": body.get("action")}
# ----------------------------------------------------------------
# WebSocket endpoints
# ----------------------------------------------------------------
class WSHub:
def __init__(self):
self.clients: List[WebSocket] = []
self.targets_clients: List[WebSocket] = []
self.debugger_clients: List[WebSocket] = []
async def accept(self, ws: WebSocket, kind: str = "main"):
await ws.accept()
if kind == "main":
self.clients.append(ws)
elif kind == "targets":
self.targets_clients.append(ws)
elif kind == "debugger":
self.debugger_clients.append(ws)
hub = WSHub()
# Wire ws_stream (singleton) to broadcast to /ws clients.
# We can't await inside the sync bus.emit, so we schedule sends on the
# running loop. If there's no running loop (called from sync context),
# we silently drop — the WS hub is async-only.
def _ws_broadcast(event: Dict[str, Any]):
payload = json.dumps(event)
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return # no running loop; skip
for c in list(hub.clients):
try:
loop.create_task(c.send_text(payload))
except Exception:
pass
ws_stream.clients = hub.clients # share the list
bus.subscribe(_ws_broadcast)
async def _broadcast_targets(event: Dict[str, Any]):
payload = json.dumps(event)
dead = []
for c in hub.targets_clients:
try:
await c.send_text(payload)
except Exception:
dead.append(c)
for d in dead:
if d in hub.targets_clients:
hub.targets_clients.remove(d)
@app.websocket("/ws")
async def ws_main(ws: WebSocket):
await hub.accept(ws, "main")
try:
await ws.send_text(json.dumps({
"type": "session",
"timestamp": time.time(),
"data": {"action": "subscribe", "id": "live"},
}))
while True:
await ws.receive_text()
except WebSocketDisconnect:
if ws in hub.clients:
hub.clients.remove(ws)
@app.websocket("/ws-targets")
async def ws_targets(ws: WebSocket):
await hub.accept(ws, "targets")
try:
while True:
await ws.receive_text()
except WebSocketDisconnect:
if ws in hub.targets_clients:
hub.targets_clients.remove(ws)
@app.websocket("/ws-debugger")
async def ws_debugger(ws: WebSocket):
"""Real debugger WS — responds to step/pause/resume + streams timeline state."""
await hub.accept(ws, "debugger")
try:
# Send initial state on connect
eng = build_runner.get_engine()
active = eng is not None
await ws.send_text(json.dumps({
"type": "timeline-update",
"data": {
"events": _debugger_timeline_snapshot(),
"current": _debugger_current(),
"engine_attached": active,
},
}))
while True:
msg = await ws.receive_text()
try:
cmd = json.loads(msg)
except Exception:
continue
action = cmd.get("action")
eng = build_runner.get_engine()
if action == "pause":
if eng and hasattr(eng, "pause"):
eng.pause()
from backend.api.debugger import _DEBUG_STATE
_DEBUG_STATE["paused"] = True
response = {"type": "state-change", "data": {"state": "paused"}}
elif action == "resume":
if eng and hasattr(eng, "resume"):
eng.resume()
from backend.api.debugger import _DEBUG_STATE
_DEBUG_STATE["paused"] = False
response = {"type": "state-change", "data": {"state": "running"}}
elif action == "step_forward" or action == "step":
if eng and hasattr(eng, "step"):
eng.step()
from backend.api.debugger import _DEBUG_STATE
_DEBUG_STATE["current_step"] += 1
response = {"type": "state-change",
"data": {"state": "stepped", "step": _DEBUG_STATE["current_step"]}}
elif action == "step_back":
from backend.api.debugger import _DEBUG_STATE
if _DEBUG_STATE["current_step"] > 0:
_DEBUG_STATE["current_step"] -= 1
response = {"type": "state-change",
"data": {"state": "stepped_back", "step": _DEBUG_STATE["current_step"]}}
else:
response = {"type": "error", "data": {"error": f"unknown action: {action}"}}
# Augment with the latest timeline state
response["timeline"] = _debugger_timeline_snapshot()
response["current"] = _debugger_current()
await ws.send_text(json.dumps(response))
except WebSocketDisconnect:
if ws in hub.debugger_clients:
hub.debugger_clients.remove(ws)
def _debugger_timeline_snapshot():
"""Return a list of {name, completed, node, state} for the debugger UI."""
events = TIMELINE_STORE.all()
seen = {}
for e in events:
action = e.get("action")
if not action:
continue
d = e
state = d.get("state")
seen[action] = {
"name": action,
"completed": state == "done",
"state": state,
"node": d.get("node"),
"failed": state == "failed",
}
return list(seen.values())
def _debugger_current():
"""Return the most recent running or scheduled action."""
events = TIMELINE_STORE.all()
for e in reversed(events):
if e.get("state") in ("running", "scheduled"):
return {"action": e.get("action"), "state": e.get("state"),
"node": e.get("node")}
return None
# ----------------------------------------------------------------
# Health
# ----------------------------------------------------------------
@app.get("/api/health")
async def health():
return {
"status": "ok",
"version": "0.2.0",
"bus_subscribers": len(bus.subscribers),
"ws_clients": len(hub.clients),
"builds_known": len(build_runner.list_builds()) if build_runner else 0,
"timeline_events": TIMELINE_STORE.stats()["total_events"],
}
# ----------------------------------------------------------------
# Storage API — btrfs CAS, tmpfs, qcow2 snapshots
# ----------------------------------------------------------------
from backend.storage.router import (
prepare_workspace, maybe_freeze_workspace, storage_status,
update_storage_config,
)
from backend.storage.btrfs_cas import is_btrfs, store_reflink, reflink_snapshot, cas_store, cas_exists
from backend.storage.qcow2_freeze import freeze_to_qcow2, qcow2_info, list_snapshots
@app.get("/api/storage/status")
async def api_storage_status():
"""Return current storage configuration + capability detection."""
return storage_status()
@app.post("/api/storage/config")
async def api_storage_config_update(body: Dict[str, Any] = Body(default={})):
"""Update storage configuration (e.g. {btrfs: true, qcow2_freeze: true})."""
return update_storage_config(body)
@app.post("/api/storage/freeze")
async def api_storage_freeze(body: Dict[str, Any] = Body(default={})):
"""Freeze a workdir into a qcow2 snapshot.
Body: {"workdir": "/path/to/workdir", "project": "name"}"""
workdir = body.get("workdir")
project = body.get("project", "manual")
if not workdir:
raise HTTPException(400, "workdir required")
result = maybe_freeze_workspace(workdir, project)
if result is None:
return {"status": "disabled", "reason": "qcow2_freeze not enabled in config"}
return {"status": "frozen", "path": result, "project": project}
@app.get("/api/storage/snapshots")
async def api_storage_snapshots():
"""List all qcow2 snapshots."""
from backend.storage.config import load_storage_config
cfg = load_storage_config()
return {"snapshots": list_snapshots(cfg.get("qcow2_path", "/var/lib/fester/snapshots"))}
@app.post("/api/storage/reflink")
async def api_storage_reflink(body: Dict[str, Any] = Body(default={})):
"""Reflink-copy a file or directory (CoW snapshot on btrfs).
Body: {"src": "/path/to/src", "dst": "/path/to/dst"}"""
src = body.get("src")
dst = body.get("dst")
if not src or not dst:
raise HTTPException(400, "src and dst required")
if not Path(src).exists():
raise HTTPException(404, f"src not found: {src}")
from pathlib import Path as _P
if _P(src).is_dir():
stats = reflink_snapshot(src, dst)
return {"status": "ok", "mode": "directory", "stats": stats}
else:
reflinked = store_reflink(src, dst)
return {"status": "ok", "mode": "file", "reflinked": reflinked}
@app.post("/api/workspace/prepare")
async def api_workspace_prepare(body: Dict[str, Any] = Body(default={})):
"""Prepare a workspace for a build (tmpfs + optional btrfs CAS).
Body: {"project": "name"}"""
project = body.get("project", "default")
return prepare_workspace(project)
# ----------------------------------------------------------------
# Actions API — tmux session inspection + node shell attach
# ----------------------------------------------------------------
from backend.integrations.tmux import TmuxManager as _TmuxMgr
from backend.integrations.mosh import MoshManager as _MoshMgr
_tmux_mgr = _TmuxMgr()
_mosh_mgr = _MoshMgr()
@app.get("/api/actions/active")
async def api_actions_active():
"""List all active fester-* tmux sessions (running actions)."""
if not _tmux_mgr.available:
return {"available": False, "sessions": [], "reason": "tmux not installed"}
return {
"available": True,
"sessions": _tmux_mgr.list_fester_sessions(),
}
@app.get("/api/actions/{name}/tmux")
async def api_action_tmux_state(name: str):
"""Get state of the tmux session for a specific action."""
if not _tmux_mgr.available:
return {"available": False, "reason": "tmux not installed"}
# Try with and without build_id suffix
candidates = [f"fester-{name}"]
# Also try matching any session that starts with fester-{name}-
sessions = _tmux_mgr.list_fester_sessions()
for s in sessions:
if s["name"].startswith(f"fester-{name}-"):
candidates.append(s["name"])
for cand in candidates:
state = _tmux_mgr.session_state(cand)
if state:
output = _tmux_mgr.capture_output(cand, lines=200) or ""
return {
"available": True,
"name": cand,
"state": state,
"output_tail": output[-2000:] if output else None,
}
return {"available": True, "exists": False, "name": name}
@app.get("/api/actions/{name}/tmux/output")
async def api_action_tmux_output(name: str, lines: int = 1000):
"""Capture the last N lines of tmux pane output for an action."""
if not _tmux_mgr.available:
return {"available": False, "reason": "tmux not installed"}
# Find the session
candidates = [f"fester-{name}"]
sessions = _tmux_mgr.list_fester_sessions()
for s in sessions:
if s["name"].startswith(f"fester-{name}-"):
candidates.append(s["name"])
for cand in candidates:
if _tmux_mgr.session_exists(cand):
output = _tmux_mgr.capture_output(cand, lines=lines) or ""
return {
"available": True,
"name": cand,
"lines": len(output.split("\n")) if output else 0,
"output": output,
}
return {"available": True, "exists": False, "name": name}
@app.post("/api/actions/{name}/tmux/kill")
async def api_action_tmux_kill(name: str):
"""Kill the tmux session for an action."""
if not _tmux_mgr.available:
return {"available": False, "reason": "tmux not installed"}
candidates = [f"fester-{name}"]
sessions = _tmux_mgr.list_fester_sessions()
for s in sessions:
if s["name"].startswith(f"fester-{name}-"):
candidates.append(s["name"])
killed = []
for cand in candidates:
if _tmux_mgr.kill_session(cand):
killed.append(cand)
return {"killed": killed, "count": len(killed)}
# ----------------------------------------------------------------
# Node shell — mosh / ssh command builder
# ----------------------------------------------------------------
class ShellBody(BaseModel):
user: Optional[str] = None
port: Optional[int] = None
ssh_key: Optional[str] = None
method: str = "mosh" # mosh | ssh
@app.get("/api/nodes/{name}/shell")
async def api_node_shell(name: str):
"""Return the command string to attach a shell to a node (mosh or ssh).
Doesn't actually exec it — the caller's terminal does."""
# Find the node
cfg = None
for n in CONFIG.get("nodes", []):
if n["name"] == name:
cfg = n
break
if not cfg:
raise HTTPException(404, f"node {name} not configured")
host = cfg.get("host")
if not host:
raise HTTPException(400, f"node {name} has no host")
return {
"node": name,
"host": host,
"mosh_available": _mosh_mgr.available,
"ssh_available": shutil.which("ssh") is not None,
"commands": {
"mosh": f"mosh {host}",
"ssh": f"ssh {host}",
},
}
@app.post("/api/nodes/{name}/shell")
async def api_node_shell_custom(name: str, body: ShellBody):
"""Build a custom shell command for a node with user/port/ssh-key options."""
cfg = None
for n in CONFIG.get("nodes", []):
if n["name"] == name:
cfg = n
break
if not cfg:
raise HTTPException(404, f"node {name} not configured")
host = cfg.get("host")
if not host:
raise HTTPException(400, f"node {name} has no host")
target = f"{body.user}@{host}" if body.user else host
cmd = _mosh_mgr.build_command(target, port=body.port, ssh_key=body.ssh_key,
method=body.method)
return {
"node": name,
"host": host,
"method": body.method,
"command": cmd,
"copy_to_clipboard": cmd,
}
# ----------------------------------------------------------------
# Ambient loop — probe real node agents, fall back to drift if unreachable.
# Set FESTER_NO_DRIFT=1 to disable synthetic drift entirely (production).
# ----------------------------------------------------------------
async def ambient_loop():
"""Probe each configured node's agent every 4s. If the agent responds,
update the registry with real metrics. If not, optionally drift the
last-known values so the UI doesn't go stale."""
import os
import random
from backend.nodes.probe import probe_async, normalize
from backend.storage.sqlite_db import STORAGE
NO_DRIFT = os.environ.get("FESTER_NO_DRIFT") == "1"
await asyncio.sleep(2)
while True:
for n in CONFIG.get("nodes", []):
name = n["name"]
host = n.get("host")
# Try real probe first
raw = await probe_async(host) if host else None
if raw is not None:
# Real agent responded — use its data
state = normalize(raw, name)
registry.update(
name,
cpu_load=state["cpu_load"],
memory_load=state["memory_load"],
temp=state["temp"],
instability=state["instability"],
active_jobs=state["active_jobs"],
last_seen=state["last_seen"],
)
cur = registry.get(name)
# Persist node state to SQLite
if STORAGE:
try:
STORAGE.save_node_state(name, {
"state": state.get("state", "online"),
"cpu_load": cur.cpu_load,
"memory_load": cur.memory_load,
"temp": cur.temp,
"instability": cur.instability,
"active_jobs": cur.active_jobs,
"max_jobs": state.get("max_jobs", n.get("max_jobs", 8)),
"last_seen": cur.last_seen,
"labels": state.get("labels", {}),
})
except Exception:
pass
# Emit with state from probe (may be "online" or "degraded")
bus.emit(
EventType.NODE_UPDATE,
node=name,
state=state.get("state", "online"),
meta={
"heat": cur.temp,
"jobs": cur.active_jobs,
"cpu_load": cur.cpu_load,
"instability": cur.instability,
"source": "agent",
},
)
elif not NO_DRIFT:
# Agent unreachable — drift the last-known values
existing = registry.get(name)
if existing is None:
# First-seen: initialize with plausible values
registry.update(
name,
cpu_load=random.uniform(20, 70),
memory_load=random.uniform(30, 60),
temp=random.uniform(40, 70),
instability=random.uniform(0, 0.15),
active_jobs=random.randint(0, max(1, n.get("max_jobs", 8) // 4)),
last_seen=time.time(),
)
else:
registry.update(
name,
cpu_load=max(0, min(100, existing.cpu_load + random.uniform(-5, 5))),
memory_load=max(0, min(100, existing.memory_load + random.uniform(-2, 2))),
temp=max(20, min(95, existing.temp + random.uniform(-3, 3))),
instability=max(0, min(1, existing.instability + random.uniform(-0.05, 0.05))),
active_jobs=max(0, min(n.get("max_jobs", 8), existing.active_jobs + random.randint(-1, 1))),
last_seen=time.time(),
)
cur = registry.get(name)
# Persist drifted state too
if STORAGE:
try:
STORAGE.save_node_state(name, {
"state": "online",
"cpu_load": cur.cpu_load,
"memory_load": cur.memory_load,
"temp": cur.temp,
"instability": cur.instability,
"active_jobs": cur.active_jobs,
"max_jobs": n.get("max_jobs", 8),
"last_seen": cur.last_seen,
"labels": {},
})
except Exception:
pass
bus.emit(
EventType.NODE_UPDATE,
node=name,
state="online",
meta={
"heat": cur.temp,
"jobs": cur.active_jobs,
"cpu_load": cur.cpu_load,
"instability": cur.instability,
"source": "drift",
},
)
# If NO_DRIFT and no agent, we simply skip — node stays offline
await asyncio.sleep(4)
@app.on_event("startup")
async def on_startup():
asyncio.create_task(ambient_loop())
# Optional: kick off a build every 60s for demo purposes.
# Disabled by default — set FESTER_AUTOBUILD=1 to enable.
import os
if os.environ.get("FESTER_AUTOBUILD") == "1":
async def looper():
await asyncio.sleep(15) # let things warm up first
while True:
await build_runner.start(
BuildBody(cmd="make -j$(nproc)", dir="/home/user/linux"),
)
await asyncio.sleep(60)
asyncio.create_task(looper())
# ----------------------------------------------------------------
# CLI entrypoint
# ----------------------------------------------------------------
def main():
import uvicorn
uvicorn.run("backend.main:app", host="0.0.0.0", port=8080, reload=False)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink