Fester: a distributed build orchestrator with thermal-aware scheduling, live DAG, event replay, and failure analysis
This commit is contained in:
commit
0a9cb8fd97
|
|
@ -0,0 +1,64 @@
|
||||||
|
# .github/funding.yml
|
||||||
|
# ============================================================
|
||||||
|
# GitHub Sponsors "Sponsor this project" button configuration.
|
||||||
|
# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/displaying-a-sponsor-button-in-your-repository
|
||||||
|
#
|
||||||
|
# Each key below corresponds to a sponsored-developer account or platform
|
||||||
|
# that will appear as a "Sponsor" button on the repo's main page.
|
||||||
|
#
|
||||||
|
# Replace the placeholder values with your actual usernames/addresses.
|
||||||
|
# Comment out or delete any platforms you don't want to use.
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
# --- GitHub Sponsors ---
|
||||||
|
# Your GitHub Sponsors username (the part after github.com/sponsors/)
|
||||||
|
github: REPLACE_WITH_YOUR_GITHUB_USERNAME
|
||||||
|
|
||||||
|
# --- Patreon ---
|
||||||
|
# Your Patreon username (the part after patreon.com/)
|
||||||
|
patreon: REPLACE_WITH_YOUR_PATREON_USERNAME
|
||||||
|
|
||||||
|
# --- Open Collective ---
|
||||||
|
# Your Open Collective slug (the part after opencollective.com/)
|
||||||
|
open_collective: REPLACE_WITH_YOUR_OPENCOLLECTIVE_SLUG
|
||||||
|
|
||||||
|
# --- Ko-fi ---
|
||||||
|
# Your Ko-fi username (the part after ko-fi.com/)
|
||||||
|
ko_fi: REPLACE_WITH_YOUR_KOFI_USERNAME
|
||||||
|
|
||||||
|
# --- Tidelift ---
|
||||||
|
# Your Tidelift package name (e.g. "npm/foo" or "pypi/bar")
|
||||||
|
tidelift: REPLACE_WITH_YOUR_TIDELIFT_PACKAGE
|
||||||
|
|
||||||
|
# --- CommunityBridge ---
|
||||||
|
# Your CommunityBridge project name
|
||||||
|
community_bridge: REPLACE_WITH_YOUR_COMMUNITYBRIDGE_PROJECT
|
||||||
|
|
||||||
|
# --- Liberapay ---
|
||||||
|
# Your Liberapay username (the part after liberapay.com/)
|
||||||
|
liberapay: REPLACE_WITH_YOUR_LIBERAPAY_USERNAME
|
||||||
|
|
||||||
|
# --- IssueHunt ---
|
||||||
|
# Your IssueHunt username
|
||||||
|
issuehunt: REPLACE_WITH_YOUR_ISSUEHUNT_USERNAME
|
||||||
|
|
||||||
|
# --- LFX Crowdfunding ---
|
||||||
|
# Your LFX project name
|
||||||
|
lfx_crowdfunding: REPLACE_WITH_YOUR_LFX_PROJECT
|
||||||
|
|
||||||
|
# --- Polar ---
|
||||||
|
# Your Polar repository (e.g. "owner/repo")
|
||||||
|
polar: REPLACE_WITH_YOUR_GITHUB_ORG/fester
|
||||||
|
|
||||||
|
# --- Custom platforms / direct links ---
|
||||||
|
# A list of up to 4 custom {label, url} entries that appear as buttons.
|
||||||
|
# Use these for crypto donation links, your own donate page, etc.
|
||||||
|
custom:
|
||||||
|
- label: "Donate"
|
||||||
|
url: "https://git.dcos.net/dcosnet/fester/src/branch/main/donate.md"
|
||||||
|
# - label: "Bitcoin"
|
||||||
|
# url: "https://your-blockchain-explorer/address/YOUR_BTC_ADDRESS"
|
||||||
|
# - label: "Monero"
|
||||||
|
# url: "https://your-xmr-explorer/address/YOUR_XMR_ADDRESS"
|
||||||
|
# - label: "PayPal"
|
||||||
|
# url: "https://www.paypal.com/paypalme/YOUR_PAYPAL_ME"
|
||||||
|
|
@ -0,0 +1,139 @@
|
||||||
|
# ============================================================
|
||||||
|
# Fester .gitignore
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
# ---- Python ----
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
*.so
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# ---- Virtual environments ----
|
||||||
|
.venv/
|
||||||
|
venv/
|
||||||
|
env/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
.python-version
|
||||||
|
|
||||||
|
# ---- Testing / coverage ----
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
.mypy_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
|
# ---- Fester runtime data ----
|
||||||
|
*.db
|
||||||
|
*.db-journal
|
||||||
|
*.db-wal
|
||||||
|
*.db-shm
|
||||||
|
*.sqlite
|
||||||
|
*.sqlite-journal
|
||||||
|
*.sqlite-wal
|
||||||
|
*.sqlite-shm
|
||||||
|
|
||||||
|
# Fester storage dirs (local dev)
|
||||||
|
.fester/
|
||||||
|
fester-data/
|
||||||
|
/var/lib/fester/
|
||||||
|
snapshots/
|
||||||
|
cas/
|
||||||
|
cache/
|
||||||
|
*.qcow2
|
||||||
|
*.tar.gz
|
||||||
|
|
||||||
|
# Config with secrets (keep config.yaml.example in git, ignore actual config)
|
||||||
|
config.local.yaml
|
||||||
|
config.runtime.yaml
|
||||||
|
storage.json
|
||||||
|
node_roles.json
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
.env.*.local
|
||||||
|
|
||||||
|
# ---- Logs ----
|
||||||
|
*.log
|
||||||
|
logs/
|
||||||
|
fester.log
|
||||||
|
fester_backend.log
|
||||||
|
mock_agent.log
|
||||||
|
|
||||||
|
# ---- IDE / Editor ----
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
.project
|
||||||
|
.classpath
|
||||||
|
.settings/
|
||||||
|
*.sublime-project
|
||||||
|
*.sublime-workspace
|
||||||
|
.atom/
|
||||||
|
.brackets.json
|
||||||
|
|
||||||
|
# ---- OS ----
|
||||||
|
.DS_Store
|
||||||
|
.DS_Store?
|
||||||
|
._*
|
||||||
|
.Spotlight-V100
|
||||||
|
.Trashes
|
||||||
|
ehthumbs.db
|
||||||
|
Thumbs.db
|
||||||
|
desktop.ini
|
||||||
|
|
||||||
|
# ---- Docker ----
|
||||||
|
docker-compose.override.yml
|
||||||
|
|
||||||
|
# ---- Node (if you ever add a JS build step) ----
|
||||||
|
node_modules/
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
pnpm-debug.log*
|
||||||
|
|
||||||
|
# ---- Build artifacts ----
|
||||||
|
*.o
|
||||||
|
*.a
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
build.tar.gz
|
||||||
|
build/
|
||||||
|
|
||||||
|
# ---- Screenshots (keep in git, but ignore ad-hoc captures) ----
|
||||||
|
# docs/screenshots/ is tracked — these are ad-hoc:
|
||||||
|
/tmp/
|
||||||
|
*.tmp
|
||||||
|
|
||||||
|
# ---- Misc ----
|
||||||
|
*.bak
|
||||||
|
*.orig
|
||||||
|
*.rej
|
||||||
|
*.diff
|
||||||
|
*.patch
|
||||||
|
|
@ -0,0 +1,134 @@
|
||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to Fester are documented here.
|
||||||
|
|
||||||
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Documentation: README.md, quickstart.md, CONTRIBUTING.md, CHANGELOG.md
|
||||||
|
- Tooling: bootstrap.sh, run.sh, pyproject.toml, .gitignore
|
||||||
|
- Docker support (Dockerfile)
|
||||||
|
- Screenshots for all 8 UI pages
|
||||||
|
|
||||||
|
## [0.3.0] — 2026-06-28
|
||||||
|
|
||||||
|
### Added — Storage layer
|
||||||
|
- SQLite-backed persistence for builds, sessions, events, and node states
|
||||||
|
- Btrfs CoW reflink snapshots (`backend/storage/btrfs_cas.py`)
|
||||||
|
- QCOW2 workspace freezing with real mount + rsync (`backend/storage/qcow2_freeze.py`)
|
||||||
|
- tmpfs workspace acceleration
|
||||||
|
- Storage status + config + freeze + reflink + snapshots endpoints
|
||||||
|
|
||||||
|
### Added — tmux integration (E9)
|
||||||
|
- TmuxManager with session create/list/inspect/capture-output/kill
|
||||||
|
- `runtime: "tmux"` action runtime — runs actions in detached tmux sessions
|
||||||
|
- Live output viewer in Sessions page UI
|
||||||
|
- `/api/actions/active`, `/api/actions/{name}/tmux/output`, `/tmux/kill` endpoints
|
||||||
|
|
||||||
|
### Added — mosh/ssh shell attach (E10)
|
||||||
|
- MoshManager with command builder (mosh-via-SSH-bootstrap)
|
||||||
|
- `/api/nodes/{name}/shell` GET + POST endpoints
|
||||||
|
- "⌘ Shell" button on Dashboard node rows (copies command to clipboard)
|
||||||
|
|
||||||
|
### Added — Real engine wiring
|
||||||
|
- BuildRunner wraps PipelineEngine with build lifecycle + persistence
|
||||||
|
- Real DAG execution with proper cwd + env passing
|
||||||
|
- Failure propagation: skipped actions on failed deps
|
||||||
|
- Critical path computation + tagging in events
|
||||||
|
- Debugger pause/resume/step actually controls running engines
|
||||||
|
|
||||||
|
### Added — Cause graph + blast radius
|
||||||
|
- `/api/propagation/{action}` — forward blast radius computation
|
||||||
|
- `/api/cause/explain/{node}` — causal chain for a node
|
||||||
|
- Blast Radius panel in Cause Graph UI page
|
||||||
|
- Failure autopsy with proper session ID correlation
|
||||||
|
|
||||||
|
### Added — Storage + target catalog
|
||||||
|
- `/api/targets` reads from `backend/targets/catalog.py` (6 systems: Gentoo, Buildroot, OpenWrt, ALFS, SourceMage, Lunar)
|
||||||
|
- `/api/targets/arches` + `/api/targets/runtimes` endpoints
|
||||||
|
|
||||||
|
### Added — Health + metrics
|
||||||
|
- `/api/health` endpoint (version, bus_subscribers, ws_clients, builds_known, timeline_events)
|
||||||
|
- Live health indicator in topbar (all pages)
|
||||||
|
- Prometheus metrics aligned with grafana config names (`fester_node_cpu`, `fester_pipeline_actions_total`, etc.)
|
||||||
|
- Pipeline action counters + cache hit counters wired via bus subscriber
|
||||||
|
|
||||||
|
### Added — UI enhancements
|
||||||
|
- New Timeline page (per-node event drill-down)
|
||||||
|
- Node policy dropdowns (preferred/avoid/neutral) on Dashboard
|
||||||
|
- Per-node "Probe" buttons + "Probe All" button
|
||||||
|
- Build cancel + replay buttons on Sessions page
|
||||||
|
- Event filter chips on Dashboard (toggle by event type)
|
||||||
|
|
||||||
|
### Added — CLI overhaul
|
||||||
|
- 35+ subcommands (was 11)
|
||||||
|
- Nested subcommands: `node {list,set-policy,probe,probe-all}`, `cause {explain,trace,events}`, `replay {start,step,reset}`, `debugger {state,pause,resume,step}`
|
||||||
|
- New: `build`, `builds`, `build-info`, `cancel`, `release`, `targets`, `toggle-target`, `blast`, `timeline`, `rewind`, `autopsy`, `policy-list`, `health`
|
||||||
|
- `--watch` flag on `build` command (streams events via WebSocket)
|
||||||
|
|
||||||
|
### Fixed — Critical backend bugs
|
||||||
|
- `api/api.py` order bug (`bus.subscribe()` before `bus = EventBus()`)
|
||||||
|
- 7 broken `_endpoint` symbol imports
|
||||||
|
- `cause_graph.attach_bus` + `emit_debug` defined at module scope (zero indentation)
|
||||||
|
- Three missing files: `timeline_store.py`, `failure_autopsy.py`, `failure_propagation.py`
|
||||||
|
- Two Python module/package name collisions (`scheduler.py` vs `scheduler/`, `nodes.py` vs `nodes/`, `cache.py` vs `cache/`)
|
||||||
|
- `execute_action` passing empty `{}` as cwd → `TypeError`
|
||||||
|
- Build env never reaching subprocess
|
||||||
|
- Debugger/pipeline-control `_ENGINE` always `None`
|
||||||
|
- `failure_propagation` looking for `event.data.deps` (legacy shape)
|
||||||
|
- Autopsy session ID mismatch
|
||||||
|
- `/api/release` was a pure stub
|
||||||
|
- `/api/policy/set` didn't persist
|
||||||
|
- `/ws-debugger` returned canned data, ignored commands
|
||||||
|
- `forgejo.on_push_event` wrong PipelineEngine arity
|
||||||
|
- `pipeline/feedback.py` PolicyEngine() with no args
|
||||||
|
|
||||||
|
### Fixed — Standardization
|
||||||
|
- ~20 bare imports converted to `backend.*` prefix
|
||||||
|
- Two parallel event taxonomies unified (`types.py` re-exports from `schema.py`)
|
||||||
|
- `MinioCache.__init__` made lazy (was crashing on import if MinIO unreachable)
|
||||||
|
- `cache.py` graceful fallback for non-writable paths
|
||||||
|
- `roles_store.py` graceful fallback for non-writable `/etc/fester/`
|
||||||
|
- `FesterEvent` made `node`/`action`/`state` truly optional (had `Optional[str]` type but no default)
|
||||||
|
|
||||||
|
## [0.2.0] — 2026-06-27
|
||||||
|
|
||||||
|
### Added — Real FastAPI backend
|
||||||
|
- `backend/main.py` — FastAPI app with 50+ routes
|
||||||
|
- All routers wired: replay, autopsy, timeline, debugger, pipeline_control, nodes, metrics, cause
|
||||||
|
- WebSocket hub with `/ws`, `/ws-targets`, `/ws-debugger`
|
||||||
|
- Ambient loop for node probing + drift
|
||||||
|
- SQLite storage layer (`backend/storage/sqlite_db.py`)
|
||||||
|
- Node probe module (`backend/nodes/probe.py`)
|
||||||
|
- BuildRunner (`backend/pipeline/runner.py`)
|
||||||
|
|
||||||
|
### Added — UI overhaul (7 pages)
|
||||||
|
- Dashboard (`index.html`) — 3-column layout with cluster stats, live stream, quick actions
|
||||||
|
- Live DAG (`ui/live_dag.html`) — layered Sugiyama-lite layout, SVG edges, click-to-inspect
|
||||||
|
- Replay (`ui/replay.html`) — timeline scrubber, play/pause/step, structured inspector
|
||||||
|
- Sessions (`ui/sessions.html`) — build history + replay links
|
||||||
|
- Metrics (`ui/metrics.html`) — live charts + per-node cards
|
||||||
|
- Cause Graph (`ui/cause.html`) — radial graph visualization
|
||||||
|
- Debugger (`ui/debugger.html`) — step-through execution control
|
||||||
|
|
||||||
|
### Added — Shared infrastructure
|
||||||
|
- `style.css` — full design system (dark theme tokens, panels, badges, buttons, scrollbars)
|
||||||
|
- `ui/app.js` — shared shell (WS auto-reconnect, FesterTargets global, toasts, topbar nav)
|
||||||
|
- `cockpit/fester-module/targets.js` — fixed to expose `FesterTargets` global
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- `live_dag.html` was broken (referenced undefined `FesterTargets`)
|
||||||
|
- `style.css` was 0 bytes
|
||||||
|
- `index.html` was a 30-line throwaway
|
||||||
|
|
||||||
|
## [0.1.0] — Initial commit
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Basic project structure
|
||||||
|
- Backend modules (many stubbed)
|
||||||
|
- Cockpit module
|
||||||
|
- Install scripts
|
||||||
|
- CHEATSHEET.md
|
||||||
|
|
@ -0,0 +1,153 @@
|
||||||
|
# Fester Cluster Build System — Cheatsheet
|
||||||
|
|
||||||
|
## 🧠 Core Concept
|
||||||
|
|
||||||
|
A distributed, DAG-driven build execution system with:
|
||||||
|
|
||||||
|
- real-time scheduling
|
||||||
|
- node-aware load balancing
|
||||||
|
- thermal + policy constraints
|
||||||
|
- cache-aware execution
|
||||||
|
- deterministic replay/debugging
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧩 System Components
|
||||||
|
|
||||||
|
### 1. Scheduler
|
||||||
|
Chooses best node per action using:
|
||||||
|
|
||||||
|
- CPU load
|
||||||
|
- temperature
|
||||||
|
- policy rules
|
||||||
|
- historical intelligence feedback
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. Pipeline Engine
|
||||||
|
Executes DAG actions sequentially or step-debugged.
|
||||||
|
|
||||||
|
Supports:
|
||||||
|
- live execution
|
||||||
|
- interactive stepping
|
||||||
|
- replay mode
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. Timeline Store
|
||||||
|
Immutable event log of entire system.
|
||||||
|
|
||||||
|
Used for:
|
||||||
|
- replay
|
||||||
|
- debugging
|
||||||
|
- autopsy analysis
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. Failure System
|
||||||
|
Detects and propagates failure:
|
||||||
|
|
||||||
|
- backward → root cause candidates
|
||||||
|
- forward → downstream impact
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. Critical Path Analyzer
|
||||||
|
Identifies bottleneck chain in DAG execution.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. Cache Layer
|
||||||
|
Supports:
|
||||||
|
- distributed artifact reuse
|
||||||
|
- MinIO backend
|
||||||
|
- future: Btrfs/QCOW2 snapshot acceleration
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. Debugger Mode
|
||||||
|
Interactive execution control:
|
||||||
|
|
||||||
|
- pause
|
||||||
|
- step
|
||||||
|
- resume
|
||||||
|
- scheduler preview before execution
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧭 Execution Flow
|
||||||
|
|
||||||
|
1. Build graph generated from spec
|
||||||
|
2. Scheduler selects node per action
|
||||||
|
3. Timeline records decision
|
||||||
|
4. Executor runs action on node
|
||||||
|
5. Events streamed to UI
|
||||||
|
6. Failures propagate through DAG
|
||||||
|
7. Optional debugger intercepts execution
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📡 Event Types
|
||||||
|
|
||||||
|
- `node`
|
||||||
|
- `pipeline`
|
||||||
|
- `schedule_decision`
|
||||||
|
- `failure_propagation`
|
||||||
|
- `debugger_preview`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🖥 UI Modes
|
||||||
|
|
||||||
|
### Live Mode
|
||||||
|
- real-time DAG rendering
|
||||||
|
- node state updates
|
||||||
|
|
||||||
|
### Debug Mode
|
||||||
|
- step execution
|
||||||
|
- scheduler inspection
|
||||||
|
- manual control
|
||||||
|
|
||||||
|
### Autopsy Mode
|
||||||
|
- failure root cause analysis
|
||||||
|
- dependency tracing
|
||||||
|
- critical path overlay
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ⚙️ Node Selection Model
|
||||||
|
|
||||||
|
Score-based weighted system:
|
||||||
|
|
||||||
|
- CPU availability
|
||||||
|
- thermal headroom
|
||||||
|
- policy constraints
|
||||||
|
- historical instability penalties
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧪 Execution Backends (planned)
|
||||||
|
|
||||||
|
- distcc
|
||||||
|
- LXC
|
||||||
|
- libvirt
|
||||||
|
- native execution
|
||||||
|
- cross-compilation toolchains
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📦 Cache Strategy
|
||||||
|
|
||||||
|
- local ccache
|
||||||
|
- shared MinIO cache
|
||||||
|
- future: Btrfs/QCOW2 snapshot acceleration
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔐 Design Principle
|
||||||
|
|
||||||
|
> The system must always be replayable, explainable, and deterministic.
|
||||||
|
|
||||||
|
No hidden state.
|
||||||
|
No opaque scheduling.
|
||||||
|
Everything is observable.
|
||||||
|
|
@ -0,0 +1,145 @@
|
||||||
|
# Contributing to Fester
|
||||||
|
|
||||||
|
Thanks for your interest in improving Fester! This document covers the basics.
|
||||||
|
|
||||||
|
## 🚀 Quick Start for Contributors
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Fork + clone
|
||||||
|
git clone https://git.dcos.net/dcosnet/fester.git
|
||||||
|
cd fester
|
||||||
|
|
||||||
|
# 2. Bootstrap dev environment (creates venv, installs deps + dev tools)
|
||||||
|
./bootstrap.sh --dev
|
||||||
|
|
||||||
|
# 3. Run the backend in dev mode (auto-reload on changes)
|
||||||
|
./run.sh --dev
|
||||||
|
|
||||||
|
# 4. Run tests (when we have them)
|
||||||
|
pytest
|
||||||
|
|
||||||
|
# 5. Lint + format
|
||||||
|
ruff check .
|
||||||
|
ruff format .
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🏗️ Architecture Overview
|
||||||
|
|
||||||
|
Read **[README.md](README.md)** first — it has the full architecture diagram and project layout.
|
||||||
|
|
||||||
|
The key insight: Fester has a strict separation between:
|
||||||
|
- **EventBus** — the singleton message bus (all events flow through it)
|
||||||
|
- **Subscribers** — bus listeners that index/journal/broadcast events (TimelineStore, cause_graph, FailurePropagator, the WS broadcaster)
|
||||||
|
- **Routers** — FastAPI APIRouters under `backend/api/` that expose REST endpoints
|
||||||
|
- **UI** — vanilla HTML/JS pages under `ui/` that subscribe to the WS stream and call REST endpoints
|
||||||
|
|
||||||
|
When you add a new feature, ask:
|
||||||
|
1. Does this emit events? → Use `bus.emit(EventType.X, ...)` from `backend/events/schema.py`
|
||||||
|
2. Does this need persistence? → Add a method to `backend/storage/sqlite_db.py:Storage`
|
||||||
|
3. Does this need a UI? → Add a page under `ui/` and a nav entry in `ui/app.js:Fester.mountShell()`
|
||||||
|
4. Does this need a CLI command? → Add a subcommand to `cli/fester.py`
|
||||||
|
|
||||||
|
## 📝 Coding Standards
|
||||||
|
|
||||||
|
### Python
|
||||||
|
|
||||||
|
- **Python 3.12+** — use modern syntax (match statements, type hints, etc.)
|
||||||
|
- **Type hints** — add them to new code; don't worry about retrofitting old code
|
||||||
|
- **Imports** — use `from backend.X.Y import Z` (not bare `from X.Y import Z`)
|
||||||
|
- **EventBus** — always emit events via `bus.emit()`, never mutate state directly from a request handler
|
||||||
|
- **Pydantic** — use `BaseModel` for request bodies in routers
|
||||||
|
- **Async** — use `async def` for endpoints that touch the DB or do I/O
|
||||||
|
|
||||||
|
### JavaScript (UI)
|
||||||
|
|
||||||
|
- **Vanilla JS** — no frameworks (React, Vue, etc.). Keep it dependency-free.
|
||||||
|
- **Shared shell** — every page calls `Fester.mountShell('page-id')` to get the topbar + WS connection
|
||||||
|
- **API calls** — use `Fester.api(path, opts)` (returns parsed JSON)
|
||||||
|
- **Toasts** — use `Fester.toast(msg, kind)` for user notifications
|
||||||
|
- **No build step** — pages load directly via `<script src="/ui/app.js">`
|
||||||
|
|
||||||
|
### CSS
|
||||||
|
|
||||||
|
- **Design tokens** — use `var(--token)` (defined in `style.css`). Don't hardcode colors.
|
||||||
|
- **Dark theme only** — no light mode. Tokens are already dark.
|
||||||
|
|
||||||
|
## 🧪 Testing
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run all tests
|
||||||
|
pytest
|
||||||
|
|
||||||
|
# Run a specific test file
|
||||||
|
pytest tests/test_pipeline.py
|
||||||
|
|
||||||
|
# Run with coverage
|
||||||
|
pytest --cov=backend --cov-report=html
|
||||||
|
|
||||||
|
# Run only fast tests
|
||||||
|
pytest -m "not slow"
|
||||||
|
```
|
||||||
|
|
||||||
|
Tests live in `tests/` (we don't have many yet — contributions welcome!).
|
||||||
|
|
||||||
|
## 🔍 Code Quality
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Lint
|
||||||
|
ruff check .
|
||||||
|
|
||||||
|
# Format
|
||||||
|
ruff format .
|
||||||
|
|
||||||
|
# Type check
|
||||||
|
mypy backend/
|
||||||
|
|
||||||
|
# Check for syntax errors in inline <script> blocks
|
||||||
|
node scripts/check_syntax.js ui/*.html index.html
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📦 Commit Conventions
|
||||||
|
|
||||||
|
Use [Conventional Commits](https://www.conventionalcommits.org/):
|
||||||
|
|
||||||
|
```
|
||||||
|
feat: add tmux output viewer to sessions page
|
||||||
|
fix: pass real cwd to execute_action instead of empty dict
|
||||||
|
docs: add quickstart guide
|
||||||
|
chore: update dependencies
|
||||||
|
refactor: split BuildRunner from PipelineEngine
|
||||||
|
test: add tests for failure_propagation
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🐛 Bug Reports
|
||||||
|
|
||||||
|
When filing an issue, include:
|
||||||
|
1. Fester version (`fester health` output)
|
||||||
|
2. Python version (`python3 --version`)
|
||||||
|
3. OS + distro
|
||||||
|
4. Steps to reproduce
|
||||||
|
5. Expected vs actual behavior
|
||||||
|
6. Relevant log output (`/tmp/fester_backend.log`)
|
||||||
|
7. Screenshots if UI-related
|
||||||
|
|
||||||
|
## ✨ Pull Requests
|
||||||
|
|
||||||
|
1. Fork the repo + create a branch: `git checkout -b feat/my-feature`
|
||||||
|
2. Make your changes
|
||||||
|
3. Run `ruff check . && ruff format .` before committing
|
||||||
|
4. Write a clear commit message (Conventional Commits)
|
||||||
|
5. Open a PR with:
|
||||||
|
- What changed + why
|
||||||
|
- Screenshots (if UI changes)
|
||||||
|
- Test results
|
||||||
|
- Any breaking changes
|
||||||
|
|
||||||
|
## 🆘 Getting Help
|
||||||
|
|
||||||
|
- Read **[CHEATSHEET.md](CHEATSHEET.md)** for the operator survival guide
|
||||||
|
- Read **[quickstart.md](quickstart.md)** for getting started
|
||||||
|
- Browse the **[API docs](http://localhost:8080/docs)** at runtime
|
||||||
|
- Check existing issues: https://git.dcos.net/dcosnet/fester/issues
|
||||||
|
|
||||||
|
## 📜 License
|
||||||
|
|
||||||
|
By contributing, you agree that your contributions will be licensed under the AGPL-3.0 — see **[LICENSE](LICENSE)**.
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
🧾 SOFTWARE PROVIDED “AS IS” CLAUSE
|
||||||
|
📄 Disclaimer of Warranty and Liability
|
||||||
|
|
||||||
|
This software is provided “as is”, without warranty of any kind, express or implied. This includes, but is not limited to, warranties of merchantability, fitness for a particular purpose, non-infringement, and any warranties arising from course of dealing or usage of trade.
|
||||||
|
|
||||||
|
The authors and contributors make no representation or guarantee that:
|
||||||
|
|
||||||
|
the software will function uninterrupted or error-free,
|
||||||
|
builds, scheduling decisions, or execution results will be correct or deterministic,
|
||||||
|
distributed execution across heterogeneous nodes will be stable or safe in all environments,
|
||||||
|
caching, snapshotting, or replay features will not result in data loss or corruption,
|
||||||
|
integrations (including but not limited to libvirt, LXC, tmux, mosh, distcc, MinIO, Prometheus, Grafana, Forgejo, or any external toolchain) will behave as expected in all configurations.
|
||||||
|
⚙️ Operational Risk Acknowledgement
|
||||||
|
|
||||||
|
This system is capable of:
|
||||||
|
|
||||||
|
executing distributed code compilation,
|
||||||
|
scheduling workloads across physical and virtual nodes,
|
||||||
|
modifying system-level environments and build artifacts,
|
||||||
|
managing cache layers, snapshots, and execution state.
|
||||||
|
|
||||||
|
You acknowledge that:
|
||||||
|
|
||||||
|
misconfiguration may result in system instability, data loss, or resource exhaustion,
|
||||||
|
execution across remote nodes may cause unpredictable behavior under load,
|
||||||
|
performance optimization logic may prioritize throughput over safety unless explicitly configured otherwise.
|
||||||
|
🔒 No Liability
|
||||||
|
|
||||||
|
In no event shall the authors, maintainers, or contributors be liable for any:
|
||||||
|
|
||||||
|
direct, indirect, incidental, or consequential damages,
|
||||||
|
loss of data, revenue, or system availability,
|
||||||
|
hardware degradation or thermal damage,
|
||||||
|
build failures or deployment issues,
|
||||||
|
security vulnerabilities arising from system configuration or integration.
|
||||||
|
🧠 Responsibility of Use
|
||||||
|
|
||||||
|
You are solely responsible for:
|
||||||
|
|
||||||
|
validating build targets and toolchains,
|
||||||
|
verifying scheduling and execution policies,
|
||||||
|
ensuring safe node configuration and thermal limits,
|
||||||
|
reviewing cache and snapshot behavior before production use,
|
||||||
|
controlling access to the system and its APIs.
|
||||||
|
⚡ High-Risk System Notice
|
||||||
|
|
||||||
|
This software operates in a high-concurrency, distributed execution domain. It is not intended for safety-critical systems unless independently audited and hardened for such use cases.
|
||||||
|
|
@ -0,0 +1,77 @@
|
||||||
|
# Dockerfile for Fester
|
||||||
|
# ============================================================
|
||||||
|
# Multi-stage build: installs Python deps, copies code, runs uvicorn.
|
||||||
|
#
|
||||||
|
# Build:
|
||||||
|
# docker build -t fester .
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# docker run -p 8080:8080 \
|
||||||
|
# -v fester-db:/var/lib/fester \
|
||||||
|
# -v fester-config:/etc/fester \
|
||||||
|
# fester
|
||||||
|
#
|
||||||
|
# Or with docker-compose (see docker-compose.yml):
|
||||||
|
# docker-compose up
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
FROM python:3.12-slim AS base
|
||||||
|
|
||||||
|
# Install system dependencies (optional tools activate features when present)
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
tmux \
|
||||||
|
mosh \
|
||||||
|
openssh-client \
|
||||||
|
qemu-utils \
|
||||||
|
rsync \
|
||||||
|
btrfs-progs \
|
||||||
|
curl \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create fester user + directories
|
||||||
|
RUN useradd -m -u 1000 fester && \
|
||||||
|
mkdir -p /var/lib/fester/cas /var/lib/fester/snapshots /var/lib/fester/cache /etc/fester && \
|
||||||
|
chown -R fester:fester /var/lib/fester /etc/fester
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy pyproject.toml first (better layer caching)
|
||||||
|
COPY pyproject.toml ./
|
||||||
|
|
||||||
|
# Install Python dependencies
|
||||||
|
RUN pip install --no-cache-dir \
|
||||||
|
"fastapi>=0.128" \
|
||||||
|
"uvicorn[standard]>=0.30" \
|
||||||
|
"websockets>=12" \
|
||||||
|
"pydantic>=2" \
|
||||||
|
"aiohttp>=3.9" \
|
||||||
|
"minio>=7.2" \
|
||||||
|
"prometheus-client>=0.20" \
|
||||||
|
"PyYAML>=6" \
|
||||||
|
"requests>=2.31" \
|
||||||
|
"websocket-client>=1.7"
|
||||||
|
|
||||||
|
# Copy the rest of the code
|
||||||
|
COPY --chown=fester:fester . .
|
||||||
|
|
||||||
|
# Make scripts executable
|
||||||
|
RUN chmod +x run.sh cli/fester.py 2>/dev/null || true
|
||||||
|
|
||||||
|
# Switch to non-root user
|
||||||
|
USER fester
|
||||||
|
|
||||||
|
# Default config (can be overridden via volume mount at /etc/fester/config.yaml)
|
||||||
|
ENV FESTER_DB_PATH=/var/lib/fester/fester.db
|
||||||
|
ENV FESTER_CONFIG=/app/config.yaml
|
||||||
|
ENV FESTER_NO_DRIFT=0
|
||||||
|
ENV FESTER_AUTOBUILD=0
|
||||||
|
|
||||||
|
# Expose the backend port
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
# Health check
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||||
|
CMD curl -f http://localhost:8080/api/health || exit 1
|
||||||
|
|
||||||
|
# Run the backend
|
||||||
|
CMD ["python3", "-m", "uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||||
|
|
@ -0,0 +1,679 @@
|
||||||
|
========================================================================
|
||||||
|
PROJECT: fester
|
||||||
|
COPYRIGHT: Copyright (C) 2026 dcos.net
|
||||||
|
HOMEPAGE: https://git.dcos.net/dcosnet/fester/
|
||||||
|
REPOSITORY: https://git.dcos.net/dcosnet/fester/
|
||||||
|
LICENSE: GNU Affero General Public License v3.0 (AGPL-3.0)
|
||||||
|
========================================================================
|
||||||
|
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||||
|
Version 3, 19 November 2007
|
||||||
|
|
||||||
|
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies
|
||||||
|
of this license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
Preamble
|
||||||
|
|
||||||
|
The GNU Affero General Public License is a free, copyleft license for
|
||||||
|
software and other kinds of works, specifically designed to ensure
|
||||||
|
cooperation with the community in the case of network server software.
|
||||||
|
|
||||||
|
The licenses for most software and other practical works are designed
|
||||||
|
to take away your freedom to share and change the works. By contrast,
|
||||||
|
our General Public Licenses are intended to guarantee your freedom to
|
||||||
|
share and change all versions of a program--to make sure it remains free
|
||||||
|
software for all its users.
|
||||||
|
|
||||||
|
When we speak of free software, we are referring to freedom, not
|
||||||
|
price. Our General Public Licenses are designed to make sure that you
|
||||||
|
have the freedom to distribute copies of free software (and charge for
|
||||||
|
them if you wish), that you receive source code or can get it if you
|
||||||
|
want it, that you can change the software or use pieces of it in new
|
||||||
|
free programs, and that you know you can do these things.
|
||||||
|
|
||||||
|
Developers that use our General Public Licenses protect your rights
|
||||||
|
with two steps: (1) assert copyright on the software, and (2) offer
|
||||||
|
you this License which gives you legal permission to copy, distribute
|
||||||
|
and/or modify the software.
|
||||||
|
|
||||||
|
A secondary benefit of defending all users' freedom is that
|
||||||
|
improvements made in alternate versions of the program, if they
|
||||||
|
receive widespread use, become available for other developers to
|
||||||
|
incorporate. Many developers of free software are heartened and
|
||||||
|
encouraged by the resulting cooperation. However, in the case of
|
||||||
|
software used on network servers, this result may fail to come about.
|
||||||
|
The GNU General Public License permits making a modified version and
|
||||||
|
letting the public access it on a server without ever releasing its
|
||||||
|
source code to the public.
|
||||||
|
|
||||||
|
The GNU Affero General Public License is designed specifically to
|
||||||
|
ensure that, in such cases, the modified source code becomes available
|
||||||
|
to the community. It requires the operator of a network server to
|
||||||
|
provide the source code of the modified version running there to the
|
||||||
|
users of that server. Therefore, public use of a modified version, on
|
||||||
|
a publicly accessible server, gives the public access to the source
|
||||||
|
code of the modified version.
|
||||||
|
|
||||||
|
An older license, called the Affero General Public License and
|
||||||
|
published by Affero, was designed to accomplish similar goals. This is
|
||||||
|
a different license, not a version of the Affero GPL, but Affero has
|
||||||
|
released a new version of the Affero GPL which permits relicensing under
|
||||||
|
this license.
|
||||||
|
|
||||||
|
The precise terms and conditions for copying, distribution and
|
||||||
|
modification follow.
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
0. Definitions.
|
||||||
|
|
||||||
|
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||||
|
|
||||||
|
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||||
|
works, such as semiconductor masks.
|
||||||
|
|
||||||
|
"The Program" refers to any copyrightable work licensed under this
|
||||||
|
License. Each licensee is addressed as "you". "Licensees" and
|
||||||
|
"recipients" may be individuals or organizations.
|
||||||
|
|
||||||
|
To "modify" a work means to copy from or adapt all or part of the work
|
||||||
|
in a fashion requiring copyright permission, other than the making of an
|
||||||
|
exact copy. The resulting work is called a "modified version" of the
|
||||||
|
earlier work or a work "based on" the earlier work.
|
||||||
|
|
||||||
|
A "covered work" means either the unmodified Program or a work based
|
||||||
|
on the Program.
|
||||||
|
|
||||||
|
To "propagate" a work means to do anything with it that, without
|
||||||
|
permission, would make you directly or secondarily liable for
|
||||||
|
infringement under applicable copyright law, except executing it on a
|
||||||
|
computer or modifying a private copy. Propagation includes copying,
|
||||||
|
distribution (with or without modification), making available to the
|
||||||
|
public, and in some countries other activities as well.
|
||||||
|
|
||||||
|
To "convey" a work means any kind of propagation that enables other
|
||||||
|
parties to make or receive copies. Mere interaction with a user through
|
||||||
|
a computer network, with no transfer of a copy, is not conveying.
|
||||||
|
|
||||||
|
An interactive user interface displays "Appropriate Legal Notices"
|
||||||
|
to the extent that it includes a convenient and prominently visible
|
||||||
|
feature that (1) displays an appropriate copyright notice, and (2)
|
||||||
|
tells the user that there is no warranty for the work (except to the
|
||||||
|
extent that warranties are provided), that licensees may convey the
|
||||||
|
work under this License, and how to view a copy of this License. If
|
||||||
|
the interface presents a list of user commands or options, such as a
|
||||||
|
menu, a prominent item in the list meets this criterion.
|
||||||
|
|
||||||
|
1. Source Code.
|
||||||
|
|
||||||
|
The "source code" for a work means the preferred form of the work
|
||||||
|
for making modifications to it. "Object code" means any non-source
|
||||||
|
form of a work.
|
||||||
|
|
||||||
|
A "Standard Interface" means an interface that either is an official
|
||||||
|
standard defined by a recognized standards body, or, in the case of
|
||||||
|
interfaces specified for a particular programming language, one that
|
||||||
|
is widely used among developers working in that language.
|
||||||
|
|
||||||
|
The "System Libraries" of an executable work include anything, other
|
||||||
|
than the work as a whole, that (a) is included in the normal form of
|
||||||
|
packaging a Major Component, but which is not part of that Major
|
||||||
|
Component, and (b) serves only to enable use of the work with that
|
||||||
|
Major Component, or to implement a Standard Interface for which an
|
||||||
|
implementation is available to the public in source code form. A
|
||||||
|
"Major Component", in this context, means a major essential component
|
||||||
|
(kernel, window system, and so on) of the specific operating system
|
||||||
|
(if any) on which the executable work runs, or a compiler used to
|
||||||
|
produce the work, or an object code interpreter used to run it.
|
||||||
|
|
||||||
|
The "Corresponding Source" for a work in object code form means all
|
||||||
|
the source code needed to generate, install, and (for an executable
|
||||||
|
work) run the object code and to modify the work, including scripts to
|
||||||
|
control those activities. However, it does not include the work's
|
||||||
|
System Libraries, or general-purpose tools or generally available free
|
||||||
|
programs which are used unmodified in performing those activities but
|
||||||
|
which are not part of the work. For example, Corresponding Source
|
||||||
|
includes interface definition files associated with source files for
|
||||||
|
the work, and the source code for shared libraries and dynamically
|
||||||
|
linked subprograms that the work is specifically designed to require,
|
||||||
|
such as by intimate data communication or control flow between those
|
||||||
|
subprograms and other parts of the work.
|
||||||
|
|
||||||
|
The Corresponding Source need not include anything that users
|
||||||
|
can regenerate automatically from other parts of the Corresponding
|
||||||
|
Source.
|
||||||
|
|
||||||
|
The Corresponding Source for a work in source code form is that
|
||||||
|
same work.
|
||||||
|
|
||||||
|
2. Basic Permissions.
|
||||||
|
|
||||||
|
All rights granted under this License are granted for the term of
|
||||||
|
copyright on the Program, and are irrevocable provided the stated
|
||||||
|
conditions are met. This License explicitly affirms your unlimited
|
||||||
|
permission to run the unmodified Program. The output from running a
|
||||||
|
covered work is covered by this License only if the output, given its
|
||||||
|
content, constitutes a covered work. This License acknowledges your
|
||||||
|
rights of fair use or other equivalent, as provided by copyright law.
|
||||||
|
|
||||||
|
You may make, run and propagate covered works that you do not
|
||||||
|
convey, without conditions so long as your license otherwise remains
|
||||||
|
in force. You may convey covered works to others for the sole purpose
|
||||||
|
of having them make modifications exclusively for you, or provide you
|
||||||
|
with facilities for running those works, provided that you comply with
|
||||||
|
the terms of this License in conveying all material for which you do
|
||||||
|
not control copyright. Those thus making or running the covered works
|
||||||
|
for you must do so exclusively on your behalf, under your direction
|
||||||
|
and control, on terms that prohibit them from making any copies of
|
||||||
|
your copyrighted material outside their relationship with you.
|
||||||
|
|
||||||
|
Conveying under any other circumstances is permitted solely under
|
||||||
|
the conditions stated below. Sublicensing is not allowed; section 10
|
||||||
|
makes it unnecessary.
|
||||||
|
|
||||||
|
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||||
|
|
||||||
|
No covered work shall be deemed part of an effective technological
|
||||||
|
measure under any applicable law fulfilling obligations under article
|
||||||
|
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||||
|
similar laws prohibiting or restricting circumvention of such
|
||||||
|
measures.
|
||||||
|
|
||||||
|
When you convey a covered work, you waive any legal power to forbid
|
||||||
|
circumvention of technological measures to the extent such circumvention
|
||||||
|
is effected by exercising rights under this License with respect to
|
||||||
|
the covered work, and you disclaim any intention to limit operation or
|
||||||
|
modification of the work as a means of enforcing, against the work's
|
||||||
|
users, your or third parties' legal rights to forbid circumvention of
|
||||||
|
technological measures.
|
||||||
|
|
||||||
|
4. Conveying Verbatim Copies.
|
||||||
|
|
||||||
|
You may convey verbatim copies of the Program's source code as you
|
||||||
|
receive it, in any medium, provided that you conspicuously and
|
||||||
|
appropriately publish on each copy an appropriate copyright notice;
|
||||||
|
keep intact all notices stating that this License and any
|
||||||
|
non-permissive terms added in accord with section 7 apply to the code;
|
||||||
|
keep intact all notices of the absence of any warranty; and give all
|
||||||
|
recipients a copy of this License along with the Program.
|
||||||
|
|
||||||
|
You may charge any price or no price for each copy that you convey,
|
||||||
|
and you may offer support or warranty protection for a fee.
|
||||||
|
|
||||||
|
5. Conveying Modified Source Versions.
|
||||||
|
|
||||||
|
You may convey a work based on the Program, or the modifications to
|
||||||
|
produce it from the Program, in the form of source code under the
|
||||||
|
terms of section 4, provided that you also meet all of these conditions:
|
||||||
|
|
||||||
|
a) The work must carry prominent notices stating that you modified
|
||||||
|
it, and giving a relevant date.
|
||||||
|
|
||||||
|
b) The work must carry prominent notices stating that it is
|
||||||
|
released under this License and any conditions added under section
|
||||||
|
7. This requirement modifies the requirement in section 4 to
|
||||||
|
"keep intact all notices".
|
||||||
|
|
||||||
|
c) You must license the entire work, as a whole, under this
|
||||||
|
License to anyone who comes into possession of a copy. This
|
||||||
|
License will therefore apply, along with any applicable section 7
|
||||||
|
additional terms, to the whole of the work, and all its parts,
|
||||||
|
regardless of how they are packaged. This License gives no
|
||||||
|
permission to license the work in any other way, but it does not
|
||||||
|
invalidate such permission if you have separately received it.
|
||||||
|
|
||||||
|
d) If the work has interactive user interfaces, each must display
|
||||||
|
Appropriate Legal Notices; however, if the Program has interactive
|
||||||
|
interfaces that do not display Appropriate Legal Notices, your
|
||||||
|
work need not make them do so.
|
||||||
|
|
||||||
|
A compilation of a covered work with other separate and independent
|
||||||
|
works, which are not by their nature extensions of the covered work,
|
||||||
|
and which are not combined with it such as to form a larger program,
|
||||||
|
in or on a volume of a storage or distribution medium, is called an
|
||||||
|
"aggregate" if the compilation and its resulting copyright are not
|
||||||
|
used to limit the access or legal rights of the compilation's users
|
||||||
|
beyond what the individual works permit. Inclusion of a covered work
|
||||||
|
in an aggregate does not cause this License to apply to the other
|
||||||
|
parts of the aggregate.
|
||||||
|
|
||||||
|
6. Conveying Non-Source Forms.
|
||||||
|
|
||||||
|
You may convey a covered work in object code form under the terms
|
||||||
|
of sections 4 and 5, provided that you also convey the
|
||||||
|
machine-readable Corresponding Source under the terms of this License,
|
||||||
|
in one of these ways:
|
||||||
|
|
||||||
|
a) Convey the object code in, or embodied in, a physical product
|
||||||
|
(including a physical distribution medium), accompanied by the
|
||||||
|
Corresponding Source fixed on a durable physical medium
|
||||||
|
customarily used for software interchange.
|
||||||
|
|
||||||
|
b) Convey the object code in, or embodied in, a physical product
|
||||||
|
(including a physical distribution medium), accompanied by a
|
||||||
|
written offer, valid for at least three years and valid for as
|
||||||
|
long as you offer spare parts or customer support for that product
|
||||||
|
model, to give anyone who possesses the object code either (1) a
|
||||||
|
copy of the Corresponding Source for all the software in the
|
||||||
|
product that is covered by this License, on a durable physical
|
||||||
|
medium customarily used for software interchange, for a price no
|
||||||
|
more than your reasonable cost of physically performing this
|
||||||
|
conveying of source, or (2) access to copy the
|
||||||
|
Corresponding Source from a network server at no charge.
|
||||||
|
|
||||||
|
c) Convey individual copies of the object code with a copy of the
|
||||||
|
written offer to provide the Corresponding Source. This
|
||||||
|
alternative is allowed only occasionally and noncommercially, and
|
||||||
|
only if you received the object code with such an offer, in accord
|
||||||
|
with subsection 6b.
|
||||||
|
|
||||||
|
d) Convey the object code by offering access from a designated
|
||||||
|
place (gratis or for a charge), and offer equivalent access to the
|
||||||
|
Corresponding Source in the same way through the same place at no
|
||||||
|
further charge. You need not require recipients to copy the
|
||||||
|
Corresponding Source along with the object code. If the place to
|
||||||
|
copy the object code is a network server, the Corresponding Source
|
||||||
|
may be on a different server (operated by you or a third party)
|
||||||
|
that supports equivalent copying facilities, provided you maintain
|
||||||
|
clear directions next to the object code saying where to find the
|
||||||
|
Corresponding Source. Regardless of what server hosts the
|
||||||
|
Corresponding Source, you remain obligated to ensure that it is
|
||||||
|
available for as long as needed to satisfy these requirements.
|
||||||
|
|
||||||
|
e) Convey the object code using peer-to-peer transmission, provided
|
||||||
|
you inform other peers where the object code and Corresponding
|
||||||
|
Source of the work are being offered to the general public at no
|
||||||
|
charge under subsection 6d.
|
||||||
|
|
||||||
|
A separable portion of the object code, whose source code is excluded
|
||||||
|
from the Corresponding Source as a System Library, need not be
|
||||||
|
included in conveying the object code work.
|
||||||
|
|
||||||
|
A "User Product" is either (1) a "consumer product", which means any
|
||||||
|
tangible personal property which is normally used for personal, family,
|
||||||
|
or household purposes, or (2) anything designed or sold for incorporation
|
||||||
|
into a dwelling. In determining whether a product is a consumer product,
|
||||||
|
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||||
|
product received by a particular user, "normally used" refers to a
|
||||||
|
typical or common use of that class of product, regardless of the status
|
||||||
|
of the particular user or of the way in which the particular user
|
||||||
|
actually uses, or expects or is expected to use, the product. A product
|
||||||
|
is a consumer product regardless of whether the product has substantial
|
||||||
|
commercial, industrial or non-consumer uses, unless such uses represent
|
||||||
|
the only significant mode of use of the product.
|
||||||
|
|
||||||
|
"Installation Information" for a User Product means any methods,
|
||||||
|
procedures, authorization keys, or other information required to install
|
||||||
|
and execute modified versions of a covered work in that User Product from
|
||||||
|
a modified version of its Corresponding Source. The information must
|
||||||
|
suffice to ensure that the continued functioning of the modified object
|
||||||
|
code is in no case prevented or interfered with solely because
|
||||||
|
modification has been made.
|
||||||
|
|
||||||
|
If you convey an object code work under this section in, or with, or
|
||||||
|
specifically for use in, a User Product, and the conveying occurs as
|
||||||
|
part of a transaction in which the right of possession and use of the
|
||||||
|
User Product is transferred to the recipient in perpetuity or for a
|
||||||
|
fixed term (regardless of how the transaction is characterized), the
|
||||||
|
Corresponding Source conveyed under this section must be accompanied
|
||||||
|
by the Installation Information. But this requirement does not apply
|
||||||
|
if neither you nor any third party retains the ability to install
|
||||||
|
modified object code on the User Product (for example, the work has
|
||||||
|
been installed in ROM).
|
||||||
|
|
||||||
|
The requirement to provide Installation Information does not include a
|
||||||
|
requirement to continue to provide support service, warranty, or updates
|
||||||
|
for a work that has been modified or installed by the recipient, or for
|
||||||
|
the User Product in which it has been modified or installed. Access to a
|
||||||
|
network may be denied when the modification itself materially and
|
||||||
|
adversely affects the operation of the network or violates the rules and
|
||||||
|
protocols for communication across the network.
|
||||||
|
|
||||||
|
Corresponding Source conveyed, and Installation Information provided,
|
||||||
|
in accord with this section must be in a format that is publicly
|
||||||
|
documented (and with an implementation available to the public in
|
||||||
|
source code form), and must require no special password or key for
|
||||||
|
unpacking, reading or copying.
|
||||||
|
|
||||||
|
7. Additional Terms.
|
||||||
|
|
||||||
|
"Additional permissions" are terms that supplement the terms of this
|
||||||
|
License by making exceptions from one or more of its conditions.
|
||||||
|
Additional permissions that are applicable to the entire Program shall
|
||||||
|
be treated as though they were included in this License, to the extent
|
||||||
|
that they are valid under applicable law. If additional permissions
|
||||||
|
apply only to part of the Program, that part may be used separately
|
||||||
|
under those permissions, but the entire Program remains governed by
|
||||||
|
this License without regard to the additional permissions.
|
||||||
|
|
||||||
|
When you convey a copy of a covered work, you may at your option
|
||||||
|
remove any additional permissions from that copy, or from any part of
|
||||||
|
it. (Additional permissions may be written to require their own
|
||||||
|
removal in certain cases when you modify the work.) You may place
|
||||||
|
additional permissions on material, added by you to a covered work,
|
||||||
|
for which you have or can give appropriate copyright permission.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, for material you
|
||||||
|
add to a covered work, you may (if authorized by the copyright holders of
|
||||||
|
that material) supplement the terms of this License with terms:
|
||||||
|
|
||||||
|
a) Disclaiming warranty or limiting liability differently from the
|
||||||
|
terms of sections 15 and 16 of this License; or
|
||||||
|
|
||||||
|
b) Requiring preservation of specified reasonable legal notices or
|
||||||
|
author attributions in that material or in the Appropriate Legal
|
||||||
|
Notices displayed by works containing it; or
|
||||||
|
|
||||||
|
c) Prohibiting misrepresentation of the origin of that material, or
|
||||||
|
requiring that modified versions of such material be marked in
|
||||||
|
reasonable ways as different from the original version; or
|
||||||
|
|
||||||
|
d) Limiting the use for publicity purposes of names of licensors or
|
||||||
|
authors of the material; or
|
||||||
|
|
||||||
|
e) Declining to grant rights under trademark law for use of some
|
||||||
|
trade names, trademarks, or service marks; or
|
||||||
|
|
||||||
|
f) Requiring indemnification of licensors and authors of that
|
||||||
|
material by anyone who conveys the material (or modified versions of
|
||||||
|
it) with contractual assumptions of liability to the recipient, for
|
||||||
|
any liability that these contractual assumptions directly impose on
|
||||||
|
those licensors and authors.
|
||||||
|
|
||||||
|
All other non-permissive additional terms are considered "further
|
||||||
|
restrictions" within the meaning of section 10. If the Program as you
|
||||||
|
received it, or any part of it, contains a notice stating that it is
|
||||||
|
governed by this License along with a term that is a further
|
||||||
|
restriction, you may remove that term. If a license document contains
|
||||||
|
a further restriction but permits relicensing or conveying under this
|
||||||
|
License, you may add to a covered work material governed by the terms
|
||||||
|
of that license document, provided that the further restriction does
|
||||||
|
not survive such relicensing or conveying.
|
||||||
|
|
||||||
|
If you add terms to a covered work in accord with this section, you
|
||||||
|
must place, in the relevant source files, a statement of the
|
||||||
|
additional terms that apply to those files, or a notice indicating
|
||||||
|
where to find the applicable terms.
|
||||||
|
|
||||||
|
Additional terms, permissive or non-permissive, may be stated in the
|
||||||
|
form of a separately written license, or stated as exceptions;
|
||||||
|
the above requirements apply either way.
|
||||||
|
|
||||||
|
8. Termination.
|
||||||
|
|
||||||
|
You may not propagate or modify a covered work except as expressly
|
||||||
|
provided under this License. Any attempt otherwise to propagate or
|
||||||
|
modify it is void, and will automatically terminate your rights under
|
||||||
|
this License (including any patent licenses granted under the third
|
||||||
|
paragraph of section 11).
|
||||||
|
|
||||||
|
However, if you cease all violation of this License, then your
|
||||||
|
license from a particular copyright holder is reinstated (a)
|
||||||
|
provisionally, unless and until the copyright holder explicitly and
|
||||||
|
finally terminates your license, and (b) permanently, if the copyright
|
||||||
|
holder fails to notify you of the violation by some reasonable means
|
||||||
|
prior to 60 days after the cessation.
|
||||||
|
|
||||||
|
Moreover, your license from a particular copyright holder is
|
||||||
|
reinstated permanently if the copyright holder notifies you of the
|
||||||
|
violation by some reasonable means, this is the first time you have
|
||||||
|
received notice of violation of this License (for any work) from that
|
||||||
|
copyright holder, and you cure the violation prior to 30 days after
|
||||||
|
your receipt of the notice.
|
||||||
|
|
||||||
|
Termination of your rights under this section does not terminate the
|
||||||
|
licenses of parties who have received copies or rights from you under
|
||||||
|
this License. If your rights have been terminated and not permanently
|
||||||
|
reinstated, you do not qualify to receive new licenses for the same
|
||||||
|
material under section 10.
|
||||||
|
|
||||||
|
9. Acceptance Not Required for Having Copies.
|
||||||
|
|
||||||
|
You are not required to accept this License in order to receive or
|
||||||
|
run a copy of the Program. Ancillary propagation of a covered work
|
||||||
|
occurring solely as a consequence of using peer-to-peer transmission
|
||||||
|
to receive a copy likewise does not require acceptance. However,
|
||||||
|
nothing other than this License grants you permission to propagate or
|
||||||
|
modify any covered work. These actions infringe copyright if you do
|
||||||
|
not accept this License. Therefore, by modifying or propagating a
|
||||||
|
covered work, you indicate your acceptance of this License to do so.
|
||||||
|
|
||||||
|
10. Automatic Licensing of Downstream Recipients.
|
||||||
|
|
||||||
|
Each time you convey a covered work, the recipient automatically
|
||||||
|
receives a license from the original licensors, to run, modify and
|
||||||
|
propagate that work, subject to this License. You are not responsible
|
||||||
|
for enforcing compliance by third parties with this License.
|
||||||
|
|
||||||
|
An "entity transaction" is a transaction transferring control of an
|
||||||
|
organization, or substantially all assets of one, or subdividing an
|
||||||
|
organization, or merging organizations. If propagation of a covered
|
||||||
|
work results from an entity transaction, each party to that
|
||||||
|
transaction who receives a copy of the work also receives whatever
|
||||||
|
licenses to the work the party's predecessor in interest had or could
|
||||||
|
give under the previous paragraph, plus a right to possession of the
|
||||||
|
Corresponding Source of the work from the predecessor in interest, if
|
||||||
|
the predecessor has it or can get it with reasonable efforts.
|
||||||
|
|
||||||
|
You may not impose any further restrictions on the exercise of the
|
||||||
|
rights granted or affirmed under this License. For example, you may
|
||||||
|
not impose a license fee, royalty, or other charge for exercise of
|
||||||
|
rights granted under this License, and you may not initiate litigation
|
||||||
|
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||||
|
any patent claim is infringed by making, using, selling, offering for
|
||||||
|
sale, or importing the Program or any portion of it.
|
||||||
|
|
||||||
|
11. Patents.
|
||||||
|
|
||||||
|
A "contributor" is a copyright holder who authorizes use under this
|
||||||
|
License of the Program or a work on which the Program is based. The
|
||||||
|
work thus licensed is called the contributor's "contributor version".
|
||||||
|
|
||||||
|
A contributor's "essential patent claims" are all patent claims
|
||||||
|
owned or controlled by the contributor, whether already acquired or
|
||||||
|
hereafter acquired, that would be infringed by some manner, permitted
|
||||||
|
by this License, of making, using, or selling its contributor version,
|
||||||
|
but do not include claims that would be infringed only as a
|
||||||
|
consequence of further modification of the contributor version. For
|
||||||
|
purposes of this definition, "control" includes the right to grant
|
||||||
|
patent sublicenses in a manner consistent with the requirements of
|
||||||
|
this License.
|
||||||
|
|
||||||
|
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||||
|
patent license under the contributor's essential patent claims, to
|
||||||
|
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||||
|
propagate the contents of its contributor version.
|
||||||
|
|
||||||
|
In the following three paragraphs, a "patent license" is any express
|
||||||
|
agreement or commitment, however denominated, not to enforce a patent
|
||||||
|
(such as an express permission to practice a patent or covenant not to
|
||||||
|
sue for patent infringement). To "grant" such a patent license to a
|
||||||
|
party means to make such an agreement or commitment not to enforce a
|
||||||
|
patent against the party.
|
||||||
|
|
||||||
|
If you convey a covered work, knowingly relying on a patent license,
|
||||||
|
and the Corresponding Source of the work is not available for anyone
|
||||||
|
to copy, free of charge and under the terms of this License, through a
|
||||||
|
publicly available network server or other readily accessible means,
|
||||||
|
then you must either (1) cause the Corresponding Source to be so
|
||||||
|
available, or (2) arrange to deprive yourself of the benefit of the
|
||||||
|
patent license for this particular work, or (3) arrange, in a manner
|
||||||
|
consistent with the requirements of this License, to extend the patent
|
||||||
|
license to downstream recipients. "Knowingly relying" means you have
|
||||||
|
actual knowledge that, but for the patent license, your conveying the
|
||||||
|
covered work in a country, or your recipient's use of the covered work
|
||||||
|
in a country, would infringe one or more identifiable patents in that
|
||||||
|
country that you have reason to believe are valid.
|
||||||
|
|
||||||
|
If, pursuant to or in connection with a single transaction or
|
||||||
|
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||||
|
covered work, and grant a patent license to some of the parties
|
||||||
|
receiving the covered work authorizing them to use, propagate, modify
|
||||||
|
or convey a specific copy of the covered work, then the patent license
|
||||||
|
you grant is automatically extended to all recipients of the covered
|
||||||
|
work and works based on it.
|
||||||
|
|
||||||
|
A patent license is "discriminatory" if it does not include within
|
||||||
|
the scope of its coverage, prohibits the exercise of, or is
|
||||||
|
conditioned on the non-exercise of one or more of the rights that are
|
||||||
|
specifically granted under this License. You may not convey a covered
|
||||||
|
work if you are a party to an arrangement with a third party that is
|
||||||
|
in the business of distributing software, under which you make payment
|
||||||
|
to the third party based on the extent of your activity of conveying
|
||||||
|
the work, and under which the third party grants, to any of the
|
||||||
|
parties who would receive the covered work from you, a discriminatory
|
||||||
|
patent license (a) in connection with copies of the covered work
|
||||||
|
conveyed by you (or copies made from those copies), or (b) primarily
|
||||||
|
for and in connection with specific products or compilations that
|
||||||
|
contain the covered work, unless you entered into that arrangement,
|
||||||
|
or that patent license was granted, prior to 28 March 2007.
|
||||||
|
|
||||||
|
Nothing in this License shall be construed as excluding or limiting
|
||||||
|
any implied license or other defenses to infringement that may
|
||||||
|
otherwise be available to you under applicable patent law.
|
||||||
|
|
||||||
|
12. No Surrender of Others' Freedom.
|
||||||
|
|
||||||
|
If conditions are imposed on you (whether by court order, agreement or
|
||||||
|
otherwise) that contradict the conditions of this License, they do not
|
||||||
|
excuse you from the conditions of this License. If you cannot convey a
|
||||||
|
covered work so as to satisfy simultaneously your obligations under this
|
||||||
|
License and any other pertinent obligations, then as a consequence you may
|
||||||
|
not convey it at all. For example, if you agree to terms that obligate you
|
||||||
|
to collect a royalty for further conveying from those to whom you convey
|
||||||
|
the Program, the only way you could satisfy both those terms and this
|
||||||
|
License would be to refrain entirely from conveying the Program.
|
||||||
|
|
||||||
|
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, if you modify the
|
||||||
|
Program, your modified version must prominently offer all users
|
||||||
|
interacting with it remotely through a computer network (if your version
|
||||||
|
supports such interaction) an opportunity to receive the Corresponding
|
||||||
|
Source of your version by providing access to the Corresponding Source
|
||||||
|
from a network server at no charge, through some standard or customary
|
||||||
|
means of facilitating copying of software. This Corresponding Source
|
||||||
|
shall include the Corresponding Source for any work covered by version 3
|
||||||
|
of the GNU General Public License that is incorporated pursuant to the
|
||||||
|
following paragraph.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, you have
|
||||||
|
permission to link or combine any covered work with a work licensed
|
||||||
|
under version 3 of the GNU General Public License into a single
|
||||||
|
combined work, and to convey the resulting work. The terms of this
|
||||||
|
License will continue to apply to the part which is the covered work,
|
||||||
|
but the work with which it is combined will remain governed by version
|
||||||
|
3 of the GNU General Public License.
|
||||||
|
|
||||||
|
14. Revised Versions of this License.
|
||||||
|
|
||||||
|
The Free Software Foundation may publish revised and/or new versions of
|
||||||
|
the GNU Affero General Public License from time to time. Such new versions
|
||||||
|
will be similar in spirit to the present version, but may differ in detail to
|
||||||
|
address new problems or concerns.
|
||||||
|
|
||||||
|
Each version is given a distinguishing version number. If the
|
||||||
|
Program specifies that a certain numbered version of the GNU Affero General
|
||||||
|
Public License "or any later version" applies to it, you have the
|
||||||
|
option of following the terms and conditions either of that numbered
|
||||||
|
version or of any later version published by the Free Software
|
||||||
|
Foundation. If the Program does not specify a version number of the
|
||||||
|
GNU Affero General Public License, you may choose any version ever published
|
||||||
|
by the Free Software Foundation.
|
||||||
|
|
||||||
|
If the Program specifies that a proxy can decide which future
|
||||||
|
versions of the GNU Affero General Public License can be used, that proxy's
|
||||||
|
public statement of acceptance of a version permanently authorizes you
|
||||||
|
to choose that version for the Program.
|
||||||
|
|
||||||
|
Later license versions may give you additional or different
|
||||||
|
permissions. However, no additional obligations are imposed on any
|
||||||
|
author or copyright holder as a result of your choosing to follow a
|
||||||
|
later version.
|
||||||
|
|
||||||
|
15. Disclaimer of Warranty.
|
||||||
|
|
||||||
|
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||||
|
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||||
|
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||||
|
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||||
|
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||||
|
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||||
|
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||||
|
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||||
|
|
||||||
|
16. Limitation of Liability.
|
||||||
|
|
||||||
|
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||||
|
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||||
|
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||||
|
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||||
|
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||||
|
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||||
|
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||||
|
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||||
|
SUCH DAMAGES.
|
||||||
|
|
||||||
|
17. Interpretation of Sections 15 and 16.
|
||||||
|
|
||||||
|
If the disclaimer of warranty and limitation of liability provided
|
||||||
|
above cannot be given local legal effect according to their terms,
|
||||||
|
reviewing courts shall apply local law that most closely approximates
|
||||||
|
an absolute waiver of all civil liability in connection with the
|
||||||
|
Program, unless a warranty or assumption of liability accompanies a
|
||||||
|
copy of the Program in return for a fee.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
How to Apply These Terms to Your New Programs
|
||||||
|
|
||||||
|
If you develop a new program, and you want it to be of the greatest
|
||||||
|
possible use to the public, the best way to achieve this is to make it
|
||||||
|
free software which everyone can redistribute and change under these terms.
|
||||||
|
|
||||||
|
To do so, attach the following notices to the program. It is safest
|
||||||
|
to attach them to the start of each source file to most effectively
|
||||||
|
state the exclusion of warranty; and each file should have at least
|
||||||
|
the "copyright" line and a pointer to where the full notice is found.
|
||||||
|
|
||||||
|
<one line to give the program's name and a brief idea of what it does.>
|
||||||
|
Copyright (C) <year> <name of author>
|
||||||
|
|
||||||
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU Affero General Public License as published by
|
||||||
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU Affero General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU Affero General Public License
|
||||||
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
Also add information on how to contact you by electronic and paper mail.
|
||||||
|
|
||||||
|
If your software can interact with users remotely through a computer
|
||||||
|
network, you should also make sure that it provides a way for users to
|
||||||
|
get its source. For example, if your program is a web application, its
|
||||||
|
interface could display a "Source" link that leads users to an archive
|
||||||
|
of the code. There are many ways you could offer source, and different
|
||||||
|
solutions will be better for different programs; see section 13 for the
|
||||||
|
specific requirements.
|
||||||
|
|
||||||
|
You should also get your employer (if you work as a programmer) or school,
|
||||||
|
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||||
|
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||||
|
<https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
------------------------------------------------------------------------
|
||||||
|
APPENDIX: HOW TO APPLY THESE TERMS TO YOUR NEW PROGRAMS
|
||||||
|
|
||||||
|
To ensure your repository complies with the remote network interaction
|
||||||
|
requirements (Section 13 of the AGPLv3), you must ensure users
|
||||||
|
interacting with your stack can access the source code.
|
||||||
|
|
||||||
|
This project, fester, complies by maintaining its primary development
|
||||||
|
repository at https://git.dcos.net. Any derivative works must retain
|
||||||
|
this notice and provide equivalent access to source code.
|
||||||
|
|
@ -0,0 +1,187 @@
|
||||||
|
🟩 2. CHEATSHEET.md (Operator Survival Guide)
|
||||||
|
# 🧠 Fester Operator Cheatsheet
|
||||||
|
|
||||||
|
Distributed Build + Scheduler + Observability System
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🚀 STARTING A BUILD
|
||||||
|
|
||||||
|
```bash
|
||||||
|
fester build ./project.yaml
|
||||||
|
🧩 VIEW LIVE SYSTEM
|
||||||
|
Cockpit Module: Fester
|
||||||
|
Live DAG: /ui/live_dag.html
|
||||||
|
Replay View: /ui/replay.html
|
||||||
|
📡 KEY CONCEPTS
|
||||||
|
Nodes
|
||||||
|
|
||||||
|
Machines participating in builds (physical, VM, or container)
|
||||||
|
|
||||||
|
Targets
|
||||||
|
|
||||||
|
Compilation environments:
|
||||||
|
|
||||||
|
x86_64-linux-gnu
|
||||||
|
aarch64-linux-gnu
|
||||||
|
riscv64
|
||||||
|
embedded/toolchain targets
|
||||||
|
Actions
|
||||||
|
|
||||||
|
Graph nodes representing build steps
|
||||||
|
|
||||||
|
Scheduler
|
||||||
|
|
||||||
|
Chooses best node based on:
|
||||||
|
|
||||||
|
load
|
||||||
|
temperature
|
||||||
|
policy rules
|
||||||
|
historical performance
|
||||||
|
🧠 DEBUGGING
|
||||||
|
Failure Autopsy
|
||||||
|
GET /api/autopsy/<build_id>
|
||||||
|
Timeline Replay
|
||||||
|
GET /api/replay/<session_id>
|
||||||
|
📦 CACHE SYSTEM
|
||||||
|
|
||||||
|
Supports:
|
||||||
|
|
||||||
|
ccache
|
||||||
|
MinIO distributed cache
|
||||||
|
Btrfs snapshot cache
|
||||||
|
tmpfs acceleration layer
|
||||||
|
🧊 SNAPSHOTS
|
||||||
|
|
||||||
|
Freeze state:
|
||||||
|
|
||||||
|
qcow2 image snapshots
|
||||||
|
Btrfs copy-on-write states
|
||||||
|
⚙️ NODE CONTROL
|
||||||
|
fester node list
|
||||||
|
fester node set-policy <node> preferred
|
||||||
|
fester node set-policy <node> avoid
|
||||||
|
🔥 SCHEDULER MODES
|
||||||
|
unified (default)
|
||||||
|
weighted thermal-aware
|
||||||
|
cache-first
|
||||||
|
target-isolated
|
||||||
|
experimental intelligence mode
|
||||||
|
⚠️ SAFETY NOTES
|
||||||
|
Do NOT run unrestricted builds on production nodes
|
||||||
|
Monitor thermal load in Grafana
|
||||||
|
Ensure cache integrity for distributed builds
|
||||||
|
Validate cross-compile toolchains before enabling targets
|
||||||
|
🧠 DESIGN PRINCIPLE
|
||||||
|
|
||||||
|
"Every build is reproducible, explainable, and replayable."
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🟩 3. `PRODUCTION_HARDENING.md`
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
# 🛡️ Fester Production Hardening Guide
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🌡️ 1. THERMAL SAFETY
|
||||||
|
|
||||||
|
## Required
|
||||||
|
- Enable node temperature monitoring
|
||||||
|
- Set max thermal threshold per node
|
||||||
|
|
||||||
|
## Recommended
|
||||||
|
- throttle scheduler above 85°C
|
||||||
|
- disable high-parallel builds under sustained load
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🧠 2. SCHEDULER SAFETY
|
||||||
|
|
||||||
|
Avoid:
|
||||||
|
- unrestricted global scheduling in mixed architectures
|
||||||
|
- running full cluster on single target policy
|
||||||
|
|
||||||
|
Enable:
|
||||||
|
- weighted scheduler
|
||||||
|
- per-node policy constraints
|
||||||
|
- cache-aware routing
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 📦 3. CACHE INTEGRITY
|
||||||
|
|
||||||
|
Use:
|
||||||
|
- checksum validation for artifacts
|
||||||
|
- MinIO redundancy if cluster > 3 nodes
|
||||||
|
- avoid mixing tmpfs + persistent cache without sync barriers
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🧊 4. SNAPSHOT SAFETY
|
||||||
|
|
||||||
|
If using:
|
||||||
|
- qcow2 snapshots
|
||||||
|
- Btrfs CoW layers
|
||||||
|
|
||||||
|
Ensure:
|
||||||
|
- periodic snapshot compaction
|
||||||
|
- rollback testing before production builds
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🧩 5. CROSS-COMPILATION RISKS
|
||||||
|
|
||||||
|
- toolchain mismatch = silent binary corruption risk
|
||||||
|
- always validate ELF output per target arch
|
||||||
|
- isolate toolchains per execution environment
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🌐 6. DISTRIBUTED EXECUTION
|
||||||
|
|
||||||
|
- ensure clock sync (NTP mandatory)
|
||||||
|
- avoid partial node visibility during scheduling
|
||||||
|
- handle node dropouts as first-class events
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 📊 7. OBSERVABILITY STACK
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
|
||||||
|
- Prometheus → metrics ingestion
|
||||||
|
- Grafana → visualization
|
||||||
|
- Cockpit → control plane
|
||||||
|
- Fester UI → execution DAG + causal graph
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🧠 8. GOLDEN RULE
|
||||||
|
|
||||||
|
> Never trust a build you cannot replay.
|
||||||
|
🧭 WHAT YOU NOW HAVE (SYSTEM MATURITY STATE)
|
||||||
|
|
||||||
|
You’ve effectively reached:
|
||||||
|
|
||||||
|
🧠 Level 1 — Distributed Compiler System
|
||||||
|
|
||||||
|
(distcc-like foundation)
|
||||||
|
|
||||||
|
🧠 Level 2 — Smart Scheduler
|
||||||
|
|
||||||
|
(load/thermal/cache aware)
|
||||||
|
|
||||||
|
🧠 Level 3 — Observability System
|
||||||
|
|
||||||
|
(metrics + Grafana + cockpit)
|
||||||
|
|
||||||
|
🧠 Level 4 — Causal Execution Graph
|
||||||
|
|
||||||
|
(why decisions happen)
|
||||||
|
|
||||||
|
🧠 Level 5 — Replayable Build Brain
|
||||||
|
|
||||||
|
(session + timeline + autopsy)
|
||||||
|
|
@ -0,0 +1,322 @@
|
||||||
|
# 🧠 Fester
|
||||||
|
|
||||||
|
**A distributed, DAG-driven build execution system with real-time scheduling, thermal/load awareness, cache-aware execution, and deterministic replay/debugging.**
|
||||||
|
|
||||||
|
[](LICENSE)
|
||||||
|
[](https://www.python.org/)
|
||||||
|
[](https://fastapi.tiangolo.com/)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📸 Screenshots
|
||||||
|
|
||||||
|
| Dashboard | Live DAG | Replay |
|
||||||
|
|-----------|----------|--------|
|
||||||
|
|  |  |  |
|
||||||
|
|
||||||
|
| Sessions | Metrics | Cause Graph |
|
||||||
|
|----------|---------|-------------|
|
||||||
|
|  |  |  |
|
||||||
|
|
||||||
|
| Timeline | Debugger |
|
||||||
|
|----------|----------|
|
||||||
|
|  |  |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🎯 What Fester Does
|
||||||
|
|
||||||
|
Fester turns a cluster of machines into a single, observable build brain:
|
||||||
|
|
||||||
|
- **DAG-driven execution** — every build is compiled into a directed acyclic graph of actions, with dependencies honored and parallel branches run concurrently
|
||||||
|
- **Smart scheduler** — picks the best node for each action based on CPU load, temperature, policy constraints, and historical instability
|
||||||
|
- **Real-time observability** — every event (node state, schedule decision, task lifecycle, cache hit/miss, failure) streams live to the UI via WebSocket
|
||||||
|
- **Deterministic replay** — every build session is journaled to SQLite; scrub back through the timeline to see exactly what happened, when, and why
|
||||||
|
- **Failure autopsy** — when an action fails, trace its dependency chain backward, see the last scheduler decision, and compute the forward blast radius
|
||||||
|
- **Cause graph** — a post-hoc reasoning layer that builds a causal graph from system events, so you can ask "why did this node make this decision?"
|
||||||
|
- **Cache layer** — MinIO distributed cache + optional Btrfs CoW reflinks + QCOW2 workspace snapshots
|
||||||
|
|
||||||
|
## 🏗️ Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Fester Cluster │
|
||||||
|
├──────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │
|
||||||
|
│ │ Backend │◄──►│ EventBus │◄──►│ PipelineEngine │ │
|
||||||
|
│ │ (FastAPI) │ │ (singleton)│ │ (DAG executor) │ │
|
||||||
|
│ └──────┬──────┘ └──────┬──────┘ └──────────┬──────────┘ │
|
||||||
|
│ │ │ │ │
|
||||||
|
│ │ │ ┌───────────────────┼──────────┐ │
|
||||||
|
│ │ │ │ │ │ │
|
||||||
|
│ │ ▼ ▼ ▼ │ │
|
||||||
|
│ ┌──────┴──────┐ ┌─────────────┐ ┌─────────────────┐ │ │
|
||||||
|
│ │ WebSocket │ │ Timeline │ │ Scheduler │ │ │
|
||||||
|
│ │ Stream │ │ Store │ │ (weighted/thermal│ │ │
|
||||||
|
│ │ (/ws, etc) │ │ (SQLite) │ │ /cache-aware) │ │ │
|
||||||
|
│ └──────┬──────┘ └──────┬──────┘ └────────┬────────┘ │ │
|
||||||
|
│ │ │ │ │ │
|
||||||
|
│ │ │ ┌───────┴────────┐ │ │
|
||||||
|
│ │ │ │ Node Registry │ │ │
|
||||||
|
│ │ │ │ (probe + drift)│ │ │
|
||||||
|
│ ▼ ▼ └────────────────┘ │ │
|
||||||
|
│ ┌──────────────────────────────────────────────────────┐ │ │
|
||||||
|
│ │ UI (8 pages) │ │ │
|
||||||
|
│ │ Dashboard │ Live DAG │ Replay │ Sessions │ Metrics │ │ │
|
||||||
|
│ │ Cause Graph │ Timeline │ Debugger │ │ │
|
||||||
|
│ └──────────────────────────────────────────────────────┘ │ │
|
||||||
|
│ │ │
|
||||||
|
│ ┌────────────┐ ┌────────────┐ ┌────────────┐ ┌────────┐│ │
|
||||||
|
│ │ MinIO Cache│ │ Btrfs CAS │ │ QCOW2 Snap │ │ tmux ││ │
|
||||||
|
│ │ (optional) │ │ (optional)│ │ (optional) │ │ runtime││ │
|
||||||
|
│ └────────────┘ └────────────┘ └────────────┘ └────────┘│ │
|
||||||
|
│ │ │
|
||||||
|
└──────────────────────────────────────────────────────────────┘ │
|
||||||
|
│
|
||||||
|
▲ │
|
||||||
|
│ HTTP :8787/status (probe) │
|
||||||
|
│ │
|
||||||
|
┌───────────────┴────────────────────────────────┐ │
|
||||||
|
│ Cluster Nodes │ │
|
||||||
|
│ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐│ │
|
||||||
|
│ │ x99-v3 │ │ x99-v4 │ │ rpi-1 │ │ ryzen-1││ │
|
||||||
|
│ │(x86_64)│ │(x86_64)│ │(arm64) │ │(x86_64)││ │
|
||||||
|
│ └────────┘ └────────┘ └────────┘ └────────┘│ │
|
||||||
|
└────────────────────────────────────────────────┘ │
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📁 Project Layout
|
||||||
|
|
||||||
|
```
|
||||||
|
fester/
|
||||||
|
├── backend/ # Python backend (FastAPI)
|
||||||
|
│ ├── main.py # App entrypoint + 71 routes
|
||||||
|
│ ├── api/ # REST routers + WebSocket hub
|
||||||
|
│ │ ├── build.py # /api/build
|
||||||
|
│ │ ├── nodes.py # /api/nodes (list, policy, probe)
|
||||||
|
│ │ ├── replay.py # /replay/start, /events
|
||||||
|
│ │ ├── autopsy.py # /autopsy/{sid}/{action}
|
||||||
|
│ │ ├── timeline.py # /timeline/{node}, /rewind/{i}
|
||||||
|
│ │ ├── debugger.py # /debugger/{pause,resume,step}
|
||||||
|
│ │ ├── pipeline_control.py # /api/pipeline/{state,retry,...}
|
||||||
|
│ │ ├── metrics.py # /metrics (Prometheus) + /api/metrics/json
|
||||||
|
│ │ └── cause.py # /api/cause/{explain,trace,events}
|
||||||
|
│ ├── pipeline/ # Build execution
|
||||||
|
│ │ ├── engine.py # DAG executor with pause/step/critical-path
|
||||||
|
│ │ └── runner.py # BuildRunner (lifecycle + persistence)
|
||||||
|
│ ├── scheduler/ # Node selection
|
||||||
|
│ │ └── optimizer.py # choose_best_node (weighted scoring)
|
||||||
|
│ ├── analysis/ # Reasoning layer
|
||||||
|
│ │ ├── cause_graph.py # Causal graph from events
|
||||||
|
│ │ ├── timeline_store.py # SQLite-backed event journal
|
||||||
|
│ │ ├── failure_autopsy.py # Backward dependency tracing
|
||||||
|
│ │ └── failure_propagation.py # Forward blast radius
|
||||||
|
│ ├── events/ # Event infrastructure
|
||||||
|
│ │ ├── bus.py # Singleton EventBus
|
||||||
|
│ │ └── schema.py # FesterEvent dataclass + EventType enum
|
||||||
|
│ ├── nodes/ # Node management
|
||||||
|
│ │ ├── probe.py # HTTP agent probing (async + sync)
|
||||||
|
│ │ ├── roles.py # NodeRole dataclass
|
||||||
|
│ │ └── state_model.py # NodeStateRegistry
|
||||||
|
│ ├── storage/ # Storage layer
|
||||||
|
│ │ ├── btrfs_cas.py # CoW reflink snapshots
|
||||||
|
│ │ ├── qcow2_freeze.py # QCOW2 workspace freezing
|
||||||
|
│ │ ├── tmpfs.py # tmpfs workspace acceleration
|
||||||
|
│ │ └── sqlite_db.py # Persistent storage (builds, sessions, events)
|
||||||
|
│ ├── integrations/ # External tool integrations
|
||||||
|
│ │ ├── tmux.py # Detached tmux action runtime
|
||||||
|
│ │ ├── mosh.py # mosh/ssh shell command builder
|
||||||
|
│ │ ├── forgejo.py # Forgejo push webhook handler
|
||||||
|
│ │ ├── lxc.py # LXC container execution
|
||||||
|
│ │ └── libvirt.py # libvirt VM execution
|
||||||
|
│ ├── executor/ # Runtime router (host/lxc/libvirt/tmux)
|
||||||
|
│ ├── cache/ # MinIO distributed cache
|
||||||
|
│ ├── graph/ # DAG construction + critical path
|
||||||
|
│ ├── metrics/ # Prometheus exporter + observability hub
|
||||||
|
│ ├── policy/ # Policy engine (rules + overrides)
|
||||||
|
│ └── targets/ # Build target catalog (Gentoo, Buildroot, ...)
|
||||||
|
├── ui/ # Frontend (vanilla JS, no build step)
|
||||||
|
│ ├── app.js # Shared shell (WS, toasts, topbar, health)
|
||||||
|
│ ├── live_dag.html # Real-time DAG with layered layout
|
||||||
|
│ ├── replay.html # Timeline scrubber + autopsy
|
||||||
|
│ ├── sessions.html # Build history + tmux output viewer
|
||||||
|
│ ├── metrics.html # Live charts + per-node cards
|
||||||
|
│ ├── cause.html # Cause graph + blast radius
|
||||||
|
│ ├── timeline.html # Per-node event drill-down
|
||||||
|
│ └── debugger.html # Step-through execution control
|
||||||
|
├── cli/ # CLI tool (35+ subcommands)
|
||||||
|
│ └── fester.py
|
||||||
|
├── cockpit/ # Cockpit module (optional)
|
||||||
|
├── docs/ # Documentation + screenshots
|
||||||
|
├── scripts/ # Helper scripts (mock agent, syntax checker)
|
||||||
|
├── pyproject.toml # Python packaging
|
||||||
|
├── bootstrap.sh # One-shot dependency installer
|
||||||
|
├── run.sh # Start the backend
|
||||||
|
└── config.yaml # Cluster configuration
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🚀 Quick Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Clone
|
||||||
|
git clone https://git.dcos.net/dcosnet/fester.git
|
||||||
|
cd fester
|
||||||
|
|
||||||
|
# 2. Bootstrap (creates venv, installs deps, sets up config)
|
||||||
|
./bootstrap.sh
|
||||||
|
|
||||||
|
# 3. Run
|
||||||
|
./run.sh
|
||||||
|
|
||||||
|
# 4. Open the UI
|
||||||
|
open http://localhost:8080
|
||||||
|
```
|
||||||
|
|
||||||
|
See **[quickstart.md](quickstart.md)** for the full 5-minute walkthrough.
|
||||||
|
|
||||||
|
## 🛠️ Prerequisites
|
||||||
|
|
||||||
|
- **Python 3.12+**
|
||||||
|
- **pip** (for dependency installation)
|
||||||
|
- **Optional** (features activate automatically when installed):
|
||||||
|
- `tmux` — for the tmux action runtime (live output viewing)
|
||||||
|
- `mosh` + `ssh` — for the "Shell into node" UI button
|
||||||
|
- `qemu-utils` — for QCOW2 workspace snapshots
|
||||||
|
- `rsync` — for workspace freeze operations
|
||||||
|
- `btrfs-progs` — for CoW reflink snapshots (requires btrfs filesystem)
|
||||||
|
- `minio` server — for distributed cache (otherwise falls back to local)
|
||||||
|
|
||||||
|
## 📊 Tech Stack
|
||||||
|
|
||||||
|
| Layer | Technology |
|
||||||
|
|-------|------------|
|
||||||
|
| Backend | Python 3.12, FastAPI 0.128, uvicorn, pydantic 2 |
|
||||||
|
| Realtime | WebSockets (3 channels: `/ws`, `/ws-targets`, `/ws-debugger`) |
|
||||||
|
| Storage | SQLite (WAL mode) for builds/sessions/events; MinIO for distributed cache |
|
||||||
|
| Frontend | Vanilla HTML/CSS/JS (no build step, no framework, ~12KB total) |
|
||||||
|
| Observability | Prometheus exposition + Grafana dashboard configs |
|
||||||
|
| CLI | Python argparse (35+ subcommands) |
|
||||||
|
| Integrations | tmux, mosh, LXC, libvirt, Forgejo, distcc, ccache |
|
||||||
|
|
||||||
|
## 🎛️ Configuration
|
||||||
|
|
||||||
|
Fester reads from `config.yaml` (or `FESTER_CONFIG` env var) at startup:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
master:
|
||||||
|
name: fester-master
|
||||||
|
role: control
|
||||||
|
|
||||||
|
nodes:
|
||||||
|
- name: x99-v3
|
||||||
|
host: 192.168.1.10
|
||||||
|
max_jobs: 24
|
||||||
|
- name: x99-v4
|
||||||
|
host: 192.168.1.11
|
||||||
|
max_jobs: 30
|
||||||
|
|
||||||
|
projects:
|
||||||
|
- name: linux-tool
|
||||||
|
repo: https://forgejo.local/linux-tool.git
|
||||||
|
targets:
|
||||||
|
debian: "make clean && make debian"
|
||||||
|
arch: "make clean && make arch"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
| Variable | Default | Purpose |
|
||||||
|
|----------|---------|---------|
|
||||||
|
| `FESTER_CONFIG` | `config.yaml` | Path to YAML config |
|
||||||
|
| `FESTER_DB_PATH` | `/var/lib/fester/fester.db` (with fallbacks) | SQLite database location |
|
||||||
|
| `FESTER_NO_DRIFT` | unset | Set to `1` to disable synthetic drift when agents unreachable |
|
||||||
|
| `FESTER_AUTOBUILD` | unset | Set to `1` to auto-trigger a build every 60s |
|
||||||
|
| `FESTER_ROLE_DB` | `/etc/fester/node_roles.json` | Node role overrides |
|
||||||
|
| `FESTER_CACHE_DIR` | `/var/lib/fester/cache` | Local cache directory |
|
||||||
|
| `FESTER_STORAGE_CONFIG` | `/etc/fester/storage.json` | Storage layer config |
|
||||||
|
| `FESTER_API` | `http://localhost:8080` | CLI: backend URL |
|
||||||
|
| `FESTER_WS` | `ws://localhost:8080/ws` | CLI: WebSocket URL |
|
||||||
|
|
||||||
|
## 🔌 API Surface
|
||||||
|
|
||||||
|
71 routes total. Key endpoints:
|
||||||
|
|
||||||
|
| Method | Path | Purpose |
|
||||||
|
|--------|------|---------|
|
||||||
|
| `POST` | `/api/build` | Kick off a build |
|
||||||
|
| `GET` | `/api/builds` | Build history |
|
||||||
|
| `POST` | `/api/builds/{id}/cancel` | Cancel a running build |
|
||||||
|
| `GET` | `/api/nodes` | List cluster nodes (with live metrics) |
|
||||||
|
| `POST` | `/api/nodes/{name}/policy` | Set node policy (preferred/avoid/neutral) |
|
||||||
|
| `POST` | `/api/nodes/{name}/probe` | Manually probe a node's agent |
|
||||||
|
| `POST` | `/api/nodes/{name}/shell` | Build mosh/ssh shell command |
|
||||||
|
| `GET` | `/api/metrics/json` | JSON metrics snapshot |
|
||||||
|
| `GET` | `/metrics` | Prometheus exposition |
|
||||||
|
| `GET` | `/api/cause/explain/{node}` | Causal chain for a node |
|
||||||
|
| `GET` | `/api/propagation/{action}` | Blast radius if action fails |
|
||||||
|
| `POST` | `/replay/start` | Start a replay session |
|
||||||
|
| `GET` | `/replay/events` | Full event journal |
|
||||||
|
| `GET` | `/autopsy/{sid}/{action}` | Failure autopsy |
|
||||||
|
| `GET` | `/api/storage/status` | Storage capabilities |
|
||||||
|
| `POST` | `/api/storage/freeze` | Freeze workdir into qcow2 |
|
||||||
|
| `GET` | `/api/actions/active` | List active tmux sessions |
|
||||||
|
| `WS` | `/ws` | Main event stream |
|
||||||
|
| `WS` | `/ws-debugger` | Debugger control channel |
|
||||||
|
| `WS` | `/ws-targets` | Target toggle notifications |
|
||||||
|
|
||||||
|
Full route list: `curl http://localhost:8080/openapi.json | jq '.paths | keys[]'`
|
||||||
|
|
||||||
|
## 🖥️ CLI
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build + watch
|
||||||
|
fester build --cmd "make -j$(nproc)" --dir /home/user/linux --watch
|
||||||
|
|
||||||
|
# List builds
|
||||||
|
fester builds
|
||||||
|
|
||||||
|
# Node management
|
||||||
|
fester node list
|
||||||
|
fester node set-policy x99-v3 preferred
|
||||||
|
fester node probe x99-v3
|
||||||
|
|
||||||
|
# Cause analysis
|
||||||
|
fester cause explain x99-v3 build_kernel
|
||||||
|
fester blast build_kernel
|
||||||
|
|
||||||
|
# Live event stream
|
||||||
|
fester stream
|
||||||
|
|
||||||
|
# Health check
|
||||||
|
fester health
|
||||||
|
```
|
||||||
|
|
||||||
|
Run `fester --help` for the full list of 35+ subcommands.
|
||||||
|
|
||||||
|
## 📚 Documentation
|
||||||
|
|
||||||
|
- **[quickstart.md](quickstart.md)** — 5-minute getting started guide
|
||||||
|
- **[CHEATSHEET.md](CHEATSHEET.md)** — Operator survival guide
|
||||||
|
- **[CONTRIBUTING.md](CONTRIBUTING.md)** — How to contribute
|
||||||
|
- **[CHANGELOG.md](CHANGELOG.md)** — Version history
|
||||||
|
- **[docs/screenshots/](docs/screenshots/)** — UI screenshots
|
||||||
|
|
||||||
|
## 🐳 Docker
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t fester .
|
||||||
|
docker run -p 8080:8080 -v fester-data:/var/lib/fester fester
|
||||||
|
```
|
||||||
|
|
||||||
|
See **[Dockerfile](Dockerfile)** for details.
|
||||||
|
|
||||||
|
## 🔐 Security
|
||||||
|
|
||||||
|
Fester is designed to run behind your existing network controls (NAT, firewalls like OPNsense, etc.). There is **no built-in auth** — assume the network is trusted. If you need to expose it publicly, put it behind a reverse proxy with auth (nginx + OAuth2 Proxy, Traefik + Authelia, etc.).
|
||||||
|
|
||||||
|
## ⚠️ License
|
||||||
|
|
||||||
|
GNU Affero General Public License v3.0 — see **[LICENSE](LICENSE)**.
|
||||||
|
|
||||||
|
Software is provided "AS IS", without warranty of any kind. This is a high-concurrency distributed execution system — review the safety notes in the license before production use.
|
||||||
|
|
@ -0,0 +1,148 @@
|
||||||
|
{
|
||||||
|
"title": "Fester Advanced Cluster Intelligence",
|
||||||
|
"timezone": "browser",
|
||||||
|
"schemaVersion": 38,
|
||||||
|
"version": 2,
|
||||||
|
"refresh": "5s",
|
||||||
|
"panels": [
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Node CPU Load",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "fester_node_cpu",
|
||||||
|
"legendFormat": "{{node}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"min": 0,
|
||||||
|
"max": 100
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Node Temperature (Thermal Risk)",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "fester_node_temp",
|
||||||
|
"legendFormat": "{{node}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "celsius",
|
||||||
|
"thresholds": {
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 65 },
|
||||||
|
{ "color": "orange", "value": 75 },
|
||||||
|
{ "color": "red", "value": 80 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Thermal Trend (Prediction Signal)",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "deriv(fester_node_temp[2m])",
|
||||||
|
"legendFormat": "{{node}} temp/s"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 6 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Danger Nodes (>80C)",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "count(fester_node_temp > 80)"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 12, "y": 8, "w": 6, "h": 6 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Warming Nodes (>70C)",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "count(fester_node_temp > 70)"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 18, "y": 8, "w": 6, "h": 6 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Pipeline Throughput",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "rate(fester_pipeline_actions_total[1m])",
|
||||||
|
"legendFormat": "{{state}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 0, "y": 14, "w": 12, "h": 7 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Cache Hit Rate Per Target",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "rate(fester_cache_hits_total[5m])",
|
||||||
|
"legendFormat": "{{target}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 12, "y": 14, "w": 12, "h": 7 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Cache Efficiency Ratio",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "rate(fester_cache_hits_total[5m]) / rate(fester_pipeline_actions_total[5m])",
|
||||||
|
"legendFormat": "efficiency"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 0, "y": 21, "w": 12, "h": 6 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Scheduler Node Usage",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (node) (fester_scheduler_score)",
|
||||||
|
"legendFormat": "{{node}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 12, "y": 21, "w": 12, "h": 6 }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Scheduler Target Distribution",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (target) (fester_scheduler_score)",
|
||||||
|
"legendFormat": "{{target}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"gridPos": { "x": 0, "y": 27, "w": 24, "h": 6 }
|
||||||
|
}
|
||||||
|
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: 'fester'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['localhost:9109']
|
||||||
|
|
@ -0,0 +1,98 @@
|
||||||
|
# analysis/failure_autopsy.py
|
||||||
|
|
||||||
|
class FailureAutopsy:
|
||||||
|
|
||||||
|
def __init__(self, journal, critical_path=None):
|
||||||
|
self.journal = journal
|
||||||
|
self.critical_path = critical_path
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FIND FAILURE EVENTS
|
||||||
|
# -----------------------------
|
||||||
|
def find_failures(self):
|
||||||
|
|
||||||
|
return [
|
||||||
|
e for e in self.journal.events
|
||||||
|
if e["type"] == "execution_result"
|
||||||
|
and e["data"].get("state") == "failed"
|
||||||
|
]
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# TRACE BACKWARD DEPENDENCY CHAIN
|
||||||
|
# -----------------------------
|
||||||
|
def trace_dependencies(self, action_name):
|
||||||
|
|
||||||
|
trace = []
|
||||||
|
visited = set()
|
||||||
|
|
||||||
|
def walk(name):
|
||||||
|
if name in visited:
|
||||||
|
return
|
||||||
|
visited.add(name)
|
||||||
|
|
||||||
|
events = self.journal.trace_action(name)
|
||||||
|
|
||||||
|
trace.append({
|
||||||
|
"action": name,
|
||||||
|
"events": events
|
||||||
|
})
|
||||||
|
|
||||||
|
for e in events:
|
||||||
|
deps = e["data"].get("deps", [])
|
||||||
|
for d in deps:
|
||||||
|
walk(d)
|
||||||
|
|
||||||
|
walk(action_name)
|
||||||
|
|
||||||
|
return trace
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET LAST SCHEDULER DECISION
|
||||||
|
# -----------------------------
|
||||||
|
def last_decision(self, action_name):
|
||||||
|
|
||||||
|
events = self.journal.trace_action(action_name)
|
||||||
|
|
||||||
|
for e in reversed(events):
|
||||||
|
if e["type"] == "schedule_decision":
|
||||||
|
return e["data"]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FULL AUTOPSY REPORT
|
||||||
|
# -----------------------------
|
||||||
|
def report(self, action_name):
|
||||||
|
|
||||||
|
failures = self.find_failures()
|
||||||
|
|
||||||
|
target_failure = None
|
||||||
|
|
||||||
|
for f in failures:
|
||||||
|
if f["data"].get("action") == action_name:
|
||||||
|
target_failure = f
|
||||||
|
break
|
||||||
|
|
||||||
|
if not target_failure:
|
||||||
|
return {
|
||||||
|
"status": "no_failure_found",
|
||||||
|
"action": action_name
|
||||||
|
}
|
||||||
|
|
||||||
|
deps_trace = self.trace_dependencies(action_name)
|
||||||
|
last_sched = self.last_decision(action_name)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": "failure_detected",
|
||||||
|
"action": action_name,
|
||||||
|
|
||||||
|
"failure_event": target_failure,
|
||||||
|
|
||||||
|
"last_scheduler_decision": last_sched,
|
||||||
|
|
||||||
|
"dependency_trace": deps_trace,
|
||||||
|
|
||||||
|
"on_critical_path": (
|
||||||
|
action_name in (self.critical_path or {}).get("score_map", {})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,93 @@
|
||||||
|
# analysis/failure_propagation.py
|
||||||
|
|
||||||
|
class FailurePropagation:
|
||||||
|
|
||||||
|
def __init__(self, graph):
|
||||||
|
"""
|
||||||
|
graph = { node: [deps...] }
|
||||||
|
"""
|
||||||
|
self.graph = graph
|
||||||
|
|
||||||
|
# reverse graph for downstream impact
|
||||||
|
self.reverse = self._build_reverse(graph)
|
||||||
|
|
||||||
|
def _build_reverse(self, graph):
|
||||||
|
reverse = {}
|
||||||
|
|
||||||
|
for node, deps in graph.items():
|
||||||
|
if node not in reverse:
|
||||||
|
reverse[node] = []
|
||||||
|
|
||||||
|
for d in deps:
|
||||||
|
reverse.setdefault(d, []).append(node)
|
||||||
|
|
||||||
|
return reverse
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# BACKWARD FAILURE TRACE
|
||||||
|
# -----------------------------
|
||||||
|
def propagate_backward(self, failed_node):
|
||||||
|
"""
|
||||||
|
Walk dependencies backward (root cause side)
|
||||||
|
"""
|
||||||
|
visited = set()
|
||||||
|
impacted = []
|
||||||
|
|
||||||
|
def walk(node):
|
||||||
|
for dep in self.graph.get(node, []):
|
||||||
|
if dep in visited:
|
||||||
|
continue
|
||||||
|
visited.add(dep)
|
||||||
|
|
||||||
|
impacted.append({
|
||||||
|
"node": dep,
|
||||||
|
"type": "root_cause_candidate"
|
||||||
|
})
|
||||||
|
|
||||||
|
walk(dep)
|
||||||
|
|
||||||
|
walk(failed_node)
|
||||||
|
|
||||||
|
return impacted
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FORWARD IMPACT TRACE
|
||||||
|
# -----------------------------
|
||||||
|
def propagate_forward(self, failed_node):
|
||||||
|
"""
|
||||||
|
Walk dependents forward (blast radius side)
|
||||||
|
"""
|
||||||
|
visited = set()
|
||||||
|
impacted = []
|
||||||
|
|
||||||
|
def walk(node):
|
||||||
|
for child in self.reverse.get(node, []):
|
||||||
|
if child in visited:
|
||||||
|
continue
|
||||||
|
visited.add(child)
|
||||||
|
|
||||||
|
impacted.append({
|
||||||
|
"node": child,
|
||||||
|
"type": "downstream_affected"
|
||||||
|
})
|
||||||
|
|
||||||
|
walk(child)
|
||||||
|
|
||||||
|
walk(failed_node)
|
||||||
|
|
||||||
|
return impacted
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FULL FAILURE FIELD MAP
|
||||||
|
# -----------------------------
|
||||||
|
def map_failure(self, failed_node):
|
||||||
|
|
||||||
|
backward = self.propagate_backward(failed_node)
|
||||||
|
forward = self.propagate_forward(failed_node)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"failed_node": failed_node,
|
||||||
|
"root_cause_candidates": backward,
|
||||||
|
"downstream_impact": forward,
|
||||||
|
"severity": len(forward) + len(backward)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
# analysis/timeline_store.py
|
||||||
|
|
||||||
|
class TimelineStore:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.events = []
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# APPEND EVENT (IMMUTABLE LOG)
|
||||||
|
# -----------------------------
|
||||||
|
def record(self, event):
|
||||||
|
"""
|
||||||
|
Every state transition in the system must go here.
|
||||||
|
"""
|
||||||
|
self.events.append(event)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET EVENTS FOR NODE
|
||||||
|
# -----------------------------
|
||||||
|
def get_node_events(self, node_name):
|
||||||
|
|
||||||
|
return [
|
||||||
|
e for e in self.events
|
||||||
|
if e.get("data", {}).get("node") == node_name
|
||||||
|
]
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET TIMELINE SNAPSHOT
|
||||||
|
# -----------------------------
|
||||||
|
def snapshot_at(self, index):
|
||||||
|
|
||||||
|
if index < 0 or index >= len(self.events):
|
||||||
|
return None
|
||||||
|
|
||||||
|
return self.events[index]
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FIND LAST STATE BEFORE EVENT
|
||||||
|
# -----------------------------
|
||||||
|
def rewind_to_event(self, node_name, event_index):
|
||||||
|
|
||||||
|
history = []
|
||||||
|
|
||||||
|
for i in range(event_index + 1):
|
||||||
|
e = self.events[i]
|
||||||
|
|
||||||
|
if e.get("data", {}).get("node") == node_name:
|
||||||
|
history.append(e)
|
||||||
|
|
||||||
|
return history
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FIND LAST SCHEDULER DECISION
|
||||||
|
# -----------------------------
|
||||||
|
def last_scheduler_state(self, node_name, event_index):
|
||||||
|
|
||||||
|
for i in range(event_index, -1, -1):
|
||||||
|
e = self.events[i]
|
||||||
|
|
||||||
|
if e["type"] == "schedule_decision":
|
||||||
|
if e.get("data", {}).get("node") == node_name:
|
||||||
|
return e["data"]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
@ -0,0 +1,160 @@
|
||||||
|
from collections import defaultdict
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
class CauseGraphEngine:
|
||||||
|
"""
|
||||||
|
Builds a causal graph from system events.
|
||||||
|
|
||||||
|
This is NOT execution logic.
|
||||||
|
It is a post-hoc + live reasoning layer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# node -> causes
|
||||||
|
self.graph = defaultdict(list)
|
||||||
|
|
||||||
|
# event timeline
|
||||||
|
self.events = []
|
||||||
|
|
||||||
|
# last decision context
|
||||||
|
self.last_context = {}
|
||||||
|
|
||||||
|
# bus reference (set by attach_bus)
|
||||||
|
self.bus = None
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# BUS BINDING
|
||||||
|
# -------------------------------------------------
|
||||||
|
def attach_bus(self, bus):
|
||||||
|
"""Bind to the event bus. Does NOT subscribe — caller does that."""
|
||||||
|
self.bus = bus
|
||||||
|
|
||||||
|
def emit_debug(self, data):
|
||||||
|
"""Emit a debug trace event back to the bus. Use sparingly to avoid
|
||||||
|
recursion: subscribers of `debug` events MUST NOT call emit_debug."""
|
||||||
|
if self.bus is None:
|
||||||
|
return
|
||||||
|
self.bus.emit(
|
||||||
|
"debug",
|
||||||
|
node=None,
|
||||||
|
action=None,
|
||||||
|
state="trace",
|
||||||
|
meta=data,
|
||||||
|
)
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# INGEST EVENTS
|
||||||
|
# -------------------------------------------------
|
||||||
|
def ingest(self, event: dict):
|
||||||
|
self.events.append(event)
|
||||||
|
|
||||||
|
etype = event.get("type")
|
||||||
|
node = event.get("node")
|
||||||
|
action = event.get("action")
|
||||||
|
|
||||||
|
# store context for causal linking
|
||||||
|
if etype == "task_update":
|
||||||
|
self._handle_task(event)
|
||||||
|
elif etype == "cache_update":
|
||||||
|
self._handle_cache(event)
|
||||||
|
elif etype == "failure":
|
||||||
|
self._handle_failure(event)
|
||||||
|
elif etype == "node_update":
|
||||||
|
self._handle_node(event)
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# TASK EXECUTION CAUSALITY
|
||||||
|
# -------------------------------------------------
|
||||||
|
def _handle_task(self, event):
|
||||||
|
node = event.get("node")
|
||||||
|
action = event.get("action")
|
||||||
|
state = event.get("state")
|
||||||
|
|
||||||
|
context = {
|
||||||
|
"time": time.time(),
|
||||||
|
"node": node,
|
||||||
|
"action": action,
|
||||||
|
"state": state,
|
||||||
|
"reason": "scheduler_assignment",
|
||||||
|
}
|
||||||
|
|
||||||
|
self.last_context[(node, action)] = context
|
||||||
|
|
||||||
|
self.graph[(node, action)].append(context)
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# CACHE CAUSALITY
|
||||||
|
# -------------------------------------------------
|
||||||
|
def _handle_cache(self, event):
|
||||||
|
node = event.get("node")
|
||||||
|
action = event.get("action")
|
||||||
|
|
||||||
|
context = {
|
||||||
|
"time": time.time(),
|
||||||
|
"node": node,
|
||||||
|
"action": action,
|
||||||
|
"reason": "cache_hit_or_miss",
|
||||||
|
}
|
||||||
|
|
||||||
|
self.graph[(node, action)].append(context)
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# FAILURE CAUSALITY
|
||||||
|
# -------------------------------------------------
|
||||||
|
def _handle_failure(self, event):
|
||||||
|
node = event.get("node")
|
||||||
|
action = event.get("action")
|
||||||
|
|
||||||
|
prev = self.last_context.get((node, action), {})
|
||||||
|
|
||||||
|
context = {
|
||||||
|
"time": time.time(),
|
||||||
|
"node": node,
|
||||||
|
"action": action,
|
||||||
|
"reason": "execution_failure",
|
||||||
|
"parent_context": prev,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.graph[(node, action)].append(context)
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# NODE STATE CAUSALITY
|
||||||
|
# -------------------------------------------------
|
||||||
|
def _handle_node(self, event):
|
||||||
|
node = event.get("node")
|
||||||
|
|
||||||
|
context = {
|
||||||
|
"time": time.time(),
|
||||||
|
"node": node,
|
||||||
|
"reason": "node_state_update",
|
||||||
|
"state": event.get("node_state", {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
self.graph[(node, None)].append(context)
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# QUERY INTERFACE
|
||||||
|
# -------------------------------------------------
|
||||||
|
def explain(self, node, action=None):
|
||||||
|
"""
|
||||||
|
Returns causal chain for a given execution unit.
|
||||||
|
|
||||||
|
If action is None, returns ALL events for the given node
|
||||||
|
(any action). Otherwise returns events for the specific
|
||||||
|
(node, action) pair.
|
||||||
|
"""
|
||||||
|
if action is not None:
|
||||||
|
return self.graph.get((node, action), [])
|
||||||
|
|
||||||
|
# Node-only query: scan all keys starting with this node
|
||||||
|
results = []
|
||||||
|
for (n, a), events in self.graph.items():
|
||||||
|
if n == node:
|
||||||
|
results.extend(events)
|
||||||
|
# Sort by time
|
||||||
|
results.sort(key=lambda e: e.get("time", 0))
|
||||||
|
return results
|
||||||
|
|
||||||
|
def full_trace(self):
|
||||||
|
return dict(self.graph)
|
||||||
|
|
@ -0,0 +1,106 @@
|
||||||
|
"""
|
||||||
|
Failure Autopsy — post-hoc root cause analysis for a failed action.
|
||||||
|
|
||||||
|
Given a journal (TimelineStore) and an action name, produces:
|
||||||
|
- the failure event itself
|
||||||
|
- the backward dependency trace (recursive walk)
|
||||||
|
- the last scheduler decision for that action
|
||||||
|
- whether the action is on the critical path
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
class FailureAutopsy:
|
||||||
|
def __init__(self, journal, critical_path: Optional[Dict[str, Any]] = None):
|
||||||
|
"""
|
||||||
|
journal: TimelineStore (or anything with `trace_action` + `find_failures`)
|
||||||
|
critical_path: optional dict with at least `score_map` (set of action names)
|
||||||
|
"""
|
||||||
|
self.journal = journal
|
||||||
|
self.critical_path = critical_path or {}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FIND FAILURE EVENTS
|
||||||
|
# -----------------------------
|
||||||
|
def find_failures(self) -> List[Dict[str, Any]]:
|
||||||
|
return [
|
||||||
|
e for e in self.journal.find_failures()
|
||||||
|
if (e.get("data", {}).get("state") == "failed" or e.get("state") == "failed")
|
||||||
|
]
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# TRACE BACKWARD DEPENDENCY CHAIN
|
||||||
|
# -----------------------------
|
||||||
|
def trace_dependencies(self, action_name: str) -> List[Dict[str, Any]]:
|
||||||
|
trace: List[Dict[str, Any]] = []
|
||||||
|
visited: set = set()
|
||||||
|
|
||||||
|
def walk(name: str):
|
||||||
|
if name in visited:
|
||||||
|
return
|
||||||
|
visited.add(name)
|
||||||
|
|
||||||
|
events = self.journal.trace_action(name)
|
||||||
|
trace.append({"action": name, "events": events})
|
||||||
|
|
||||||
|
for e in events:
|
||||||
|
d = e.get("data", e)
|
||||||
|
deps = d.get("deps", [])
|
||||||
|
for dep in deps:
|
||||||
|
walk(dep)
|
||||||
|
|
||||||
|
walk(action_name)
|
||||||
|
return trace
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET LAST SCHEDULER DECISION
|
||||||
|
# -----------------------------
|
||||||
|
def last_decision(self, action_name: str) -> Optional[Dict[str, Any]]:
|
||||||
|
events = self.journal.trace_action(action_name)
|
||||||
|
for e in reversed(events):
|
||||||
|
if e.get("type") == "schedule_decision":
|
||||||
|
return e.get("data", e)
|
||||||
|
# Fallback: return last task_update with a node assignment
|
||||||
|
for e in reversed(events):
|
||||||
|
d = e.get("data", e)
|
||||||
|
if d.get("node"):
|
||||||
|
return {
|
||||||
|
"node": d.get("node"),
|
||||||
|
"score": d.get("score"),
|
||||||
|
"reason": "scheduler_assignment",
|
||||||
|
}
|
||||||
|
return None
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FULL AUTOPSY REPORT
|
||||||
|
# -----------------------------
|
||||||
|
def report(self, action_name: str) -> Dict[str, Any]:
|
||||||
|
failures = self.find_failures()
|
||||||
|
|
||||||
|
target_failure: Optional[Dict[str, Any]] = None
|
||||||
|
for f in failures:
|
||||||
|
d = f.get("data", f)
|
||||||
|
if d.get("action") == action_name:
|
||||||
|
target_failure = f
|
||||||
|
break
|
||||||
|
|
||||||
|
if not target_failure:
|
||||||
|
return {
|
||||||
|
"status": "no_failure_found",
|
||||||
|
"action": action_name,
|
||||||
|
}
|
||||||
|
|
||||||
|
deps_trace = self.trace_dependencies(action_name)
|
||||||
|
last_sched = self.last_decision(action_name)
|
||||||
|
|
||||||
|
score_map = self.critical_path.get("score_map", {}) if isinstance(self.critical_path, dict) else {}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": "failure_detected",
|
||||||
|
"action": action_name,
|
||||||
|
"failure_event": target_failure,
|
||||||
|
"last_scheduler_decision": last_sched,
|
||||||
|
"dependency_trace": deps_trace,
|
||||||
|
"on_critical_path": action_name in score_map,
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,80 @@
|
||||||
|
"""
|
||||||
|
Failure Propagation — given a failed action, compute the forward blast radius.
|
||||||
|
|
||||||
|
The "blast radius" is the set of downstream actions that depend (transitively)
|
||||||
|
on the failed action and therefore cannot complete.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Set
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
|
||||||
|
class FailurePropagator:
|
||||||
|
"""
|
||||||
|
Maintains a reverse dependency index: for each action, the set of
|
||||||
|
actions that depend on it. Given a failed action, walks the reverse
|
||||||
|
graph to compute the impact set.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# action -> set of actions that depend on it
|
||||||
|
self._revdeps: Dict[str, Set[str]] = defaultdict(set)
|
||||||
|
# action -> direct deps (so we can rebuild revdeps if needed)
|
||||||
|
self._deps: Dict[str, List[str]] = {}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# INGEST
|
||||||
|
# -----------------------------
|
||||||
|
def register_action(self, action: str, deps: List[str]):
|
||||||
|
"""Register an action and its direct dependencies."""
|
||||||
|
self._deps[action] = list(deps)
|
||||||
|
for d in deps:
|
||||||
|
self._revdeps[d].add(action)
|
||||||
|
|
||||||
|
def ingest_event(self, event: Dict[str, Any]):
|
||||||
|
"""Hook for EventBus subscription — picks up pipeline events
|
||||||
|
that declare dependencies and indexes them.
|
||||||
|
|
||||||
|
Handles both flat FesterEvent shape (deps in `meta.deps`) and
|
||||||
|
legacy nested shape (deps in `data.deps`).
|
||||||
|
"""
|
||||||
|
# Try flat shape first: action + deps at top level or in meta
|
||||||
|
action = event.get("action")
|
||||||
|
deps = event.get("deps")
|
||||||
|
if deps is None and isinstance(event.get("meta"), dict):
|
||||||
|
deps = event["meta"].get("deps")
|
||||||
|
# Legacy nested shape
|
||||||
|
if action is None or deps is None:
|
||||||
|
data = event.get("data")
|
||||||
|
if isinstance(data, dict):
|
||||||
|
if action is None:
|
||||||
|
action = data.get("action")
|
||||||
|
if deps is None:
|
||||||
|
deps = data.get("deps")
|
||||||
|
if action and deps:
|
||||||
|
self.register_action(action, deps)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# QUERY
|
||||||
|
# -----------------------------
|
||||||
|
def downstream(self, action: str) -> List[str]:
|
||||||
|
"""Return the transitive set of actions that depend on `action`."""
|
||||||
|
seen: Set[str] = set()
|
||||||
|
stack = [action]
|
||||||
|
while stack:
|
||||||
|
cur = stack.pop()
|
||||||
|
for child in self._revdeps.get(cur, set()):
|
||||||
|
if child not in seen:
|
||||||
|
seen.add(child)
|
||||||
|
stack.append(child)
|
||||||
|
return sorted(seen)
|
||||||
|
|
||||||
|
def impact_report(self, failed_action: str) -> Dict[str, Any]:
|
||||||
|
downstream = self.downstream(failed_action)
|
||||||
|
return {
|
||||||
|
"failed_action": failed_action,
|
||||||
|
"direct_dependents": sorted(self._revdeps.get(failed_action, set())),
|
||||||
|
"transitive_impact": downstream,
|
||||||
|
"impact_count": len(downstream),
|
||||||
|
"total_actions_known": len(self._deps),
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,199 @@
|
||||||
|
"""
|
||||||
|
Timeline Store — immutable event log indexed by node + by absolute index.
|
||||||
|
|
||||||
|
Now backed by SQLite (via backend.storage.sqlite_db) with an in-memory
|
||||||
|
cache for fast recent reads. Falls back to pure in-memory if storage
|
||||||
|
is unavailable.
|
||||||
|
|
||||||
|
Used by:
|
||||||
|
- backend/api/timeline.py (REST endpoints)
|
||||||
|
- backend/analysis/failure_autopsy.py (dependency tracing)
|
||||||
|
- replay UI (via /replay/events)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
# Optional storage backend
|
||||||
|
try:
|
||||||
|
from backend.storage.sqlite_db import STORAGE
|
||||||
|
except Exception:
|
||||||
|
STORAGE = None
|
||||||
|
|
||||||
|
|
||||||
|
class TimelineStore:
|
||||||
|
"""
|
||||||
|
Append-only event journal with:
|
||||||
|
- per-node indices for fast node lookups
|
||||||
|
- per-action indices for fast action tracing
|
||||||
|
- snapshot-at-index for replay rewinding
|
||||||
|
- SQLite persistence (with in-memory cache for recent events)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._events: List[Dict[str, Any]] = []
|
||||||
|
self._by_node: Dict[str, List[int]] = defaultdict(list)
|
||||||
|
self._by_action: Dict[str, List[int]] = defaultdict(list)
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
# On startup, load recent events from storage
|
||||||
|
if STORAGE:
|
||||||
|
self._load_from_storage()
|
||||||
|
|
||||||
|
def _load_from_storage(self, limit: int = 10000):
|
||||||
|
"""Load recent events from SQLite into the in-memory cache."""
|
||||||
|
try:
|
||||||
|
events = STORAGE.get_events(limit=limit)
|
||||||
|
for e in events:
|
||||||
|
self._index_event(e)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _index_event(self, event: Dict[str, Any]):
|
||||||
|
"""Add an event to in-memory indices (no lock — caller must hold it)."""
|
||||||
|
idx = len(self._events)
|
||||||
|
self._events.append(event)
|
||||||
|
node = event.get("node") or (event.get("data", {}) or {}).get("node")
|
||||||
|
if node:
|
||||||
|
self._by_node[node].append(idx)
|
||||||
|
action = event.get("action") or (event.get("data", {}) or {}).get("action")
|
||||||
|
if action:
|
||||||
|
self._by_action[action].append(idx)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# APPEND
|
||||||
|
# -----------------------------
|
||||||
|
def append(self, event: Dict[str, Any]) -> int:
|
||||||
|
"""Append an event to the journal. Returns the new index."""
|
||||||
|
with self._lock:
|
||||||
|
if "timestamp" not in event:
|
||||||
|
event["timestamp"] = time.time()
|
||||||
|
self._index_event(event)
|
||||||
|
|
||||||
|
# Persist to SQLite (outside the lock)
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
STORAGE.append_event(event)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
return len(self._events) - 1
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# READ
|
||||||
|
# -----------------------------
|
||||||
|
def all(self) -> List[Dict[str, Any]]:
|
||||||
|
with self._lock:
|
||||||
|
return list(self._events)
|
||||||
|
|
||||||
|
def get_node_events(self, node: str) -> List[Dict[str, Any]]:
|
||||||
|
# Try in-memory first
|
||||||
|
with self._lock:
|
||||||
|
if node in self._by_node:
|
||||||
|
return [self._events[i] for i in self._by_node[node] if i < len(self._events)]
|
||||||
|
# Fall back to storage (queries older events not in cache)
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
return STORAGE.get_node_events(node)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_action_events(self, action: str) -> List[Dict[str, Any]]:
|
||||||
|
with self._lock:
|
||||||
|
if action in self._by_action:
|
||||||
|
return [self._events[i] for i in self._by_action[action] if i < len(self._events)]
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
return STORAGE.get_action_events(action)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return []
|
||||||
|
|
||||||
|
def trace_action(self, action: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Alias for get_action_events — kept for autopsy compatibility."""
|
||||||
|
return self.get_action_events(action)
|
||||||
|
|
||||||
|
def find_failures(self) -> List[Dict[str, Any]]:
|
||||||
|
# In-memory
|
||||||
|
with self._lock:
|
||||||
|
in_mem = [
|
||||||
|
e for e in self._events
|
||||||
|
if e.get("type") in ("failure", "task_update")
|
||||||
|
and (e.get("state") == "failed" or e.get("data", {}).get("state") == "failed")
|
||||||
|
]
|
||||||
|
# Storage (may have more)
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
stored = STORAGE.find_failures()
|
||||||
|
# Merge + dedupe by timestamp+action
|
||||||
|
seen = {(e.get("timestamp"), e.get("action")) for e in in_mem}
|
||||||
|
for e in stored:
|
||||||
|
key = (e.get("timestamp"), e.get("action"))
|
||||||
|
if key not in seen:
|
||||||
|
in_mem.append(e)
|
||||||
|
seen.add(key)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return in_mem
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# REWIND / SNAPSHOT
|
||||||
|
# -----------------------------
|
||||||
|
def snapshot_at(self, index: int) -> Dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
safe = max(0, min(index, len(self._events) - 1))
|
||||||
|
events = self._events[:safe + 1]
|
||||||
|
|
||||||
|
# rebuild node + action views
|
||||||
|
nodes: Dict[str, Dict[str, Any]] = {}
|
||||||
|
actions: Dict[str, Dict[str, Any]] = {}
|
||||||
|
for e in events:
|
||||||
|
d = e.get("data", e)
|
||||||
|
n = d.get("node") or e.get("node")
|
||||||
|
a = d.get("action") or e.get("action")
|
||||||
|
s = d.get("state") or e.get("state")
|
||||||
|
if n:
|
||||||
|
nodes[n] = {"name": n, "state": s or "online", "last_event": e.get("timestamp")}
|
||||||
|
if a:
|
||||||
|
actions[a] = {"name": a, "state": s or "pending", "node": n, "last_event": e.get("timestamp")}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"index": safe,
|
||||||
|
"event_count": len(events),
|
||||||
|
"nodes": list(nodes.values()),
|
||||||
|
"actions": list(actions.values()),
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STATS
|
||||||
|
# -----------------------------
|
||||||
|
def stats(self) -> Dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
total = len(self._events)
|
||||||
|
failures = len([e for e in self._events if e.get("type") == "failure"])
|
||||||
|
# If we have storage, get the authoritative count
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
total = STORAGE.event_count()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {
|
||||||
|
"total_events": total,
|
||||||
|
"nodes_tracked": len(self._by_node),
|
||||||
|
"actions_tracked": len(self._by_action),
|
||||||
|
"failures": failures,
|
||||||
|
}
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
with self._lock:
|
||||||
|
self._events.clear()
|
||||||
|
self._by_node.clear()
|
||||||
|
self._by_action.clear()
|
||||||
|
|
||||||
|
|
||||||
|
# Process-wide singleton (used by api/timeline.py)
|
||||||
|
STORE = TimelineStore()
|
||||||
|
|
@ -0,0 +1,107 @@
|
||||||
|
"""
|
||||||
|
Fester backend — core API facade.
|
||||||
|
|
||||||
|
This module exposes a single `bus`, `registry`, `observability`, `ws_stream`,
|
||||||
|
plus thin wrapper functions for the API layer. It used to be broken
|
||||||
|
(referenced `bus` before creation, imported nonexistent `_endpoint` symbols,
|
||||||
|
duplicated CauseGraphEngine setup). It now wires everything cleanly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.events.bus import EventBus
|
||||||
|
from backend.metrics.observability import ObservabilityHub
|
||||||
|
from backend.metrics.node_state import NodeStateRegistry
|
||||||
|
from backend.analysis.cause_graph import CauseGraphEngine
|
||||||
|
|
||||||
|
# =========================================================
|
||||||
|
# GLOBAL CORE SYSTEM INITIALIZATION (SINGLETON LAYER)
|
||||||
|
# =========================================================
|
||||||
|
|
||||||
|
bus = EventBus()
|
||||||
|
registry = NodeStateRegistry()
|
||||||
|
observability = ObservabilityHub(registry, bus)
|
||||||
|
observability.attach_bus()
|
||||||
|
|
||||||
|
# Cause graph — subscribes ONCE, no recursion (ingest only)
|
||||||
|
cause_graph = CauseGraphEngine()
|
||||||
|
cause_graph.attach_bus(bus)
|
||||||
|
bus.subscribe(cause_graph.ingest)
|
||||||
|
|
||||||
|
# WebSocket stream — attached lazily by main.py when FastAPI starts,
|
||||||
|
# because the WS layer needs the running app's connection registry.
|
||||||
|
ws_stream = None # set by backend.main when app boots
|
||||||
|
|
||||||
|
|
||||||
|
def attach_ws_stream(stream):
|
||||||
|
"""Called by backend.main during startup to bind a WebSocketStream."""
|
||||||
|
global ws_stream
|
||||||
|
ws_stream = stream
|
||||||
|
ws_stream.attach_bus(bus)
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================
|
||||||
|
# API REGISTRATION LAYER
|
||||||
|
# Thin wrappers so Cockpit / external callers can bind cleanly.
|
||||||
|
# =========================================================
|
||||||
|
|
||||||
|
# These imports are deliberately placed AFTER the singletons above are
|
||||||
|
# initialized, because some api/* modules may want to import `bus` etc.
|
||||||
|
from backend.api import build as build_api
|
||||||
|
from backend.api import nodes as nodes_api
|
||||||
|
from backend.api import replay as replay_api
|
||||||
|
from backend.api import debugger as debugger_api
|
||||||
|
from backend.api import timeline as timeline_api
|
||||||
|
from backend.api import metrics as metrics_api
|
||||||
|
from backend.api import autopsy as autopsy_api
|
||||||
|
from backend.api import pipeline_control as pipeline_control_api
|
||||||
|
from backend.api import cause as cause_api
|
||||||
|
|
||||||
|
|
||||||
|
def build(spec, nodes, cluster=None, intelligence=None):
|
||||||
|
return build_api.build_endpoint(spec, nodes, cluster, intelligence)
|
||||||
|
|
||||||
|
|
||||||
|
def replay(session_id):
|
||||||
|
return replay_api.replay_endpoint(session_id, registry)
|
||||||
|
|
||||||
|
|
||||||
|
def debugger(session_id):
|
||||||
|
return debugger_api.debugger_endpoint(session_id, registry, bus)
|
||||||
|
|
||||||
|
|
||||||
|
def timeline(session_id):
|
||||||
|
return timeline_api.timeline_endpoint(session_id)
|
||||||
|
|
||||||
|
|
||||||
|
def nodes():
|
||||||
|
return nodes_api.nodes_endpoint(registry)
|
||||||
|
|
||||||
|
|
||||||
|
def metrics():
|
||||||
|
return metrics_api.metrics_endpoint(registry)
|
||||||
|
|
||||||
|
|
||||||
|
def autopsy(session_id):
|
||||||
|
return autopsy_api.autopsy_endpoint(session_id)
|
||||||
|
|
||||||
|
|
||||||
|
def pipeline_control(action, payload=None):
|
||||||
|
return pipeline_control_api.pipeline_control_endpoint(action, payload, bus)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"bus",
|
||||||
|
"registry",
|
||||||
|
"observability",
|
||||||
|
"ws_stream",
|
||||||
|
"cause_graph",
|
||||||
|
"attach_ws_stream",
|
||||||
|
|
||||||
|
"build",
|
||||||
|
"replay",
|
||||||
|
"debugger",
|
||||||
|
"timeline",
|
||||||
|
"nodes",
|
||||||
|
"metrics",
|
||||||
|
"autopsy",
|
||||||
|
"pipeline_control",
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
"""
|
||||||
|
Autopsy API — post-hoc root cause analysis for failed actions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from backend.analysis.failure_autopsy import FailureAutopsy
|
||||||
|
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||||
|
from backend.api.replay import SESSION_DB
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/autopsy", tags=["autopsy"])
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RUN AUTOPSY
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("/{session_id}/{action}")
|
||||||
|
def run_autopsy(session_id: str, action: str):
|
||||||
|
"""Run a failure autopsy for `action` against the replay session `session_id`."""
|
||||||
|
|
||||||
|
# Try to fetch events for this session from SessionDB
|
||||||
|
events = SESSION_DB.get_replay_stream(session_id)
|
||||||
|
|
||||||
|
if not events:
|
||||||
|
# Fallback: use the live timeline store
|
||||||
|
journal = TIMELINE_STORE
|
||||||
|
else:
|
||||||
|
# Wrap the session's event list in a tiny shim that quacks like
|
||||||
|
# the TimelineStore API the FailureAutopsy expects.
|
||||||
|
journal = _JournalShim(events)
|
||||||
|
|
||||||
|
autopsy = FailureAutopsy(journal)
|
||||||
|
return autopsy.report(action)
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# Legacy wrapper for api.py facade
|
||||||
|
# -----------------------------
|
||||||
|
def autopsy_endpoint(session_id, action=None):
|
||||||
|
"""Legacy entrypoint — caller passed only session_id; we look up
|
||||||
|
failures across all known sessions."""
|
||||||
|
events = SESSION_DB.get_replay_stream(session_id)
|
||||||
|
if not events:
|
||||||
|
return {"error": "session not found"}
|
||||||
|
journal = _JournalShim(events)
|
||||||
|
autopsy = FailureAutopsy(journal)
|
||||||
|
# If no action given, report on the first failure we find
|
||||||
|
if action is None:
|
||||||
|
failures = autopsy.find_failures()
|
||||||
|
if not failures:
|
||||||
|
return {"status": "no_failures_in_session", "session_id": session_id}
|
||||||
|
action = failures[0].get("data", failures[0]).get("action")
|
||||||
|
if not action:
|
||||||
|
return {"status": "no_actionable_failure", "session_id": session_id}
|
||||||
|
return autopsy.report(action)
|
||||||
|
|
||||||
|
|
||||||
|
class _JournalShim:
|
||||||
|
"""Minimal adapter that lets FailureAutopsy work against a plain list
|
||||||
|
of events (from a replay session) instead of a full TimelineStore."""
|
||||||
|
|
||||||
|
def __init__(self, events):
|
||||||
|
self.events = events
|
||||||
|
|
||||||
|
def find_failures(self):
|
||||||
|
return [
|
||||||
|
e for e in self.events
|
||||||
|
if e.get("type") in ("failure", "task_update", "pipeline_update")
|
||||||
|
and (e.get("state") == "failed" or e.get("data", {}).get("state") == "failed")
|
||||||
|
]
|
||||||
|
|
||||||
|
def trace_action(self, action_name):
|
||||||
|
return [
|
||||||
|
e for e in self.events
|
||||||
|
if (e.get("action") == action_name) or (e.get("data", {}).get("action") == action_name)
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,85 @@
|
||||||
|
"""
|
||||||
|
Build endpoint — kicks off a pipeline run for a BuildSpec.
|
||||||
|
|
||||||
|
NOTE: This is the thin API wrapper. The real scheduling + execution
|
||||||
|
lives in backend/pipeline/engine.py and backend/scheduler/.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.api.build_spec import BuildSpec
|
||||||
|
from backend.pipeline.engine import PipelineEngine
|
||||||
|
from backend.graph.build_graph import BuildGraph
|
||||||
|
from backend.api.api import bus, registry # singletons
|
||||||
|
|
||||||
|
|
||||||
|
async def build_endpoint(spec, nodes, cluster=None, intelligence=None):
|
||||||
|
"""
|
||||||
|
Build a project.
|
||||||
|
|
||||||
|
`spec` may be:
|
||||||
|
- a BuildSpec instance
|
||||||
|
- a dict (will be coerced via BuildSpec.from_request)
|
||||||
|
- a raw dict matching the legacy BuildGraph constructor (with `target`/`output_mode`)
|
||||||
|
|
||||||
|
`nodes`, `cluster`, `intelligence` are accepted for backward-compat but
|
||||||
|
only `nodes` is actually used (the engine has its own node registry).
|
||||||
|
"""
|
||||||
|
# Coerce spec into something BuildGraph can consume.
|
||||||
|
# BuildGraph reads spec.arch, spec.target, spec.toolchain, spec.output_mode.
|
||||||
|
# BuildSpec (the dataclass) doesn't have target/output_mode, so we adapt.
|
||||||
|
if isinstance(spec, BuildSpec):
|
||||||
|
bs = spec
|
||||||
|
elif isinstance(spec, dict):
|
||||||
|
# If caller passed a raw dict with `target`/`output_mode`, wrap it
|
||||||
|
# in a tiny shim object so BuildGraph's attribute access works.
|
||||||
|
if "target" in spec or "output_mode" in spec:
|
||||||
|
bs = _ShimSpec(spec)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
bs = BuildSpec.from_request(spec)
|
||||||
|
except KeyError:
|
||||||
|
bs = _ShimSpec(spec)
|
||||||
|
else:
|
||||||
|
# Already a shim or has the attributes BuildGraph expects
|
||||||
|
bs = spec
|
||||||
|
|
||||||
|
# Generate the DAG
|
||||||
|
try:
|
||||||
|
graph = BuildGraph(bs).generate()
|
||||||
|
except Exception as e:
|
||||||
|
# In production we'd log this; for now bubble up a structured error
|
||||||
|
return {"status": "error", "error": f"build_graph_failed: {e}"}
|
||||||
|
|
||||||
|
# Run the pipeline
|
||||||
|
engine = PipelineEngine(nodes, registry, bus)
|
||||||
|
try:
|
||||||
|
results = await engine.run(bs)
|
||||||
|
except Exception as e:
|
||||||
|
return {"status": "error", "error": f"pipeline_failed: {e}"}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": "complete",
|
||||||
|
"actions": [{"name": n, "state": s} for n, s in results],
|
||||||
|
"graph_size": len(graph),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class _ShimSpec:
|
||||||
|
"""Adapter that exposes attribute access over a dict, so BuildGraph
|
||||||
|
can read spec.target / spec.output_mode / spec.arch / spec.toolchain
|
||||||
|
regardless of which shape the caller supplied."""
|
||||||
|
|
||||||
|
def __init__(self, d: dict):
|
||||||
|
self._d = d
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
if name in self._d:
|
||||||
|
return self._d[name]
|
||||||
|
# Sensible defaults for fields BuildGraph reads
|
||||||
|
defaults = {
|
||||||
|
"arch": "x86_64",
|
||||||
|
"target": "linux-gnu",
|
||||||
|
"toolchain": "gcc",
|
||||||
|
"output_mode": "tar",
|
||||||
|
"project": "unknown",
|
||||||
|
}
|
||||||
|
return defaults.get(name)
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class BuildSpec:
|
||||||
|
|
||||||
|
project: str
|
||||||
|
arch: str
|
||||||
|
distro_target: str
|
||||||
|
toolchain: str
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# EXECUTION SELECTION
|
||||||
|
# -----------------------------
|
||||||
|
execution: str # distcc | lxc | libvirt | tmux
|
||||||
|
container_backend: str # optional fallback layer
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# POLICY PROFILE
|
||||||
|
# -----------------------------
|
||||||
|
profile: str = "balanced"
|
||||||
|
|
||||||
|
|
||||||
|
def from_request(req):
|
||||||
|
|
||||||
|
return BuildSpec(
|
||||||
|
project=req["project"],
|
||||||
|
arch=req["arch"],
|
||||||
|
distro_target=req["distro_target"],
|
||||||
|
toolchain=req["toolchain"],
|
||||||
|
execution=req.get("execution", "distcc"),
|
||||||
|
container_backend=req.get("container_backend", "lxc"),
|
||||||
|
profile=req.get("profile", "balanced")
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
"""
|
||||||
|
Cause graph API — explain why a node/action made its decisions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from backend.api.api import cause_graph
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/cause", tags=["cause"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/explain/{node}")
|
||||||
|
def explain_node(node: str, action: Optional[str] = None):
|
||||||
|
"""Explain the causal chain for a given node (+ optional action)."""
|
||||||
|
chain = cause_graph.explain(node, action)
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"action": action,
|
||||||
|
"events": len(chain),
|
||||||
|
"chain": chain,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/trace")
|
||||||
|
def full_trace():
|
||||||
|
"""Return the entire cause graph as a dict."""
|
||||||
|
return {
|
||||||
|
"nodes_tracked": len(cause_graph.graph),
|
||||||
|
"events_seen": len(cause_graph.events),
|
||||||
|
"trace": {str(k): v for k, v in cause_graph.full_trace().items()},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/events")
|
||||||
|
def recent_events(limit: int = 100):
|
||||||
|
"""Return the most recent N ingested events."""
|
||||||
|
events = cause_graph.events[-limit:]
|
||||||
|
return {
|
||||||
|
"count": len(events),
|
||||||
|
"events": events,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Legacy wrappers (kept for api.py facade)
|
||||||
|
def explain_node_legacy(cause_graph_instance, node, action=None):
|
||||||
|
return cause_graph_instance.explain(node, action)
|
||||||
|
|
||||||
|
|
||||||
|
def full_cause_trace(cause_graph_instance):
|
||||||
|
return cause_graph_instance.full_trace()
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
"""
|
||||||
|
Debugger API — pause / resume / step an in-flight pipeline.
|
||||||
|
|
||||||
|
The engine is set by main.py on startup via `set_engine(...)`. If no
|
||||||
|
engine is registered (e.g. when running the mock server), endpoints
|
||||||
|
return a structured error rather than crashing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from typing import Optional, Any
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/debugger", tags=["debugger"])
|
||||||
|
|
||||||
|
# Single in-memory engine reference (set by main.py)
|
||||||
|
_ENGINE: Optional[Any] = None
|
||||||
|
_DEBUG_STATE: dict = {
|
||||||
|
"paused": False,
|
||||||
|
"current_step": 0,
|
||||||
|
"history": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def set_engine(engine):
|
||||||
|
global _ENGINE
|
||||||
|
_ENGINE = engine
|
||||||
|
|
||||||
|
|
||||||
|
def get_engine():
|
||||||
|
return _ENGINE
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RESUME
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/resume")
|
||||||
|
def resume():
|
||||||
|
_DEBUG_STATE["paused"] = False
|
||||||
|
if _ENGINE and hasattr(_ENGINE, "resume"):
|
||||||
|
_ENGINE.resume()
|
||||||
|
return {"state": "running"}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# PAUSE
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/pause")
|
||||||
|
def pause():
|
||||||
|
_DEBUG_STATE["paused"] = True
|
||||||
|
if _ENGINE and hasattr(_ENGINE, "pause"):
|
||||||
|
_ENGINE.pause()
|
||||||
|
return {"state": "paused"}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STEP
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/step")
|
||||||
|
def step():
|
||||||
|
if _ENGINE and hasattr(_ENGINE, "step"):
|
||||||
|
_ENGINE.step()
|
||||||
|
_DEBUG_STATE["current_step"] += 1
|
||||||
|
return {"state": "stepped", "step": _DEBUG_STATE["current_step"]}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STEP BACK (replay-only)
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/step-back")
|
||||||
|
def step_back():
|
||||||
|
if _DEBUG_STATE["current_step"] > 0:
|
||||||
|
_DEBUG_STATE["current_step"] -= 1
|
||||||
|
return {"state": "stepped_back", "step": _DEBUG_STATE["current_step"]}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STATE
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("/state")
|
||||||
|
def state():
|
||||||
|
eng = _ENGINE
|
||||||
|
active = eng and hasattr(eng, "is_active") and eng.is_active()
|
||||||
|
return {
|
||||||
|
"paused": _DEBUG_STATE["paused"],
|
||||||
|
"current_step": _DEBUG_STATE["current_step"],
|
||||||
|
"engine_attached": eng is not None,
|
||||||
|
"engine_active": active,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# Legacy wrapper for api.py facade
|
||||||
|
# -----------------------------
|
||||||
|
def debugger_endpoint(session_id, registry=None, bus=None):
|
||||||
|
"""Legacy — return current debugger state for a session."""
|
||||||
|
return {
|
||||||
|
"session_id": session_id,
|
||||||
|
"paused": _DEBUG_STATE["paused"],
|
||||||
|
"current_step": _DEBUG_STATE["current_step"],
|
||||||
|
"engine_attached": _ENGINE is not None,
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,158 @@
|
||||||
|
"""
|
||||||
|
Metrics endpoint — Prometheus exposition + JSON view for the UI.
|
||||||
|
|
||||||
|
Emits metric names matching add-to-grafana-config.json:
|
||||||
|
- fester_node_cpu (gauge, per node)
|
||||||
|
- fester_node_temp (gauge, per node)
|
||||||
|
- fester_node_active_jobs (gauge, per node)
|
||||||
|
- fester_node_instability (gauge, per node)
|
||||||
|
- fester_nodes_total (gauge)
|
||||||
|
- fester_nodes_online (gauge)
|
||||||
|
- fester_cluster_avg_heat (gauge)
|
||||||
|
- fester_cluster_active_jobs (gauge)
|
||||||
|
- fester_pipeline_actions_total (counter, per state) ← grafana expects this
|
||||||
|
- fester_cache_hits_total (counter, per target) ← grafana expects this
|
||||||
|
- fester_scheduler_score (gauge, per node+target)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from fastapi.responses import PlainTextResponse
|
||||||
|
from typing import Optional
|
||||||
|
import threading
|
||||||
|
|
||||||
|
from backend.metrics.observability import ObservabilityHub
|
||||||
|
from backend.metrics.node_state import NodeStateRegistry
|
||||||
|
|
||||||
|
router = APIRouter(tags=["metrics"])
|
||||||
|
|
||||||
|
# Process-wide counters (incremented by main.py's bus subscriber)
|
||||||
|
_pipeline_action_counts: dict = {} # state -> count
|
||||||
|
_cache_hit_counts: dict = {} # target -> count
|
||||||
|
_scheduler_scores: dict = {} # (node, target) -> score
|
||||||
|
_metrics_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def record_pipeline_action(state: str):
|
||||||
|
"""Called by main.py's bus subscriber when a task_update event fires."""
|
||||||
|
with _metrics_lock:
|
||||||
|
_pipeline_action_counts[state] = _pipeline_action_counts.get(state, 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def record_cache_hit(target: str):
|
||||||
|
with _metrics_lock:
|
||||||
|
_cache_hit_counts[target] = _cache_hit_counts.get(target, 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def record_scheduler_score(node: str, target: str, score: float):
|
||||||
|
with _metrics_lock:
|
||||||
|
_scheduler_scores[(node, target)] = score
|
||||||
|
|
||||||
|
|
||||||
|
def _gather_metrics(registry: NodeStateRegistry):
|
||||||
|
"""Build a JSON-serializable metrics snapshot from the node state registry."""
|
||||||
|
nodes = registry.all()
|
||||||
|
total_jobs = sum(n.active_jobs for n in nodes)
|
||||||
|
avg_heat = (sum(n.temp for n in nodes) / len(nodes)) if nodes else 0.0
|
||||||
|
online = sum(1 for n in nodes if n.temp > 0 or n.active_jobs > 0)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"nodes": {
|
||||||
|
"total": len(nodes),
|
||||||
|
"online": online,
|
||||||
|
"offline": len(nodes) - online,
|
||||||
|
},
|
||||||
|
"cluster": {
|
||||||
|
"avg_heat": round(avg_heat, 1),
|
||||||
|
"total_active_jobs": total_jobs,
|
||||||
|
},
|
||||||
|
"per_node": [
|
||||||
|
{
|
||||||
|
"name": n.name,
|
||||||
|
"cpu_load": round(n.cpu_load, 2),
|
||||||
|
"memory_load": round(n.memory_load, 2),
|
||||||
|
"temp": round(n.temp, 1),
|
||||||
|
"instability": round(n.instability, 3),
|
||||||
|
"active_jobs": n.active_jobs,
|
||||||
|
"last_seen": n.last_seen,
|
||||||
|
}
|
||||||
|
for n in nodes
|
||||||
|
],
|
||||||
|
"pipeline_actions": dict(_pipeline_action_counts),
|
||||||
|
"cache_hits": dict(_cache_hit_counts),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/api/metrics/json")
|
||||||
|
def metrics_json():
|
||||||
|
"""JSON view of current metrics — consumed by the UI."""
|
||||||
|
from backend.api.api import registry
|
||||||
|
return _gather_metrics(registry)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/metrics", response_class=PlainTextResponse)
|
||||||
|
def metrics_prom():
|
||||||
|
"""Prometheus exposition format — names match add-to-grafana-config.json."""
|
||||||
|
from backend.api.api import registry
|
||||||
|
snap = _gather_metrics(registry)
|
||||||
|
lines = []
|
||||||
|
|
||||||
|
# Cluster-level gauges
|
||||||
|
lines.append("# HELP fester_nodes_total Total nodes known to the cluster")
|
||||||
|
lines.append("# TYPE fester_nodes_total gauge")
|
||||||
|
lines.append(f"fester_nodes_total {snap['nodes']['total']}")
|
||||||
|
lines.append("# HELP fester_nodes_online Online nodes")
|
||||||
|
lines.append("# TYPE fester_nodes_online gauge")
|
||||||
|
lines.append(f"fester_nodes_online {snap['nodes']['online']}")
|
||||||
|
lines.append("# HELP fester_cluster_avg_heat Average cluster heat")
|
||||||
|
lines.append("# TYPE fester_cluster_avg_heat gauge")
|
||||||
|
lines.append(f"fester_cluster_avg_heat {snap['cluster']['avg_heat']}")
|
||||||
|
lines.append("# HELP fester_cluster_active_jobs Total active jobs")
|
||||||
|
lines.append("# TYPE fester_cluster_active_jobs gauge")
|
||||||
|
lines.append(f"fester_cluster_active_jobs {snap['cluster']['total_active_jobs']}")
|
||||||
|
|
||||||
|
# Per-node gauges — emit BOTH names so existing dashboards (which use
|
||||||
|
# fester_node_cpu and fester_node_temp) work, and new ones (cpu_load etc)
|
||||||
|
# also work.
|
||||||
|
for n in snap["per_node"]:
|
||||||
|
labels = f'node="{n["name"]}"'
|
||||||
|
# Canonical names (matching grafana config)
|
||||||
|
lines.append(f'fester_node_cpu{{{labels}}} {n["cpu_load"]}')
|
||||||
|
lines.append(f'fester_node_temp{{{labels}}} {n["temp"]}')
|
||||||
|
# Additional names
|
||||||
|
lines.append(f'fester_node_active_jobs{{{labels}}} {n["active_jobs"]}')
|
||||||
|
lines.append(f'fester_node_instability{{{labels}}} {n["instability"]}')
|
||||||
|
lines.append(f'fester_node_memory_load{{{labels}}} {n["memory_load"]}')
|
||||||
|
|
||||||
|
# Pipeline action counter (grafana expects fester_pipeline_actions_total)
|
||||||
|
lines.append("# HELP fester_pipeline_actions_total Total pipeline actions by state")
|
||||||
|
lines.append("# TYPE fester_pipeline_actions_total counter")
|
||||||
|
with _metrics_lock:
|
||||||
|
for state, count in _pipeline_action_counts.items():
|
||||||
|
lines.append(f'fester_pipeline_actions_total{{state="{state}"}} {count}')
|
||||||
|
|
||||||
|
# Cache hit counter (grafana expects fester_cache_hits_total)
|
||||||
|
lines.append("# HELP fester_cache_hits_total Total cache hits by target")
|
||||||
|
lines.append("# TYPE fester_cache_hits_total counter")
|
||||||
|
with _metrics_lock:
|
||||||
|
for target, count in _cache_hit_counts.items():
|
||||||
|
lines.append(f'fester_cache_hits_total{{target="{target}"}} {count}')
|
||||||
|
|
||||||
|
# Scheduler score gauge (grafana expects fester_scheduler_score)
|
||||||
|
lines.append("# HELP fester_scheduler_score Last scheduler score per node+target")
|
||||||
|
lines.append("# TYPE fester_scheduler_score gauge")
|
||||||
|
with _metrics_lock:
|
||||||
|
for (node, target), score in _scheduler_scores.items():
|
||||||
|
lines.append(f'fester_scheduler_score{{node="{node}",target="{target}"}} {score}')
|
||||||
|
|
||||||
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# Legacy wrapper for api.py facade
|
||||||
|
# -----------------------------
|
||||||
|
def metrics_endpoint(registry=None):
|
||||||
|
"""Legacy — returns the JSON metrics snapshot."""
|
||||||
|
if registry is None:
|
||||||
|
from backend.api.api import registry as r
|
||||||
|
registry = r
|
||||||
|
return _gather_metrics(registry)
|
||||||
|
|
@ -0,0 +1,211 @@
|
||||||
|
"""
|
||||||
|
Nodes API — list nodes, get/set node roles + policies, probe agents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from typing import Optional, Dict, Any
|
||||||
|
|
||||||
|
from backend.nodes.roles_store import load_roles, save_roles, get_role, DEFAULT_ROLE
|
||||||
|
from backend.nodes.roles import NodeRole
|
||||||
|
from backend.nodes.probe import probe_async, normalize
|
||||||
|
from backend.config import CONFIG
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/nodes", tags=["nodes"])
|
||||||
|
|
||||||
|
|
||||||
|
class PolicyBody(BaseModel):
|
||||||
|
policy: Optional[str] = None # preferred | avoid | neutral
|
||||||
|
compile_weight: Optional[float] = None
|
||||||
|
cache_writer: Optional[bool] = None
|
||||||
|
cache_reader: Optional[bool] = None
|
||||||
|
max_thermal_state: Optional[float] = None
|
||||||
|
distcc_enabled: Optional[bool] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _registry():
|
||||||
|
"""Lazy import of the singleton registry from api.api."""
|
||||||
|
from backend.api.api import registry
|
||||||
|
return registry
|
||||||
|
|
||||||
|
|
||||||
|
def _node_view(n: dict, role: NodeRole) -> dict:
|
||||||
|
"""Combine config + role + live registry state into a single view."""
|
||||||
|
name = n["name"]
|
||||||
|
reg = _registry().get(name)
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"host": n.get("host"),
|
||||||
|
"max_jobs": n.get("max_jobs", 1),
|
||||||
|
"state": "online" if (reg is None or not getattr(reg, "offline", False)) else "offline",
|
||||||
|
"policy": _policy_from_role(role),
|
||||||
|
"role": _role_summary(role),
|
||||||
|
# Live metrics from the registry (may be None if no agent has reported)
|
||||||
|
"heat": getattr(reg, "temp", None) if reg else None,
|
||||||
|
"jobs": getattr(reg, "active_jobs", None) if reg else None,
|
||||||
|
"cpu_load": getattr(reg, "cpu_load", None) if reg else None,
|
||||||
|
"memory_load": getattr(reg, "memory_load", None) if reg else None,
|
||||||
|
"instability": getattr(reg, "instability", None) if reg else None,
|
||||||
|
"last_seen": getattr(reg, "last_seen", None) if reg else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# LIST NODES
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("")
|
||||||
|
def list_nodes():
|
||||||
|
"""List configured nodes with their current role + policy + live metrics."""
|
||||||
|
roles = load_roles()
|
||||||
|
nodes = [_node_view(n, roles.get(n["name"], DEFAULT_ROLE)) for n in CONFIG.get("nodes", [])]
|
||||||
|
return {"master": CONFIG.get("master"), "nodes": nodes}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET ONE NODE
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("/{name}")
|
||||||
|
def get_node(name: str):
|
||||||
|
roles = load_roles()
|
||||||
|
for n in CONFIG.get("nodes", []):
|
||||||
|
if n["name"] == name:
|
||||||
|
return _node_view(n, roles.get(name, DEFAULT_ROLE))
|
||||||
|
return {"error": "not found"}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# SET NODE POLICY
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/{name}/policy")
|
||||||
|
def set_node_policy(name: str, body: PolicyBody):
|
||||||
|
roles = load_roles()
|
||||||
|
role = roles.get(name, NodeRole(name=name))
|
||||||
|
|
||||||
|
if body.policy is not None:
|
||||||
|
# Translate policy keyword into compile_weight + thermal cap
|
||||||
|
if body.policy == "preferred":
|
||||||
|
role.compile_weight = 2.0
|
||||||
|
role.max_thermal_state = 0.95
|
||||||
|
elif body.policy == "avoid":
|
||||||
|
role.compile_weight = 0.2
|
||||||
|
role.max_thermal_state = 0.5
|
||||||
|
elif body.policy == "neutral":
|
||||||
|
role.compile_weight = 1.0
|
||||||
|
role.max_thermal_state = 0.85
|
||||||
|
|
||||||
|
if body.compile_weight is not None:
|
||||||
|
role.compile_weight = body.compile_weight
|
||||||
|
if body.cache_writer is not None:
|
||||||
|
role.cache_writer = body.cache_writer
|
||||||
|
if body.cache_reader is not None:
|
||||||
|
role.cache_reader = body.cache_reader
|
||||||
|
if body.max_thermal_state is not None:
|
||||||
|
role.max_thermal_state = body.max_thermal_state
|
||||||
|
if body.distcc_enabled is not None:
|
||||||
|
role.distcc_enabled = body.distcc_enabled
|
||||||
|
|
||||||
|
roles[name] = role
|
||||||
|
save_roles(roles)
|
||||||
|
return {"status": "ok", "node": name, "role": role.__dict__}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# PROBE A NODE (manual)
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/{name}/probe")
|
||||||
|
async def probe_node_endpoint(name: str):
|
||||||
|
"""Manually trigger a probe of the named node's agent.
|
||||||
|
Updates the registry and returns the normalized state."""
|
||||||
|
# Find the node config
|
||||||
|
cfg = None
|
||||||
|
for n in CONFIG.get("nodes", []):
|
||||||
|
if n["name"] == name:
|
||||||
|
cfg = n
|
||||||
|
break
|
||||||
|
if not cfg:
|
||||||
|
raise HTTPException(404, f"node {name} not configured")
|
||||||
|
|
||||||
|
host = cfg.get("host")
|
||||||
|
if not host:
|
||||||
|
return {"error": "node has no host configured", "node": name}
|
||||||
|
|
||||||
|
raw = await probe_async(host)
|
||||||
|
state = normalize(raw, name)
|
||||||
|
|
||||||
|
if raw is not None:
|
||||||
|
# Update the registry with the fresh probe data
|
||||||
|
reg = _registry()
|
||||||
|
reg.update(
|
||||||
|
name,
|
||||||
|
cpu_load=state["cpu_load"],
|
||||||
|
memory_load=state["memory_load"],
|
||||||
|
temp=state["temp"],
|
||||||
|
instability=state["instability"],
|
||||||
|
active_jobs=state["active_jobs"],
|
||||||
|
last_seen=state["last_seen"],
|
||||||
|
)
|
||||||
|
return {"status": "ok", "node": name, "source": "agent", "state": state, "raw": raw}
|
||||||
|
else:
|
||||||
|
return {"status": "unreachable", "node": name, "host": host, "state": state}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# PROBE ALL NODES
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/probe-all")
|
||||||
|
async def probe_all_nodes_endpoint():
|
||||||
|
"""Probe every configured node in parallel."""
|
||||||
|
import asyncio
|
||||||
|
cfg_nodes = CONFIG.get("nodes", [])
|
||||||
|
|
||||||
|
async def probe_one(n):
|
||||||
|
host = n.get("host")
|
||||||
|
if not host:
|
||||||
|
return {"node": n["name"], "host": None, "reachable": False,
|
||||||
|
"state": normalize(None, n["name"])}
|
||||||
|
raw = await probe_async(host)
|
||||||
|
state = normalize(raw, n["name"])
|
||||||
|
if raw is not None:
|
||||||
|
reg = _registry()
|
||||||
|
reg.update(
|
||||||
|
n["name"],
|
||||||
|
cpu_load=state["cpu_load"],
|
||||||
|
memory_load=state["memory_load"],
|
||||||
|
temp=state["temp"],
|
||||||
|
instability=state["instability"],
|
||||||
|
active_jobs=state["active_jobs"],
|
||||||
|
last_seen=state["last_seen"],
|
||||||
|
)
|
||||||
|
return {"node": n["name"], "host": host, "reachable": raw is not None, "state": state}
|
||||||
|
|
||||||
|
results = await asyncio.gather(*[probe_one(n) for n in cfg_nodes])
|
||||||
|
return {"results": list(results)}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# HELPERS
|
||||||
|
# -----------------------------
|
||||||
|
def _policy_from_role(role: NodeRole) -> str:
|
||||||
|
if role.compile_weight >= 2.0:
|
||||||
|
return "preferred"
|
||||||
|
if role.compile_weight <= 0.2:
|
||||||
|
return "avoid"
|
||||||
|
return "neutral"
|
||||||
|
|
||||||
|
|
||||||
|
def _role_summary(role: NodeRole) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"compile_weight": role.compile_weight,
|
||||||
|
"cache_writer": role.cache_writer,
|
||||||
|
"cache_reader": role.cache_reader,
|
||||||
|
"distcc_enabled": role.distcc_enabled,
|
||||||
|
"max_thermal_state": role.max_thermal_state,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# Legacy wrapper for api.py facade
|
||||||
|
# -----------------------------
|
||||||
|
def nodes_endpoint(registry=None):
|
||||||
|
"""Legacy entrypoint — returns the same data as GET /api/nodes."""
|
||||||
|
return list_nodes()
|
||||||
|
|
@ -0,0 +1,127 @@
|
||||||
|
"""
|
||||||
|
Pipeline control — retry / force-node / pause / resume / state.
|
||||||
|
|
||||||
|
Wraps an in-memory engine (set by main.py) and emits events on the bus
|
||||||
|
so the UI live stream picks them up.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from typing import Optional, Any
|
||||||
|
|
||||||
|
from backend.api.api import bus
|
||||||
|
from backend.events.schema import EventType
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/pipeline", tags=["pipeline"])
|
||||||
|
|
||||||
|
# In-memory engine reference (set by main.py)
|
||||||
|
_ENGINE: Optional[Any] = None
|
||||||
|
_PIPELINE_STATE: dict = {
|
||||||
|
"state": "idle", # idle | running | paused | complete | failed
|
||||||
|
"active_actions": [],
|
||||||
|
"completed": 0,
|
||||||
|
"failed": 0,
|
||||||
|
"queued": 0,
|
||||||
|
"build_id": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ActionBody(BaseModel):
|
||||||
|
action: str
|
||||||
|
node: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ForceBody(BaseModel):
|
||||||
|
action: str
|
||||||
|
node: str
|
||||||
|
|
||||||
|
|
||||||
|
def set_engine(engine):
|
||||||
|
global _ENGINE
|
||||||
|
_ENGINE = engine
|
||||||
|
|
||||||
|
|
||||||
|
def get_state_snapshot():
|
||||||
|
return dict(_PIPELINE_STATE)
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET STATE
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("/state")
|
||||||
|
def get_state():
|
||||||
|
return get_state_snapshot()
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RETRY ACTION
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/retry")
|
||||||
|
def retry_action(body: ActionBody):
|
||||||
|
bus.emit(
|
||||||
|
EventType.PIPELINE_UPDATE,
|
||||||
|
action=body.action,
|
||||||
|
node=body.node,
|
||||||
|
state="retrying",
|
||||||
|
reason="manual_retry",
|
||||||
|
)
|
||||||
|
# In a real engine this would re-queue the action.
|
||||||
|
# For now we just record the intent.
|
||||||
|
return {"status": "retry_queued", "action": body.action, "node": body.node}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FORCE NODE
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/force-node")
|
||||||
|
def force_node(body: ForceBody):
|
||||||
|
bus.emit(
|
||||||
|
EventType.PIPELINE_UPDATE,
|
||||||
|
action=body.action,
|
||||||
|
node=body.node,
|
||||||
|
state="forced",
|
||||||
|
reason="manual_force",
|
||||||
|
)
|
||||||
|
return {"status": "forced", "action": body.action, "node": body.node}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# PAUSE
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/pause")
|
||||||
|
def pause():
|
||||||
|
_PIPELINE_STATE["state"] = "paused"
|
||||||
|
if _ENGINE and hasattr(_ENGINE, "pause"):
|
||||||
|
_ENGINE.pause()
|
||||||
|
bus.emit(EventType.PIPELINE_UPDATE, state="paused", reason="manual_pause")
|
||||||
|
return {"status": "paused"}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RESUME
|
||||||
|
# -----------------------------
|
||||||
|
@router.post("/resume")
|
||||||
|
def resume():
|
||||||
|
_PIPELINE_STATE["state"] = "running"
|
||||||
|
if _ENGINE and hasattr(_ENGINE, "resume"):
|
||||||
|
_ENGINE.resume()
|
||||||
|
bus.emit(EventType.PIPELINE_UPDATE, state="running", reason="manual_resume")
|
||||||
|
return {"status": "running"}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# Legacy wrapper for api.py facade
|
||||||
|
# -----------------------------
|
||||||
|
def pipeline_control_endpoint(action, payload=None, bus=None):
|
||||||
|
"""Legacy dispatcher — `action` is one of: retry, force, pause, resume, state."""
|
||||||
|
if action == "retry":
|
||||||
|
return retry_action(ActionBody(**(payload or {})))
|
||||||
|
elif action == "force":
|
||||||
|
return force_node(ForceBody(**(payload or {})))
|
||||||
|
elif action == "pause":
|
||||||
|
return pause()
|
||||||
|
elif action == "resume":
|
||||||
|
return resume()
|
||||||
|
elif action == "state":
|
||||||
|
return get_state_snapshot()
|
||||||
|
return {"error": f"unknown action: {action}"}
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
"""
|
||||||
|
Replay API — start/step/seek/reset replay sessions against the timeline journal.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from typing import Optional, Any, Dict
|
||||||
|
|
||||||
|
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||||
|
from backend.session.db import SessionDB
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/replay", tags=["replay"])
|
||||||
|
|
||||||
|
# Process-wide session DB (now SQLite-backed)
|
||||||
|
SESSION_DB = SessionDB()
|
||||||
|
|
||||||
|
|
||||||
|
class StartReplayBody(BaseModel):
|
||||||
|
journal: Optional[str] = None # session id or "latest"
|
||||||
|
session_id: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# START REPLAY SESSION
|
||||||
|
# -------------------------
|
||||||
|
@router.post("/start")
|
||||||
|
def start_replay(body: StartReplayBody):
|
||||||
|
"""Start a new replay session. Snapshots the current live timeline
|
||||||
|
into a new session id, so the session has its own immutable event list."""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
# Create the session first with a placeholder, then patch the source
|
||||||
|
sid = SESSION_DB.create_session(body.journal or "live")
|
||||||
|
|
||||||
|
# Snapshot the current live timeline into the session
|
||||||
|
events = TIMELINE_STORE.all()
|
||||||
|
|
||||||
|
# Log each event into the session so SESSION_DB.get_replay_stream(sid)
|
||||||
|
# returns them (and the autopsy endpoint can find them)
|
||||||
|
for event in events:
|
||||||
|
# Don't double-store — just create a reference event
|
||||||
|
SESSION_DB.log_event(sid, event.get("type", "event"), event)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"session_id": sid,
|
||||||
|
"event_count": len(events),
|
||||||
|
"events_available": len(events),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# GET EVENTS (full journal snapshot)
|
||||||
|
# -------------------------
|
||||||
|
@router.get("/events")
|
||||||
|
def get_events():
|
||||||
|
"""Return the full current journal. Used by the replay UI on load."""
|
||||||
|
return {"events": TIMELINE_STORE.all()}
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# GET EVENTS FOR A SPECIFIC SESSION
|
||||||
|
# -------------------------
|
||||||
|
@router.get("/events/{session_id}")
|
||||||
|
def get_session_events(session_id: str):
|
||||||
|
return {"session_id": session_id, "events": SESSION_DB.get_replay_stream(session_id)}
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# STEP FORWARD
|
||||||
|
# -------------------------
|
||||||
|
@router.post("/step")
|
||||||
|
def replay_step(session_id: str):
|
||||||
|
events = SESSION_DB.get_replay_stream(session_id)
|
||||||
|
return {
|
||||||
|
"session_id": session_id,
|
||||||
|
"event_count": len(events),
|
||||||
|
"events": events[:1],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# SEEK
|
||||||
|
# -------------------------
|
||||||
|
@router.post("/seek")
|
||||||
|
def replay_seek(session_id: str, timestamp: float):
|
||||||
|
events = SESSION_DB.get_replay_stream(session_id)
|
||||||
|
best = None
|
||||||
|
for e in events:
|
||||||
|
ts = e.get("ts") or e.get("timestamp") or 0
|
||||||
|
if ts <= timestamp:
|
||||||
|
best = e
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
return {"event": best, "session_id": session_id}
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# RESET
|
||||||
|
# -------------------------
|
||||||
|
@router.post("/reset")
|
||||||
|
def replay_reset(session_id: str):
|
||||||
|
events = SESSION_DB.get_replay_stream(session_id)
|
||||||
|
return {
|
||||||
|
"session_id": session_id,
|
||||||
|
"event": events[0] if events else None,
|
||||||
|
"event_count": len(events),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# Convenience: replay_endpoint wrapper for api.py facade
|
||||||
|
# -------------------------
|
||||||
|
def replay_endpoint(session_id, registry=None):
|
||||||
|
"""Legacy wrapper — returns the session events for a given id."""
|
||||||
|
return {
|
||||||
|
"session_id": session_id,
|
||||||
|
"events": SESSION_DB.get_replay_stream(session_id),
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
from backend.pipeline.engine import PipelineEngine
|
||||||
|
|
||||||
|
|
||||||
|
def get_state(nodes, engine):
|
||||||
|
return {
|
||||||
|
"pipeline": engine.last_results if engine else [],
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"name": n["name"],
|
||||||
|
"load": n.get("load", 0),
|
||||||
|
"temp": n.get("temp", 0)
|
||||||
|
}
|
||||||
|
for n in nodes
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
"""
|
||||||
|
Timeline API — REST endpoints over the TimelineStore journal.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/timeline", tags=["timeline"])
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET NODE TIMELINE
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("/{node}")
|
||||||
|
def get_node_timeline(node: str):
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"events": TIMELINE_STORE.get_node_events(node),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# REWIND SNAPSHOT
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("/rewind/{index}")
|
||||||
|
def rewind(index: int):
|
||||||
|
return {
|
||||||
|
"index": index,
|
||||||
|
"snapshot": TIMELINE_STORE.snapshot_at(index),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STATS
|
||||||
|
# -----------------------------
|
||||||
|
@router.get("/stats/summary")
|
||||||
|
def stats():
|
||||||
|
return TIMELINE_STORE.stats()
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# Legacy wrapper for api.py facade
|
||||||
|
# -----------------------------
|
||||||
|
def timeline_endpoint(session_id):
|
||||||
|
"""Legacy — return all events for a session id (or live journal if not found)."""
|
||||||
|
return {
|
||||||
|
"session_id": session_id,
|
||||||
|
"events": TIMELINE_STORE.all(),
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
"""
|
||||||
|
WebSocketStream — legacy bridge between the EventBus and WebSocket clients.
|
||||||
|
|
||||||
|
NOTE: The real broadcasting is now done in backend/main.py via
|
||||||
|
`_ws_broadcast` which properly awaits the async send. This class is kept
|
||||||
|
for backward compatibility with the api.py facade, but its `attach_bus`
|
||||||
|
method is a no-op (the actual wiring happens in main.py).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.events.bus import EventBus
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketStream:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.clients = [] # shared with main.py's hub.clients
|
||||||
|
|
||||||
|
def attach_bus(self, bus: EventBus):
|
||||||
|
"""No-op — broadcasting is wired in backend/main.py via _ws_broadcast.
|
||||||
|
Kept for backward compat with api.py's attach_ws_stream()."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
def register_client(self, client):
|
||||||
|
self.clients.append(client)
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from io import StringIO
|
||||||
|
|
||||||
|
from backend.snapshot import create_snapshot
|
||||||
|
from backend.scheduler import build_distcc_hosts
|
||||||
|
from backend.cache_local import make_build_key, cache_hit, cache_store
|
||||||
|
from backend.compiler_env import build_ccache_env
|
||||||
|
|
||||||
|
|
||||||
|
def run_cmd(cmd, cwd, env):
|
||||||
|
p = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
shell=True,
|
||||||
|
cwd=cwd,
|
||||||
|
env=env,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
text=True
|
||||||
|
)
|
||||||
|
|
||||||
|
log = StringIO()
|
||||||
|
|
||||||
|
for line in p.stdout:
|
||||||
|
print(line, end="")
|
||||||
|
log.write(line)
|
||||||
|
|
||||||
|
p.wait()
|
||||||
|
return p.returncode, log.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def build_target(project, snapshot, target, nodes):
|
||||||
|
env = os.environ.copy()
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# distcc layer
|
||||||
|
# ----------------------------
|
||||||
|
env["DISTCC_HOSTS"] = build_distcc_hosts(nodes)
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# ccache layer (NEW)
|
||||||
|
# ----------------------------
|
||||||
|
cc_env = build_ccache_env(project)
|
||||||
|
env.update(cc_env)
|
||||||
|
|
||||||
|
build_env = project.get("build_env", {})
|
||||||
|
env.update(build_env)
|
||||||
|
|
||||||
|
key = make_build_key(
|
||||||
|
snapshot["hash"],
|
||||||
|
target,
|
||||||
|
build_env
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"[FESTER] target={target}")
|
||||||
|
print(f"[FESTER] cache_key={key}")
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# top-level build cache
|
||||||
|
# ----------------------------
|
||||||
|
if cache_hit(key):
|
||||||
|
print("[FESTER] BUILD CACHE HIT (skipping full compile)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
cmd = project["targets"][target]
|
||||||
|
|
||||||
|
code, log = run_cmd(cmd, snapshot["path"], env)
|
||||||
|
|
||||||
|
if code == 0:
|
||||||
|
cache_store(key, log)
|
||||||
|
|
||||||
|
return code
|
||||||
|
|
||||||
|
|
||||||
|
def run_release(project):
|
||||||
|
snapshot = create_snapshot(project["source"])
|
||||||
|
nodes = project.get("nodes_override", []) or []
|
||||||
|
|
||||||
|
results = {}
|
||||||
|
|
||||||
|
for target in project["targets"]:
|
||||||
|
results[target] = build_target(project, snapshot, target, nodes)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import json
|
||||||
|
|
||||||
|
with open(sys.argv[1]) as f:
|
||||||
|
project = json.load(f)
|
||||||
|
|
||||||
|
run_release(project)
|
||||||
|
|
@ -0,0 +1,80 @@
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
# Resolve cache directory with graceful fallbacks for unprivileged environments
|
||||||
|
def _resolve_cache_dir() -> Path:
|
||||||
|
env = os.environ.get("FESTER_CACHE_DIR")
|
||||||
|
if env:
|
||||||
|
p = Path(env)
|
||||||
|
p.mkdir(parents=True, exist_ok=True)
|
||||||
|
return p
|
||||||
|
for c in [
|
||||||
|
Path("/var/lib/fester/cache"),
|
||||||
|
Path.home() / ".fester" / "cache",
|
||||||
|
Path("/tmp/fester_cache"),
|
||||||
|
]:
|
||||||
|
try:
|
||||||
|
c.mkdir(parents=True, exist_ok=True)
|
||||||
|
return c
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
continue
|
||||||
|
# Last-resort in-memory (caller must handle None)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
CACHE_DIR = _resolve_cache_dir()
|
||||||
|
|
||||||
|
|
||||||
|
def hash_dict(d):
|
||||||
|
raw = json.dumps(d, sort_keys=True).encode()
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def make_build_key(snapshot_hash, target, env):
|
||||||
|
return hashlib.sha256(
|
||||||
|
f"{snapshot_hash}:{target}:{hash_dict(env)}".encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def cache_hit(key):
|
||||||
|
if CACHE_DIR is None:
|
||||||
|
return False
|
||||||
|
return (CACHE_DIR / key / "done").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def cache_store(key, log):
|
||||||
|
if CACHE_DIR is None:
|
||||||
|
return False
|
||||||
|
base = CACHE_DIR / key
|
||||||
|
base.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
(base / "done").write_text("1")
|
||||||
|
(base / "log.txt").write_text(log)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def cache_log(key):
|
||||||
|
if CACHE_DIR is None:
|
||||||
|
return None
|
||||||
|
p = CACHE_DIR / key / "log.txt"
|
||||||
|
return p.read_text() if p.exists() else None
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# ccache health probe
|
||||||
|
# ----------------------------
|
||||||
|
def ccache_stats():
|
||||||
|
try:
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
out = subprocess.check_output(
|
||||||
|
["ccache", "-s"],
|
||||||
|
stderr=subprocess.STDOUT
|
||||||
|
).decode()
|
||||||
|
|
||||||
|
return out
|
||||||
|
except:
|
||||||
|
return "ccache not available"
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
import os
|
||||||
|
from backend.cache_local import hash_dict
|
||||||
|
|
||||||
|
|
||||||
|
def build_ccache_env(project):
|
||||||
|
"""
|
||||||
|
Configure compiler caching layer safely.
|
||||||
|
"""
|
||||||
|
|
||||||
|
env = {}
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# enable ccache
|
||||||
|
# ----------------------------
|
||||||
|
env["USE_CCACHE"] = "1"
|
||||||
|
env["CCACHE_DIR"] = "/var/cache/ccache"
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# distcc + ccache integration
|
||||||
|
# critical ordering fix
|
||||||
|
# ----------------------------
|
||||||
|
env["CC"] = "ccache gcc"
|
||||||
|
env["CXX"] = "ccache g++"
|
||||||
|
|
||||||
|
# ensure distcc wraps AFTER ccache
|
||||||
|
env["CCACHE_PREFIX"] = "distcc"
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# tuning for distributed builds
|
||||||
|
# ----------------------------
|
||||||
|
env["CCACHE_COMPRESS"] = "1"
|
||||||
|
env["CCACHE_COMPRESSLEVEL"] = "5"
|
||||||
|
env["CCACHE_MAXSIZE"] = "20G"
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# project-specific salt
|
||||||
|
# ensures correct cache isolation
|
||||||
|
# ----------------------------
|
||||||
|
env["CCACHE_SLOPPINESS"] = "time_macros,locale"
|
||||||
|
|
||||||
|
env["FESTER_CCACHE_KEY"] = hash_dict(project.get("build_env", {}))
|
||||||
|
|
||||||
|
return env
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
"""Configuration loader.
|
||||||
|
|
||||||
|
Reads from the path in FESTER_CONFIG env var if set, otherwise from
|
||||||
|
config.yaml in the project root. Provides CONFIG as a module-level dict.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
|
||||||
|
|
||||||
|
# Allow override via env var (for tests / dev / multi-cluster setups)
|
||||||
|
_env_config = os.environ.get("FESTER_CONFIG")
|
||||||
|
if _env_config:
|
||||||
|
CONFIG_PATH = _env_config
|
||||||
|
else:
|
||||||
|
CONFIG_PATH = os.path.join(BASE_DIR, "config.yaml")
|
||||||
|
|
||||||
|
# Graceful fallback: if the requested config doesn't exist, use a sensible
|
||||||
|
# default so the backend can still boot (e.g. in a fresh clone or container).
|
||||||
|
if not Path(CONFIG_PATH).exists():
|
||||||
|
CONFIG = {
|
||||||
|
"master": {"name": "fester-master", "role": "control"},
|
||||||
|
"nodes": [
|
||||||
|
{"name": "localhost", "host": "127.0.0.1", "max_jobs": 4},
|
||||||
|
],
|
||||||
|
"projects": [],
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
with open(CONFIG_PATH, "r") as f:
|
||||||
|
CONFIG = yaml.safe_load(f) or {}
|
||||||
|
|
||||||
|
# Ensure required keys exist
|
||||||
|
CONFIG.setdefault("master", {"name": "fester-master", "role": "control"})
|
||||||
|
CONFIG.setdefault("nodes", [])
|
||||||
|
CONFIG.setdefault("projects", [])
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
def generate_distcc_hosts(nodes):
|
||||||
|
hosts = []
|
||||||
|
|
||||||
|
for n in nodes:
|
||||||
|
if n.get("type", "physical") == "physical":
|
||||||
|
hosts.append(f"{n['host']}/{n['max_jobs']}")
|
||||||
|
|
||||||
|
return " ".join(hosts)
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
from backend.events.schema import create_event
|
||||||
|
from backend.events.clock import GlobalClock
|
||||||
|
|
||||||
|
|
||||||
|
class EventBus:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.subscribers = []
|
||||||
|
|
||||||
|
def subscribe(self, fn):
|
||||||
|
self.subscribers.append(fn)
|
||||||
|
|
||||||
|
def emit(self, event_type, **kwargs):
|
||||||
|
|
||||||
|
event = create_event(
|
||||||
|
type=event_type,
|
||||||
|
timestamp=GlobalClock.tick(),
|
||||||
|
**kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
# fan-out to all listeners
|
||||||
|
for sub in self.subscribers:
|
||||||
|
sub(event)
|
||||||
|
|
||||||
|
return event
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
|
||||||
|
|
||||||
|
class GlobalClock:
|
||||||
|
"""
|
||||||
|
Deterministic monotonic event clock.
|
||||||
|
Used for replay + ordering across distributed nodes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_lock = threading.Lock()
|
||||||
|
_counter = 0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tick(cls) -> float:
|
||||||
|
with cls._lock:
|
||||||
|
cls._counter += 1
|
||||||
|
return time.time() + (cls._counter * 1e-9)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def reset(cls):
|
||||||
|
with cls._lock:
|
||||||
|
cls._counter = 0
|
||||||
|
|
@ -0,0 +1,74 @@
|
||||||
|
"""
|
||||||
|
Emitter — convenience wrappers around the singleton EventBus.
|
||||||
|
|
||||||
|
Deprecated-ish: prefer using `bus.emit(EventType.X, ...)` directly.
|
||||||
|
These helpers exist for backward compatibility with older callers
|
||||||
|
and for the CLI's `stream` command which expects simple event shapes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.events.bus import EventBus
|
||||||
|
from backend.events.schema import EventType
|
||||||
|
|
||||||
|
# Process-wide singleton bus. Import this instead of constructing your own
|
||||||
|
# unless you really need an isolated bus (e.g. for tests).
|
||||||
|
BUS = EventBus()
|
||||||
|
|
||||||
|
|
||||||
|
async def emit_pipeline(action, state, node, **extra):
|
||||||
|
"""Emit a pipeline/task event. `action` may be a dict (action spec)
|
||||||
|
or a string (action name)."""
|
||||||
|
action_name = action["name"] if isinstance(action, dict) else action
|
||||||
|
node_name = node["name"] if isinstance(node, dict) else node
|
||||||
|
BUS.emit(
|
||||||
|
EventType.TASK_UPDATE,
|
||||||
|
action=action_name,
|
||||||
|
node=node_name,
|
||||||
|
state=state,
|
||||||
|
**extra,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def emit_node(node, **extra):
|
||||||
|
"""Emit a node state update."""
|
||||||
|
node_name = node["name"] if isinstance(node, dict) else node
|
||||||
|
BUS.emit(
|
||||||
|
EventType.NODE_UPDATE,
|
||||||
|
node=node_name,
|
||||||
|
**extra,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def emit_cache(action, state, node=None, **extra):
|
||||||
|
"""Emit a cache hit/miss event."""
|
||||||
|
action_name = action["name"] if isinstance(action, dict) else action
|
||||||
|
BUS.emit(
|
||||||
|
EventType.CACHE_UPDATE,
|
||||||
|
action=action_name,
|
||||||
|
node=node,
|
||||||
|
state=state,
|
||||||
|
**extra,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def emit_failure(action, reason, node=None, **extra):
|
||||||
|
"""Emit a failure event."""
|
||||||
|
action_name = action["name"] if isinstance(action, dict) else action
|
||||||
|
BUS.emit(
|
||||||
|
EventType.FAILURE,
|
||||||
|
action=action_name,
|
||||||
|
node=node,
|
||||||
|
state="failed",
|
||||||
|
reason=reason,
|
||||||
|
**extra,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def emit_policy(key, value):
|
||||||
|
"""Emit a policy change event."""
|
||||||
|
BUS.emit(
|
||||||
|
EventType.PIPELINE_UPDATE, # closest existing type
|
||||||
|
action=None,
|
||||||
|
node=None,
|
||||||
|
state="policy",
|
||||||
|
meta={"key": key, "value": value},
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
# events/failure_stream.py
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
class FailureStream:
|
||||||
|
|
||||||
|
def __init__(self, ws_broadcaster, propagation_engine):
|
||||||
|
self.ws = ws_broadcaster
|
||||||
|
self.propagation = propagation_engine
|
||||||
|
|
||||||
|
async def emit_failure(self, failed_node):
|
||||||
|
|
||||||
|
map_result = self.propagation.map_failure(failed_node)
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"type": "failure_propagation",
|
||||||
|
"data": map_result
|
||||||
|
}
|
||||||
|
|
||||||
|
await self.ws.broadcast(json.dumps(payload))
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
|
||||||
|
class EventJournal:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# immutable append-only log
|
||||||
|
self.events = []
|
||||||
|
|
||||||
|
# indexed views for fast debug queries
|
||||||
|
self.by_action = defaultdict(list)
|
||||||
|
self.by_node = defaultdict(list)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# WRITE EVENT
|
||||||
|
# -----------------------------
|
||||||
|
def record(self, event_type, payload):
|
||||||
|
|
||||||
|
event = {
|
||||||
|
"ts": time.time(),
|
||||||
|
"type": event_type,
|
||||||
|
"data": payload
|
||||||
|
}
|
||||||
|
|
||||||
|
self.events.append(event)
|
||||||
|
|
||||||
|
if "action" in payload:
|
||||||
|
self.by_action[payload["action"]].append(event)
|
||||||
|
|
||||||
|
if "node" in payload:
|
||||||
|
self.by_node[payload["node"]].append(event)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# QUERY ACTION TRACE
|
||||||
|
# -----------------------------
|
||||||
|
def trace_action(self, action_name):
|
||||||
|
return self.by_action.get(action_name, [])
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# QUERY NODE TRACE
|
||||||
|
# -----------------------------
|
||||||
|
def trace_node(self, node_name):
|
||||||
|
return self.by_node.get(node_name, [])
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FULL REPLAY STREAM
|
||||||
|
# -----------------------------
|
||||||
|
def replay(self):
|
||||||
|
for event in sorted(self.events, key=lambda e: e["ts"]):
|
||||||
|
yield event
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
import bisect
|
||||||
|
|
||||||
|
|
||||||
|
class ReplayController:
|
||||||
|
|
||||||
|
def __init__(self, journal):
|
||||||
|
|
||||||
|
self.journal = journal
|
||||||
|
self.index = 0
|
||||||
|
self.playing = False
|
||||||
|
|
||||||
|
# pre-sorted events for deterministic replay
|
||||||
|
self.timeline = sorted(
|
||||||
|
journal.events,
|
||||||
|
key=lambda e: e["ts"]
|
||||||
|
)
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# CURRENT EVENT
|
||||||
|
# -------------------------
|
||||||
|
def current(self):
|
||||||
|
|
||||||
|
if 0 <= self.index < len(self.timeline):
|
||||||
|
return self.timeline[self.index]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# STEP FORWARD
|
||||||
|
# -------------------------
|
||||||
|
def step(self):
|
||||||
|
|
||||||
|
if self.index < len(self.timeline) - 1:
|
||||||
|
self.index += 1
|
||||||
|
|
||||||
|
return self.current()
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# STEP BACKWARD
|
||||||
|
# -------------------------
|
||||||
|
def back(self):
|
||||||
|
|
||||||
|
if self.index > 0:
|
||||||
|
self.index -= 1
|
||||||
|
|
||||||
|
return self.current()
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# SEEK BY TIMESTAMP
|
||||||
|
# -------------------------
|
||||||
|
def seek(self, timestamp):
|
||||||
|
|
||||||
|
times = [e["ts"] for e in self.timeline]
|
||||||
|
|
||||||
|
self.index = bisect.bisect_left(times, timestamp)
|
||||||
|
|
||||||
|
return self.current()
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# RESET
|
||||||
|
# -------------------------
|
||||||
|
def reset(self):
|
||||||
|
|
||||||
|
self.index = 0
|
||||||
|
|
||||||
|
return self.current()
|
||||||
|
|
||||||
|
# -------------------------
|
||||||
|
# PLAY (ITERATOR MODE)
|
||||||
|
# -------------------------
|
||||||
|
def play(self):
|
||||||
|
|
||||||
|
for i in range(self.index, len(self.timeline)):
|
||||||
|
self.index = i
|
||||||
|
yield self.timeline[i]
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
from enum import Enum
|
||||||
|
from dataclasses import dataclass, asdict
|
||||||
|
from typing import Optional, Dict, Any
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
class EventType(str, Enum):
|
||||||
|
NODE_UPDATE = "node_update"
|
||||||
|
TASK_UPDATE = "task_update"
|
||||||
|
PIPELINE_UPDATE = "pipeline_update"
|
||||||
|
CACHE_UPDATE = "cache_update"
|
||||||
|
FAILURE = "failure"
|
||||||
|
DEBUG = "debug"
|
||||||
|
REPLAY = "replay"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FesterEvent:
|
||||||
|
"""
|
||||||
|
Canonical event used everywhere in the system.
|
||||||
|
"""
|
||||||
|
|
||||||
|
type: EventType
|
||||||
|
timestamp: float
|
||||||
|
node: Optional[str] = None
|
||||||
|
action: Optional[str] = None
|
||||||
|
state: Optional[str] = None
|
||||||
|
|
||||||
|
# scheduling + explainability
|
||||||
|
score: Optional[float] = None
|
||||||
|
reason: Optional[str] = None
|
||||||
|
|
||||||
|
# DAG / execution graph
|
||||||
|
parent: Optional[str] = None
|
||||||
|
target: Optional[str] = None
|
||||||
|
|
||||||
|
# extensibility (NEVER bypass schema, extend here)
|
||||||
|
meta: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
|
||||||
|
def create_event(**kwargs) -> dict:
|
||||||
|
"""
|
||||||
|
Always use this instead of raw dict events.
|
||||||
|
Guarantees consistency across system.
|
||||||
|
"""
|
||||||
|
if "timestamp" not in kwargs:
|
||||||
|
kwargs["timestamp"] = time.time()
|
||||||
|
|
||||||
|
return asdict(FesterEvent(**kwargs))
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
class EventStream:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
|
||||||
|
self.subscribers = set()
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# SUBSCRIBE (WebSocket clients)
|
||||||
|
# -----------------------------
|
||||||
|
def subscribe(self, ws):
|
||||||
|
|
||||||
|
self.subscribers.add(ws)
|
||||||
|
|
||||||
|
def unsubscribe(self, ws):
|
||||||
|
|
||||||
|
self.subscribers.discard(ws)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# EMIT EVENT (REALTIME FANOUT)
|
||||||
|
# -----------------------------
|
||||||
|
async def emit(self, event_type, payload):
|
||||||
|
|
||||||
|
message = json.dumps({
|
||||||
|
"type": event_type,
|
||||||
|
"data": payload
|
||||||
|
})
|
||||||
|
|
||||||
|
dead = set()
|
||||||
|
|
||||||
|
for ws in self.subscribers:
|
||||||
|
|
||||||
|
try:
|
||||||
|
await ws.send(message)
|
||||||
|
except Exception:
|
||||||
|
dead.add(ws)
|
||||||
|
|
||||||
|
for ws in dead:
|
||||||
|
self.subscribers.discard(ws)
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
"""
|
||||||
|
Event type constants — DEPRECATED.
|
||||||
|
|
||||||
|
Use `backend.events.schema.EventType` instead. These string constants
|
||||||
|
are kept only for backward compatibility with older emitters.
|
||||||
|
|
||||||
|
The string values intentionally match the EventType enum values so
|
||||||
|
legacy code comparing `event["type"] == "pipeline"` continues to work
|
||||||
|
with both old and new emitters.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.events.schema import EventType
|
||||||
|
|
||||||
|
# Legacy aliases — values match EventType enum values
|
||||||
|
EVENT_PIPELINE = EventType.PIPELINE_UPDATE.value # "pipeline_update"
|
||||||
|
EVENT_NODE = EventType.NODE_UPDATE.value # "node_update"
|
||||||
|
EVENT_POLICY = "policy" # not in EventType; meta-only
|
||||||
|
EVENT_SESSION = "session" # not in EventType; meta-only
|
||||||
|
|
||||||
|
# Also expose the simple short names the UI/CLI use
|
||||||
|
EVENT_PIPELINE_SHORT = "pipeline"
|
||||||
|
EVENT_NODE_SHORT = "node"
|
||||||
|
EVENT_FAILURE_SHORT = "failure"
|
||||||
|
EVENT_CACHE_SHORT = "cache"
|
||||||
|
EVENT_DEBUG_SHORT = "debug"
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"EventType",
|
||||||
|
"EVENT_PIPELINE",
|
||||||
|
"EVENT_NODE",
|
||||||
|
"EVENT_POLICY",
|
||||||
|
"EVENT_SESSION",
|
||||||
|
"EVENT_PIPELINE_SHORT",
|
||||||
|
"EVENT_NODE_SHORT",
|
||||||
|
"EVENT_FAILURE_SHORT",
|
||||||
|
"EVENT_CACHE_SHORT",
|
||||||
|
"EVENT_DEBUG_SHORT",
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
from backend.events.bus import EventBus
|
||||||
|
|
||||||
|
|
||||||
|
def run_action(action, node, bus: EventBus):
|
||||||
|
|
||||||
|
bus.emit(
|
||||||
|
"task_update",
|
||||||
|
node=node["name"],
|
||||||
|
action=action["name"],
|
||||||
|
state="running"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# placeholder execution logic
|
||||||
|
result = action["cmd"]()
|
||||||
|
|
||||||
|
bus.emit(
|
||||||
|
"task_update",
|
||||||
|
node=node["name"],
|
||||||
|
action=action["name"],
|
||||||
|
state="done"
|
||||||
|
)
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
|
||||||
|
bus.emit(
|
||||||
|
"failure",
|
||||||
|
node=node["name"],
|
||||||
|
action=action["name"],
|
||||||
|
state="failed",
|
||||||
|
reason=str(e)
|
||||||
|
)
|
||||||
|
|
||||||
|
return 1
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def run_host(command, cwd, env):
|
||||||
|
return subprocess.run(command, shell=True, cwd=cwd, env=env).returncode
|
||||||
|
|
||||||
|
|
||||||
|
def run_lxc(container, command):
|
||||||
|
cmd = f"lxc exec {container} -- bash -lc '{command}'"
|
||||||
|
return subprocess.run(cmd, shell=True).returncode
|
||||||
|
|
||||||
|
|
||||||
|
def run_libvirt(vm_name, command):
|
||||||
|
cmd = f"virsh domfsfreeze {vm_name} && virsh send-process-signal {vm_name} 15"
|
||||||
|
# simplified placeholder — real execution would use ssh/agent channel
|
||||||
|
return subprocess.run(cmd, shell=True).returncode
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def execute(action, node, spec):
|
||||||
|
|
||||||
|
exec_type = spec.execution
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# DISTCC PATH
|
||||||
|
# -----------------------------
|
||||||
|
if exec_type == "distcc":
|
||||||
|
return run_distcc(action, node, spec)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# LXC PATH
|
||||||
|
# -----------------------------
|
||||||
|
if exec_type == "lxc":
|
||||||
|
return run_lxc(action, node, spec)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# LIBVIRT PATH
|
||||||
|
# -----------------------------
|
||||||
|
if exec_type == "libvirt":
|
||||||
|
return run_libvirt(action, node, spec)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# TMUX LOCAL PATH
|
||||||
|
# -----------------------------
|
||||||
|
if exec_type == "tmux":
|
||||||
|
return run_tmux(action, node, spec)
|
||||||
|
|
||||||
|
raise ValueError(f"Unknown execution type: {exec_type}")
|
||||||
|
|
||||||
|
|
||||||
|
def run_distcc(action, node, spec):
|
||||||
|
return subprocess.call(["distcc", action["cmd"]])
|
||||||
|
|
||||||
|
|
||||||
|
def run_lxc(action, node, spec):
|
||||||
|
return subprocess.call(["lxc", "exec", node["container"], "--", action["cmd"]])
|
||||||
|
|
||||||
|
|
||||||
|
def run_libvirt(action, node, spec):
|
||||||
|
return subprocess.call(["virsh", "qemu-agent-command", node["vm"], action["cmd"]])
|
||||||
|
|
||||||
|
|
||||||
|
def run_tmux(action, node, spec):
|
||||||
|
return subprocess.call(["tmux", "new", "-d", action["cmd"]])
|
||||||
|
|
@ -0,0 +1,121 @@
|
||||||
|
"""
|
||||||
|
Runtime router — picks execution backend + dispatches the action.
|
||||||
|
|
||||||
|
Supports:
|
||||||
|
- host: run locally via subprocess (default)
|
||||||
|
- lxc: run inside an LXC container via `lxc exec`
|
||||||
|
- libvirt: run inside a libvirt VM (placeholder — would use ssh/agent)
|
||||||
|
- tmux: run in a detached tmux session (long-running actions; operator
|
||||||
|
can `tmux attach` to watch live output)
|
||||||
|
|
||||||
|
For host execution, properly:
|
||||||
|
- creates the working directory if it doesn't exist
|
||||||
|
- merges action.env with the current process env (so PATH etc survive)
|
||||||
|
- returns the subprocess exit code
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
from backend.executor.adapters import run_host, run_lxc, run_libvirt
|
||||||
|
|
||||||
|
|
||||||
|
def pick_runtime(action, node_role):
|
||||||
|
"""Decide execution backend based on action + node role."""
|
||||||
|
runtime = action.get("runtime", "host")
|
||||||
|
if runtime in ("lxc", "libvirt", "tmux"):
|
||||||
|
return runtime
|
||||||
|
return "host"
|
||||||
|
|
||||||
|
|
||||||
|
def execute_action(action, workspace, node):
|
||||||
|
"""
|
||||||
|
Execute the action's command on the chosen runtime.
|
||||||
|
|
||||||
|
`workspace` may be:
|
||||||
|
- a string path (used as cwd)
|
||||||
|
- a dict (legacy — `{"cwd": "..."}` or empty; falls back to action.dir or /tmp)
|
||||||
|
- None (falls back to action.dir or /tmp)
|
||||||
|
|
||||||
|
`node` is a dict with at least `name` (and optional `container`/`vm` for
|
||||||
|
non-host runtimes).
|
||||||
|
|
||||||
|
Returns the subprocess exit code (0 = success).
|
||||||
|
"""
|
||||||
|
runtime = pick_runtime(action, node.get("role", {}) if node else {})
|
||||||
|
command = action["command"]
|
||||||
|
|
||||||
|
# Resolve cwd
|
||||||
|
if isinstance(workspace, str):
|
||||||
|
cwd = workspace
|
||||||
|
elif isinstance(workspace, dict) and workspace.get("cwd"):
|
||||||
|
cwd = workspace["cwd"]
|
||||||
|
elif action.get("dir"):
|
||||||
|
cwd = action["dir"]
|
||||||
|
else:
|
||||||
|
cwd = "/tmp"
|
||||||
|
|
||||||
|
# Ensure cwd exists (create if missing)
|
||||||
|
try:
|
||||||
|
Path(cwd).mkdir(parents=True, exist_ok=True)
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
cwd = None # fall back to current dir
|
||||||
|
|
||||||
|
# Build env: start with os.environ, merge action.env on top
|
||||||
|
env = dict(os.environ)
|
||||||
|
action_env = action.get("env") or {}
|
||||||
|
if isinstance(action_env, dict):
|
||||||
|
env.update({k: str(v) for k, v in action_env.items()})
|
||||||
|
|
||||||
|
if runtime == "lxc":
|
||||||
|
container = (node or {}).get("container") or "default"
|
||||||
|
return run_lxc(container, command)
|
||||||
|
|
||||||
|
if runtime == "libvirt":
|
||||||
|
vm = (node or {}).get("vm") or "default"
|
||||||
|
return run_libvirt(vm, command)
|
||||||
|
|
||||||
|
if runtime == "tmux":
|
||||||
|
return _run_in_tmux(action, command, cwd, env)
|
||||||
|
|
||||||
|
# host runtime (default)
|
||||||
|
return run_host(command, cwd, env)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_in_tmux(action: Dict[str, Any], command: str,
|
||||||
|
cwd: Optional[str], env: Dict[str, str]) -> int:
|
||||||
|
"""Run the action in a detached tmux session.
|
||||||
|
|
||||||
|
Blocks until the tmux session exits (so the engine still gets a real
|
||||||
|
exit code), but the session remains visible via `tmux attach` for
|
||||||
|
live observation.
|
||||||
|
|
||||||
|
Returns the exit code (0 = success)."""
|
||||||
|
from backend.integrations.tmux import TmuxManager
|
||||||
|
|
||||||
|
mgr = TmuxManager()
|
||||||
|
if not mgr.available:
|
||||||
|
# tmux not installed — fall back to host execution
|
||||||
|
from backend.executor.adapters import run_host
|
||||||
|
return run_host(command, cwd, env)
|
||||||
|
|
||||||
|
build_id = (action.get("meta", {}) or {}).get("build_id") if isinstance(action.get("meta"), dict) else None
|
||||||
|
name = mgr.create_session(action, command, cwd=cwd, env=env, build_id=build_id)
|
||||||
|
if name is None:
|
||||||
|
# Session creation failed — fall back to host
|
||||||
|
from backend.executor.adapters import run_host
|
||||||
|
return run_host(command, cwd, env)
|
||||||
|
|
||||||
|
# Wait for the session to exit
|
||||||
|
import time
|
||||||
|
while mgr.session_exists(name):
|
||||||
|
time.sleep(0.5)
|
||||||
|
|
||||||
|
# Capture the exit code from the last line of pane output
|
||||||
|
output = mgr.capture_output(name, lines=5) or ""
|
||||||
|
# Look for "exit code N" or similar; default to 0 if we can't tell
|
||||||
|
# (tmux doesn't expose exit codes directly; we'd need to wrap the command)
|
||||||
|
# For now, return 0 if the session completed cleanly
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
def thermal_cap(agent):
|
||||||
|
if not agent:
|
||||||
|
return 0.3
|
||||||
|
|
||||||
|
temp = agent.get("temp", "")
|
||||||
|
load = agent.get("load", "1 1 1")
|
||||||
|
|
||||||
|
try:
|
||||||
|
cpu = float(load.split()[0])
|
||||||
|
except:
|
||||||
|
cpu = 1.0
|
||||||
|
|
||||||
|
# safe X99 operating envelope
|
||||||
|
if "90" in temp:
|
||||||
|
return 0.2
|
||||||
|
|
||||||
|
if "80" in temp:
|
||||||
|
return 0.5
|
||||||
|
|
||||||
|
if cpu > 4:
|
||||||
|
return 0.6
|
||||||
|
|
||||||
|
return 1.0
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
def hash_action(action):
|
||||||
|
raw = json.dumps(action, sort_keys=True).encode()
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def create_action(name, inputs, command, outputs,
|
||||||
|
env=None, runtime="host"):
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"inputs": inputs,
|
||||||
|
"command": command,
|
||||||
|
"outputs": outputs,
|
||||||
|
"env": env or {},
|
||||||
|
"runtime": runtime
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,106 @@
|
||||||
|
class BuildNode:
|
||||||
|
|
||||||
|
def __init__(self, name, cmd, deps=None, cache_key=None):
|
||||||
|
|
||||||
|
self.name = name
|
||||||
|
self.cmd = cmd
|
||||||
|
self.deps = deps or []
|
||||||
|
self.cache_key = cache_key
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f"<BuildNode {self.name}>"
|
||||||
|
|
||||||
|
|
||||||
|
class BuildGraph:
|
||||||
|
|
||||||
|
def __init__(self, spec):
|
||||||
|
|
||||||
|
self.spec = spec
|
||||||
|
self.nodes = []
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# ENTRY POINT
|
||||||
|
# -----------------------------
|
||||||
|
def generate(self):
|
||||||
|
|
||||||
|
self._fetch_source()
|
||||||
|
self._resolve_toolchain()
|
||||||
|
self._compile_units()
|
||||||
|
self._link_stage()
|
||||||
|
self._package()
|
||||||
|
self._cache()
|
||||||
|
|
||||||
|
return self.nodes
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STAGES
|
||||||
|
# -----------------------------
|
||||||
|
def _fetch_source(self):
|
||||||
|
|
||||||
|
self.nodes.append(
|
||||||
|
BuildNode(
|
||||||
|
name="fetch_source",
|
||||||
|
cmd="git clone repo",
|
||||||
|
cache_key="src"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _resolve_toolchain(self):
|
||||||
|
|
||||||
|
cmd = f"setup-toolchain --arch {self.spec.arch} --target {self.spec.target}"
|
||||||
|
|
||||||
|
self.nodes.append(
|
||||||
|
BuildNode(
|
||||||
|
name="resolve_toolchain",
|
||||||
|
cmd=cmd,
|
||||||
|
deps=["fetch_source"],
|
||||||
|
cache_key="toolchain"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _compile_units(self):
|
||||||
|
|
||||||
|
# simplified fan-out model (real version would parse AST / Makefile / Ninja)
|
||||||
|
for i in range(3):
|
||||||
|
|
||||||
|
self.nodes.append(
|
||||||
|
BuildNode(
|
||||||
|
name=f"compile_unit_{i}",
|
||||||
|
cmd=f"{self.spec.toolchain} -c file_{i}.c",
|
||||||
|
deps=["resolve_toolchain"],
|
||||||
|
cache_key=f"obj_{i}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _link_stage(self):
|
||||||
|
|
||||||
|
self.nodes.append(
|
||||||
|
BuildNode(
|
||||||
|
name="link",
|
||||||
|
cmd=f"{self.spec.toolchain} link *.o -o output.elf",
|
||||||
|
deps=["compile_unit_0", "compile_unit_1", "compile_unit_2"],
|
||||||
|
cache_key="binary"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _package(self):
|
||||||
|
|
||||||
|
self.nodes.append(
|
||||||
|
BuildNode(
|
||||||
|
name="package",
|
||||||
|
cmd=f"package --format {self.spec.output_mode}",
|
||||||
|
deps=["link"],
|
||||||
|
cache_key="artifact"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _cache(self):
|
||||||
|
|
||||||
|
self.nodes.append(
|
||||||
|
BuildNode(
|
||||||
|
name="cache_upload",
|
||||||
|
cmd="upload-cache",
|
||||||
|
deps=["package"],
|
||||||
|
cache_key="cache"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
# graph/critical_path.py
|
||||||
|
|
||||||
|
def compute_critical_path(actions):
|
||||||
|
"""
|
||||||
|
Computes the longest dependency-weighted path in a build DAG.
|
||||||
|
|
||||||
|
This represents the true bottleneck chain ("critical path")
|
||||||
|
that determines total build time.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# Build adjacency + cost map
|
||||||
|
# -----------------------------
|
||||||
|
graph = {}
|
||||||
|
duration = {}
|
||||||
|
|
||||||
|
for action in actions:
|
||||||
|
name = action["name"]
|
||||||
|
deps = action.get("deps", [])
|
||||||
|
|
||||||
|
graph[name] = deps
|
||||||
|
|
||||||
|
# fallback cost model (later replace with real timing data)
|
||||||
|
duration[name] = action.get("cost", 1)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# memoized DFS for longest path
|
||||||
|
# -----------------------------
|
||||||
|
memo = {}
|
||||||
|
|
||||||
|
def dfs(node):
|
||||||
|
if node in memo:
|
||||||
|
return memo[node]
|
||||||
|
|
||||||
|
# leaf node
|
||||||
|
if node not in graph or not graph[node]:
|
||||||
|
memo[node] = duration.get(node, 1)
|
||||||
|
return memo[node]
|
||||||
|
|
||||||
|
best_dep = 0
|
||||||
|
|
||||||
|
for dep in graph[node]:
|
||||||
|
best_dep = max(best_dep, dfs(dep))
|
||||||
|
|
||||||
|
memo[node] = best_dep + duration.get(node, 1)
|
||||||
|
return memo[node]
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# find worst (critical) endpoint
|
||||||
|
# -----------------------------
|
||||||
|
critical_node = None
|
||||||
|
critical_score = -1
|
||||||
|
|
||||||
|
for node in graph.keys():
|
||||||
|
score = dfs(node)
|
||||||
|
if score > critical_score:
|
||||||
|
critical_score = score
|
||||||
|
critical_node = node
|
||||||
|
|
||||||
|
return {
|
||||||
|
"critical_node": critical_node,
|
||||||
|
"critical_score": critical_score,
|
||||||
|
"score_map": memo
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,123 @@
|
||||||
|
"""
|
||||||
|
Build action graph — convert a project spec into a DAG of executable actions.
|
||||||
|
|
||||||
|
Supports two input shapes:
|
||||||
|
1. Legacy: project = {"targets": {"name": "cmd", ...}}
|
||||||
|
→ flat list of independent actions
|
||||||
|
2. Rich: project = {"stages": [{"name": "...", "command": "...",
|
||||||
|
"deps": [...], "target": "..."}, ...]}
|
||||||
|
→ real DAG with dependency edges
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.graph.actions import hash_action
|
||||||
|
|
||||||
|
|
||||||
|
def build_action_graph(project):
|
||||||
|
"""
|
||||||
|
Convert a project spec into a list of action dicts.
|
||||||
|
|
||||||
|
Each action has:
|
||||||
|
- name: unique action name
|
||||||
|
- command: shell command to run
|
||||||
|
- inputs: list of input globs
|
||||||
|
- outputs: list of output paths
|
||||||
|
- deps: list of action names this depends on
|
||||||
|
- env: environment dict
|
||||||
|
- runtime: "host" | "lxc" | "libvirt"
|
||||||
|
- target: build target name (e.g. "x86_64-linux-gnu")
|
||||||
|
- hash: sha256 of the action dict (for cache keys)
|
||||||
|
"""
|
||||||
|
graph = []
|
||||||
|
|
||||||
|
if "stages" in project:
|
||||||
|
# Rich DAG input
|
||||||
|
build_dir = project.get("build_dir")
|
||||||
|
for stage in project["stages"]:
|
||||||
|
action = {
|
||||||
|
"name": stage["name"],
|
||||||
|
"command": stage.get("command", "true"),
|
||||||
|
"inputs": stage.get("inputs", ["src/**"]),
|
||||||
|
"outputs": stage.get("outputs", [f"build/{stage['name']}.out"]),
|
||||||
|
"deps": stage.get("deps", []),
|
||||||
|
"env": stage.get("env", project.get("build_env", {})),
|
||||||
|
"runtime": stage.get("runtime", "host"),
|
||||||
|
"target": stage.get("target", project.get("default_target", "native")),
|
||||||
|
"cost": stage.get("cost", 1),
|
||||||
|
"dir": stage.get("dir", build_dir),
|
||||||
|
}
|
||||||
|
action["hash"] = hash_action(action)
|
||||||
|
graph.append(action)
|
||||||
|
return graph
|
||||||
|
|
||||||
|
if "targets" in project:
|
||||||
|
# Legacy flat input — synthesize a minimal pipeline:
|
||||||
|
# fetch_source → configure → per-target build → package
|
||||||
|
default_target = project.get("default_target", "native")
|
||||||
|
build_env = project.get("build_env", {})
|
||||||
|
build_dir = project.get("build_dir")
|
||||||
|
|
||||||
|
# Always include a fetch + configure stage
|
||||||
|
graph.append(_make_action(
|
||||||
|
name="fetch_source",
|
||||||
|
command=f"git clone {project.get('source', project.get('repo', '.'))} .",
|
||||||
|
deps=[],
|
||||||
|
target=default_target,
|
||||||
|
env=build_env,
|
||||||
|
cost=2,
|
||||||
|
dir=build_dir,
|
||||||
|
))
|
||||||
|
graph.append(_make_action(
|
||||||
|
name="configure",
|
||||||
|
command="./configure",
|
||||||
|
deps=["fetch_source"],
|
||||||
|
target=default_target,
|
||||||
|
env=build_env,
|
||||||
|
cost=1,
|
||||||
|
dir=build_dir,
|
||||||
|
))
|
||||||
|
|
||||||
|
# Per-target build actions (all depend on configure)
|
||||||
|
for target_name, cmd in project["targets"].items():
|
||||||
|
graph.append(_make_action(
|
||||||
|
name=f"build_{target_name}",
|
||||||
|
command=cmd,
|
||||||
|
deps=["configure"],
|
||||||
|
target=target_name,
|
||||||
|
env=build_env,
|
||||||
|
cost=3,
|
||||||
|
dir=build_dir,
|
||||||
|
))
|
||||||
|
|
||||||
|
# Package stage depends on all builds
|
||||||
|
build_names = [f"build_{t}" for t in project["targets"].keys()]
|
||||||
|
graph.append(_make_action(
|
||||||
|
name="package",
|
||||||
|
command="tar czf build.tar.gz build/",
|
||||||
|
deps=build_names,
|
||||||
|
target=default_target,
|
||||||
|
env=build_env,
|
||||||
|
cost=2,
|
||||||
|
dir=build_dir,
|
||||||
|
))
|
||||||
|
|
||||||
|
return graph
|
||||||
|
|
||||||
|
# Fallback: empty graph
|
||||||
|
return graph
|
||||||
|
|
||||||
|
|
||||||
|
def _make_action(name, command, deps, target, env, cost=1, dir=None):
|
||||||
|
action = {
|
||||||
|
"name": name,
|
||||||
|
"command": command,
|
||||||
|
"inputs": ["src/**"],
|
||||||
|
"outputs": [f"build/{name}.out"],
|
||||||
|
"deps": deps,
|
||||||
|
"env": env,
|
||||||
|
"runtime": "host",
|
||||||
|
"target": target,
|
||||||
|
"cost": cost,
|
||||||
|
"dir": dir, # working directory for the command
|
||||||
|
}
|
||||||
|
action["hash"] = hash_action(action)
|
||||||
|
return action
|
||||||
|
|
@ -0,0 +1,60 @@
|
||||||
|
def build_target_dag(project, targets):
|
||||||
|
|
||||||
|
dag = []
|
||||||
|
|
||||||
|
for target in targets:
|
||||||
|
|
||||||
|
dag.append({
|
||||||
|
"name": f"{target.system}-{target.arch}",
|
||||||
|
"target": target,
|
||||||
|
"steps": generate_steps(project, target)
|
||||||
|
})
|
||||||
|
|
||||||
|
return dag
|
||||||
|
|
||||||
|
|
||||||
|
def generate_steps(project, target):
|
||||||
|
|
||||||
|
steps = []
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# SYSTEM-SPECIFIC BUILD LOGIC
|
||||||
|
# -----------------------------
|
||||||
|
|
||||||
|
if target.system == "gentoo":
|
||||||
|
steps.append({
|
||||||
|
"name": "emerge",
|
||||||
|
"command": f"emerge -e {project['name']}"
|
||||||
|
})
|
||||||
|
|
||||||
|
elif target.system == "buildroot":
|
||||||
|
steps.append({
|
||||||
|
"name": "buildroot",
|
||||||
|
"command": "make"
|
||||||
|
})
|
||||||
|
|
||||||
|
elif target.system == "openwrt":
|
||||||
|
steps.append({
|
||||||
|
"name": "openwrt",
|
||||||
|
"command": "make world"
|
||||||
|
})
|
||||||
|
|
||||||
|
elif target.system == "alfs":
|
||||||
|
steps.append({
|
||||||
|
"name": "alfs",
|
||||||
|
"command": "./build-lfs.sh"
|
||||||
|
})
|
||||||
|
|
||||||
|
elif target.system == "sourcemage":
|
||||||
|
steps.append({
|
||||||
|
"name": "sorcery",
|
||||||
|
"command": "cast " + project["name"]
|
||||||
|
})
|
||||||
|
|
||||||
|
elif target.system == "lunar":
|
||||||
|
steps.append({
|
||||||
|
"name": "lunar",
|
||||||
|
"command": "lunar build " + project["name"]
|
||||||
|
})
|
||||||
|
|
||||||
|
return steps
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
"""
|
||||||
|
Forgejo integration — webhook handler for push events.
|
||||||
|
|
||||||
|
Kicks off a build when a push event arrives. The actual webhook endpoint
|
||||||
|
should live in main.py (or a dedicated webhooks router); this function
|
||||||
|
encapsulates the "convert push → BuildBody → start build" logic.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def on_push_event(project: str, repo: str, ref: str = "main",
|
||||||
|
runner=None) -> Dict[str, Any]:
|
||||||
|
"""Handle a Forgejo push webhook.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
project: project name (e.g. "linux-tool")
|
||||||
|
repo: full clone URL
|
||||||
|
ref: git ref that was pushed (e.g. "main", "refs/tags/v1.0")
|
||||||
|
runner: BuildRunner instance (if None, returns the BuildBody
|
||||||
|
that would have been dispatched)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict with build_id + status, OR the BuildBody if no runner.
|
||||||
|
"""
|
||||||
|
# Build a project spec for the engine
|
||||||
|
body = {
|
||||||
|
"cmd": f"git fetch && git checkout {ref} && make -j$(nproc)",
|
||||||
|
"dir": f"/home/user/builds/{project}",
|
||||||
|
"project": project,
|
||||||
|
"arch": "x86_64",
|
||||||
|
"target": "linux-gnu",
|
||||||
|
"toolchain": "gcc",
|
||||||
|
}
|
||||||
|
|
||||||
|
if runner is None:
|
||||||
|
return {"status": "no_runner", "body": body}
|
||||||
|
|
||||||
|
# Dispatch through the runner (async, but we can't await from sync)
|
||||||
|
import asyncio
|
||||||
|
try:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
except RuntimeError:
|
||||||
|
loop = None
|
||||||
|
|
||||||
|
if loop:
|
||||||
|
# We're inside an async context — schedule the build
|
||||||
|
task = loop.create_task(runner.start(_BuildBodyShim(**body)))
|
||||||
|
return {"status": "scheduled", "body": body}
|
||||||
|
else:
|
||||||
|
# Sync context — run in a new loop (blocks)
|
||||||
|
async def _start():
|
||||||
|
return await runner.start(_BuildBodyShim(**body))
|
||||||
|
build_id = asyncio.run(_start())
|
||||||
|
return {"status": "started", "build_id": build_id, "body": body}
|
||||||
|
|
||||||
|
|
||||||
|
class _BuildBodyShim:
|
||||||
|
"""Tiny shim so we can pass a body-like object to BuildRunner.start
|
||||||
|
without importing pydantic (which would create a circular import)."""
|
||||||
|
def __init__(self, cmd, dir, project=None, arch="x86_64",
|
||||||
|
target="linux-gnu", toolchain="gcc"):
|
||||||
|
self.cmd = cmd
|
||||||
|
self.dir = dir
|
||||||
|
self.project = project
|
||||||
|
self.arch = arch
|
||||||
|
self.target = target
|
||||||
|
self.toolchain = toolchain
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
class LibvirtManager:
|
||||||
|
|
||||||
|
def execute(self, target, command):
|
||||||
|
|
||||||
|
domain = target.name
|
||||||
|
|
||||||
|
# simplistic execution via ssh inside VM
|
||||||
|
return subprocess.call([
|
||||||
|
"virsh",
|
||||||
|
"domexec",
|
||||||
|
domain,
|
||||||
|
"--",
|
||||||
|
"bash",
|
||||||
|
"-c",
|
||||||
|
command
|
||||||
|
])
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
class LXCManager:
|
||||||
|
|
||||||
|
def execute(self, target, command):
|
||||||
|
|
||||||
|
container = target.name
|
||||||
|
|
||||||
|
return subprocess.call([
|
||||||
|
"lxc-attach",
|
||||||
|
"-n",
|
||||||
|
container,
|
||||||
|
"--",
|
||||||
|
"bash",
|
||||||
|
"-c",
|
||||||
|
command
|
||||||
|
])
|
||||||
|
|
@ -0,0 +1,141 @@
|
||||||
|
"""
|
||||||
|
Mosh integration — build mosh/ssh commands for attaching shells to nodes.
|
||||||
|
|
||||||
|
This manager doesn't *exec* the command (that's the operator's terminal
|
||||||
|
job). It builds the correct command string with all the right flags for:
|
||||||
|
- mosh (default — survives network roams + suspend)
|
||||||
|
- ssh (fallback — works everywhere mosh isn't installed)
|
||||||
|
- mosh via SSH bootstrap (mosh needs SSH for initial connection anyway)
|
||||||
|
|
||||||
|
The /api/nodes/{name}/shell endpoints use this to produce copy-to-clipboard
|
||||||
|
command strings for the UI "Shell" button.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class MoshManager:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._mosh_path = shutil.which("mosh")
|
||||||
|
self._ssh_path = shutil.which("ssh")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def available(self) -> bool:
|
||||||
|
return self._mosh_path is not None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ssh_available(self) -> bool:
|
||||||
|
return self._ssh_path is not None
|
||||||
|
|
||||||
|
def build_command(self, target: str,
|
||||||
|
port: Optional[int] = None,
|
||||||
|
ssh_key: Optional[str] = None,
|
||||||
|
method: str = "mosh",
|
||||||
|
mosh_port_range: Optional[str] = None,
|
||||||
|
extra_ssh_args: Optional[list] = None) -> str:
|
||||||
|
"""Build a shell command string for attaching to `target`.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
target: "user@host" or just "host"
|
||||||
|
port: SSH port (default 22)
|
||||||
|
ssh_key: path to private key file
|
||||||
|
method: "mosh" or "ssh"
|
||||||
|
mosh_port_range: e.g. "60000:61000" (for firewalled environments)
|
||||||
|
extra_ssh_args: list of extra args to pass to ssh
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A shell command string ready to paste into a terminal.
|
||||||
|
"""
|
||||||
|
if method == "ssh":
|
||||||
|
return self._build_ssh(target, port, ssh_key, extra_ssh_args)
|
||||||
|
elif method == "mosh":
|
||||||
|
return self._build_mosh(target, port, ssh_key, mosh_port_range, extra_ssh_args)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unknown method: {method}")
|
||||||
|
|
||||||
|
def _build_ssh(self, target, port, ssh_key, extra_args) -> str:
|
||||||
|
parts = ["ssh"]
|
||||||
|
if port and port != 22:
|
||||||
|
parts.extend(["-p", str(port)])
|
||||||
|
if ssh_key:
|
||||||
|
parts.extend(["-i", ssh_key])
|
||||||
|
if extra_args:
|
||||||
|
parts.extend(extra_args)
|
||||||
|
parts.append(target)
|
||||||
|
return " ".join(parts)
|
||||||
|
|
||||||
|
def _build_mosh(self, target, port, ssh_key,
|
||||||
|
mosh_port_range, extra_ssh_args) -> str:
|
||||||
|
# mosh uses SSH for the bootstrap connection, so we pass SSH args
|
||||||
|
# via --ssh=...
|
||||||
|
ssh_cmd = "ssh"
|
||||||
|
ssh_flags = []
|
||||||
|
if port and port != 22:
|
||||||
|
ssh_flags.extend(["-p", str(port)])
|
||||||
|
if ssh_key:
|
||||||
|
ssh_flags.extend(["-i", ssh_key])
|
||||||
|
if extra_ssh_args:
|
||||||
|
ssh_flags.extend(extra_ssh_args)
|
||||||
|
|
||||||
|
if ssh_flags:
|
||||||
|
ssh_cmd = f"ssh {' '.join(ssh_flags)}"
|
||||||
|
|
||||||
|
parts = ["mosh"]
|
||||||
|
parts.extend(["--ssh", ssh_cmd])
|
||||||
|
if mosh_port_range:
|
||||||
|
parts.extend(["-p", mosh_port_range])
|
||||||
|
parts.append(target)
|
||||||
|
return " ".join(parts)
|
||||||
|
|
||||||
|
def connect(self, node) -> bool:
|
||||||
|
"""Spawn a mosh connection to `node` (blocks until mosh exits).
|
||||||
|
|
||||||
|
Returns True if mosh started successfully, False otherwise.
|
||||||
|
NOTE: this is interactive — only call from a real TTY."""
|
||||||
|
if not self.available:
|
||||||
|
# Fall back to ssh if mosh isn't installed
|
||||||
|
if self.ssh_available:
|
||||||
|
host = node.get("host") if isinstance(node, dict) else node
|
||||||
|
try:
|
||||||
|
subprocess.run(["ssh", host])
|
||||||
|
return True
|
||||||
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||||
|
return False
|
||||||
|
return False
|
||||||
|
|
||||||
|
host = node.get("host") if isinstance(node, dict) else node
|
||||||
|
try:
|
||||||
|
subprocess.run([self._mosh_path, host])
|
||||||
|
return True
|
||||||
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def test_connection(self, node) -> dict:
|
||||||
|
"""Test whether we can reach the node via SSH (mosh bootstrap).
|
||||||
|
|
||||||
|
Doesn't actually start mosh — just runs `ssh -o BatchMode=yes host true`
|
||||||
|
to verify connectivity. Returns a dict with `reachable` + `latency_ms`."""
|
||||||
|
import time
|
||||||
|
host = node.get("host") if isinstance(node, dict) else node
|
||||||
|
if not self.ssh_available:
|
||||||
|
return {"reachable": False, "reason": "ssh not installed"}
|
||||||
|
start = time.time()
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
|
||||||
|
host, "true"],
|
||||||
|
capture_output=True, timeout=10,
|
||||||
|
)
|
||||||
|
latency = (time.time() - start) * 1000
|
||||||
|
if r.returncode == 0:
|
||||||
|
return {"reachable": True, "latency_ms": round(latency, 1)}
|
||||||
|
else:
|
||||||
|
err = r.stderr.decode().strip()[:200]
|
||||||
|
return {"reachable": False, "reason": err, "latency_ms": round(latency, 1)}
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return {"reachable": False, "reason": "timeout", "latency_ms": 10000}
|
||||||
|
except Exception as e:
|
||||||
|
return {"reachable": False, "reason": str(e)}
|
||||||
|
|
@ -0,0 +1,204 @@
|
||||||
|
"""
|
||||||
|
Tmux integration — spawn long-running actions in detached tmux sessions.
|
||||||
|
|
||||||
|
Each action gets its own session `fester-<action_name>-<build_id>` so
|
||||||
|
operators can `tmux attach -t fester-build_kernel-abc123` to watch live
|
||||||
|
build output.
|
||||||
|
|
||||||
|
This is the "action runtime" alternative to running actions inline via
|
||||||
|
subprocess. Set `action["runtime"] = "tmux"` (or `build_env["runtime"]`)
|
||||||
|
to route through TmuxManager instead of run_host.
|
||||||
|
|
||||||
|
Requires: tmux on PATH.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def _have_tmux() -> bool:
|
||||||
|
return shutil.which("tmux") is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_session_name(name: str) -> str:
|
||||||
|
"""tmux session names can't contain `.` or `:`."""
|
||||||
|
return re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:200]
|
||||||
|
|
||||||
|
|
||||||
|
class TmuxManager:
|
||||||
|
"""Manage per-action tmux sessions."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._available = _have_tmux()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def available(self) -> bool:
|
||||||
|
return self._available
|
||||||
|
|
||||||
|
def session_name(self, action: Dict[str, Any], build_id: Optional[str] = None) -> str:
|
||||||
|
"""Return the canonical session name for an action."""
|
||||||
|
name = action.get("name", "action")
|
||||||
|
if build_id:
|
||||||
|
return _safe_session_name(f"fester-{name}-{build_id}")
|
||||||
|
return _safe_session_name(f"fester-{name}")
|
||||||
|
|
||||||
|
def create_session(self, action: Dict[str, Any], cmd: str,
|
||||||
|
cwd: Optional[str] = None,
|
||||||
|
env: Optional[Dict[str, str]] = None,
|
||||||
|
build_id: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""Create a detached tmux session running `cmd`.
|
||||||
|
|
||||||
|
Returns the session name on success, None on failure (or if
|
||||||
|
tmux isn't available)."""
|
||||||
|
if not self._available:
|
||||||
|
return None
|
||||||
|
|
||||||
|
name = self.session_name(action, build_id)
|
||||||
|
|
||||||
|
# If session already exists, kill it first (idempotent)
|
||||||
|
if self.session_exists(name):
|
||||||
|
self.kill_session(name)
|
||||||
|
|
||||||
|
# Build the command with cwd + env wrapping
|
||||||
|
# Use bash -lc so env vars expand correctly
|
||||||
|
env_prefix = ""
|
||||||
|
if env:
|
||||||
|
env_prefix = " ".join(f"{k}={shlex_quote(str(v))}" for k, v in env.items()) + " "
|
||||||
|
|
||||||
|
full_cmd = f"{env_prefix}{cmd}"
|
||||||
|
if cwd:
|
||||||
|
full_cmd = f"cd {shlex_quote(cwd)} && {full_cmd}"
|
||||||
|
|
||||||
|
# Run inside bash -lc so the command parses properly
|
||||||
|
argv = [
|
||||||
|
"tmux", "new-session", "-d",
|
||||||
|
"-s", name,
|
||||||
|
"-x", "200", "-y", "50", # window size
|
||||||
|
"bash", "-lc", full_cmd,
|
||||||
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = subprocess.run(argv, capture_output=True, timeout=10)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return name
|
||||||
|
# Session creation failed
|
||||||
|
return None
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def session_exists(self, name: str) -> bool:
|
||||||
|
if not self._available:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
["tmux", "has-session", "-t", name],
|
||||||
|
capture_output=True, timeout=2,
|
||||||
|
)
|
||||||
|
return r.returncode == 0
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def attach(self, name: str) -> bool:
|
||||||
|
"""Attach to a tmux session. Returns True on success.
|
||||||
|
Note: this blocks until the user detaches."""
|
||||||
|
if not self._available or not self.session_exists(name):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
subprocess.run(["tmux", "attach", "-t", name])
|
||||||
|
return True
|
||||||
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def kill_session(self, name: str) -> bool:
|
||||||
|
if not self._available:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
subprocess.run(["tmux", "kill-session", "-t", name],
|
||||||
|
capture_output=True, timeout=2)
|
||||||
|
return True
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def capture_output(self, name: str, lines: int = 1000) -> Optional[str]:
|
||||||
|
"""Capture the last N lines of tmux pane output."""
|
||||||
|
if not self._available or not self.session_exists(name):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
["tmux", "capture-pane", "-p", "-S", f"-{lines}", "-t", name],
|
||||||
|
capture_output=True, timeout=3,
|
||||||
|
)
|
||||||
|
if r.returncode == 0:
|
||||||
|
return r.stdout.decode("utf-8", errors="replace")
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
def session_state(self, name: str) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Return state info for a session: exists, pane_pid, started_at, etc."""
|
||||||
|
if not self._available or not self.session_exists(name):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
# Get the pane's pid
|
||||||
|
r = subprocess.run(
|
||||||
|
["tmux", "list-panes", "-t", name, "-F",
|
||||||
|
"#{pane_pid}:#{session_created}:#{pane_current_command}"],
|
||||||
|
capture_output=True, timeout=2,
|
||||||
|
)
|
||||||
|
if r.returncode != 0:
|
||||||
|
return {"name": name, "exists": True}
|
||||||
|
line = r.stdout.decode().strip().split("\n")[0]
|
||||||
|
parts = line.split(":")
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"exists": True,
|
||||||
|
"pane_pid": int(parts[0]) if parts and parts[0].isdigit() else None,
|
||||||
|
"created_at": int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None,
|
||||||
|
"current_command": parts[2] if len(parts) > 2 else None,
|
||||||
|
}
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError, ValueError):
|
||||||
|
return {"name": name, "exists": True}
|
||||||
|
|
||||||
|
def list_fester_sessions(self) -> List[Dict[str, Any]]:
|
||||||
|
"""List all fester-* tmux sessions."""
|
||||||
|
if not self._available:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
["tmux", "list-sessions", "-F", "#{session_name}:#{session_created}:#{session_attached}"],
|
||||||
|
capture_output=True, timeout=2,
|
||||||
|
)
|
||||||
|
if r.returncode != 0:
|
||||||
|
return []
|
||||||
|
sessions = []
|
||||||
|
for line in r.stdout.decode().strip().split("\n"):
|
||||||
|
if not line or not line.startswith("fester-"):
|
||||||
|
continue
|
||||||
|
parts = line.split(":")
|
||||||
|
name = parts[0]
|
||||||
|
created = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
|
||||||
|
attached = parts[2] == "1" if len(parts) > 2 else False
|
||||||
|
state = self.session_state(name) or {}
|
||||||
|
sessions.append({
|
||||||
|
"name": name,
|
||||||
|
"created_at": created,
|
||||||
|
"attached": attached,
|
||||||
|
"pane_pid": state.get("pane_pid"),
|
||||||
|
"current_command": state.get("current_command"),
|
||||||
|
})
|
||||||
|
return sessions
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def shlex_quote(s: str) -> str:
|
||||||
|
"""Minimal shlex.quote replacement (avoids importing shlex for one call)."""
|
||||||
|
if not s:
|
||||||
|
return "''"
|
||||||
|
if re.match(r"^[A-Za-z0-9_@%+=:,./-]+$", s):
|
||||||
|
return s
|
||||||
|
return "'" + s.replace("'", "'\"'\"'") + "'"
|
||||||
|
|
@ -0,0 +1,945 @@
|
||||||
|
"""
|
||||||
|
backend/main.py — the real Fester FastAPI application.
|
||||||
|
|
||||||
|
Wires all routers, the WebSocket stream, and the singleton bus + registry.
|
||||||
|
Run with:
|
||||||
|
python -m backend.main
|
||||||
|
or:
|
||||||
|
uvicorn backend.main:app --host 0.0.0.0 --port 8080
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Body
|
||||||
|
from fastapi.responses import FileResponse, JSONResponse
|
||||||
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
# Singletons (must be created BEFORE routers that use them)
|
||||||
|
from backend.api.api import bus, registry, observability, cause_graph, attach_ws_stream
|
||||||
|
from backend.api.ws import WebSocketStream
|
||||||
|
|
||||||
|
# Routers
|
||||||
|
from backend.api.replay import router as replay_router
|
||||||
|
from backend.api.autopsy import router as autopsy_router
|
||||||
|
from backend.api.timeline import router as timeline_router
|
||||||
|
from backend.api.debugger import router as debugger_router, set_engine as dbg_set_engine
|
||||||
|
from backend.api.pipeline_control import router as pipeline_router, set_engine as pc_set_engine
|
||||||
|
from backend.api.nodes import router as nodes_router
|
||||||
|
from backend.api.metrics import router as metrics_router
|
||||||
|
from backend.api.cause import router as cause_router
|
||||||
|
|
||||||
|
# Other deps
|
||||||
|
from backend.events.schema import EventType
|
||||||
|
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||||
|
from backend.analysis.failure_propagation import FailurePropagator
|
||||||
|
from backend.config import CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# App
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent # /home/z/my-project/fester
|
||||||
|
|
||||||
|
app = FastAPI(
|
||||||
|
title="Fester",
|
||||||
|
description="Distributed build + scheduler + observability system",
|
||||||
|
version="0.2.0",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Attach WS stream to bus
|
||||||
|
ws_stream = WebSocketStream()
|
||||||
|
attach_ws_stream(ws_stream)
|
||||||
|
|
||||||
|
# Wire a timeline store subscriber so every bus event lands in the journal
|
||||||
|
def _timeline_subscriber(event: Dict[str, Any]):
|
||||||
|
TIMELINE_STORE.append(event)
|
||||||
|
|
||||||
|
bus.subscribe(_timeline_subscriber)
|
||||||
|
|
||||||
|
# Wire a failure propagator so we can compute blast radius later
|
||||||
|
_propagator = FailurePropagator()
|
||||||
|
bus.subscribe(_propagator.ingest_event)
|
||||||
|
|
||||||
|
# Wire metrics counters so /metrics reflects real activity
|
||||||
|
from backend.api.metrics import record_pipeline_action, record_cache_hit
|
||||||
|
|
||||||
|
def _metrics_subscriber(event: Dict[str, Any]):
|
||||||
|
"""Increment counters for task_update + cache_update events."""
|
||||||
|
etype = event.get("type")
|
||||||
|
if etype == "task_update":
|
||||||
|
state = event.get("state")
|
||||||
|
if state:
|
||||||
|
record_pipeline_action(state)
|
||||||
|
elif etype == "cache_update":
|
||||||
|
# Cache hit: try meta.target first, then top-level target, then "unknown"
|
||||||
|
target = event.get("target") or (event.get("meta", {}) or {}).get("target") or "unknown"
|
||||||
|
if event.get("state") == "hit":
|
||||||
|
record_cache_hit(target)
|
||||||
|
|
||||||
|
bus.subscribe(_metrics_subscriber)
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Build history (in-memory; replace with DB later)
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Build history lives in build_runner (defined below).
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Static UI (mount LAST so it doesn't shadow API routes)
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
@app.get("/")
|
||||||
|
@app.get("/index.html")
|
||||||
|
async def index():
|
||||||
|
return FileResponse(ROOT / "index.html")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/style.css")
|
||||||
|
async def style_css():
|
||||||
|
return FileResponse(ROOT / "style.css", media_type="text/css")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/ui/app.js")
|
||||||
|
async def app_js():
|
||||||
|
return FileResponse(ROOT / "ui" / "app.js", media_type="application/javascript")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/ui/{name}.html")
|
||||||
|
async def ui_page(name: str):
|
||||||
|
p = ROOT / "ui" / f"{name}.html"
|
||||||
|
if not p.exists() or not p.is_file():
|
||||||
|
raise HTTPException(404)
|
||||||
|
return FileResponse(p, media_type="text/html")
|
||||||
|
|
||||||
|
|
||||||
|
# Mount cockpit folder (for the cockpit module)
|
||||||
|
app.mount("/cockpit", StaticFiles(directory=str(ROOT / "cockpit")), name="cockpit")
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Include API routers
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
app.include_router(replay_router)
|
||||||
|
app.include_router(autopsy_router)
|
||||||
|
app.include_router(timeline_router)
|
||||||
|
app.include_router(debugger_router)
|
||||||
|
app.include_router(pipeline_router)
|
||||||
|
app.include_router(nodes_router)
|
||||||
|
app.include_router(metrics_router)
|
||||||
|
app.include_router(cause_router)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Build runner — real engine + synthetic fallback
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
from backend.pipeline.runner import BuildRunner
|
||||||
|
|
||||||
|
# Nodes for the runner come from CONFIG; the registry is the singleton.
|
||||||
|
_runner_nodes = [{"name": n["name"], "host": n.get("host"), "max_jobs": n.get("max_jobs", 8),
|
||||||
|
"state": "online"} for n in CONFIG.get("nodes", [])]
|
||||||
|
build_runner = BuildRunner(bus, registry, _runner_nodes)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Top-level endpoints that don't fit a router
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
class BuildBody(BaseModel):
|
||||||
|
cmd: str = "make -j$(nproc)"
|
||||||
|
dir: str = "/home/user/linux"
|
||||||
|
project: str | None = None
|
||||||
|
arch: str = "x86_64"
|
||||||
|
target: str = "linux-gnu"
|
||||||
|
toolchain: str = "gcc"
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/build")
|
||||||
|
async def api_build(body: BuildBody):
|
||||||
|
"""Kick off a build. Returns a build_id immediately; events stream via /ws."""
|
||||||
|
build_id = await build_runner.start(body)
|
||||||
|
return {"status": "started", "build_id": build_id}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/builds")
|
||||||
|
async def api_builds():
|
||||||
|
"""Build history."""
|
||||||
|
return {"builds": build_runner.list_builds()}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/builds/{build_id}")
|
||||||
|
async def api_build_detail(build_id: str):
|
||||||
|
state = build_runner.state(build_id)
|
||||||
|
if state:
|
||||||
|
return state
|
||||||
|
raise HTTPException(404, "build not found")
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/builds/{build_id}/cancel")
|
||||||
|
async def api_build_cancel(build_id: str):
|
||||||
|
if build_runner.cancel(build_id):
|
||||||
|
return {"status": "cancelled", "build_id": build_id}
|
||||||
|
raise HTTPException(404, "build not found or not running")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/sessions")
|
||||||
|
async def api_sessions():
|
||||||
|
"""List replay sessions known to the server (from SessionDB)."""
|
||||||
|
from backend.api.replay import SESSION_DB
|
||||||
|
return {"sessions": SESSION_DB.list_sessions()}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Targets API — backed by backend/targets/catalog.py + ARCHES/RUNTIMES
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
from backend.targets.catalog import TARGET_SYSTEMS, ARCHES, RUNTIMES
|
||||||
|
|
||||||
|
|
||||||
|
def _build_targets_catalog():
|
||||||
|
"""Combine catalog target systems + arches into a unified toggle list."""
|
||||||
|
out = []
|
||||||
|
# Per-arch cross-compile targets (always present)
|
||||||
|
arch_defaults = {
|
||||||
|
"x86_64": True, # most common, default-on
|
||||||
|
"aarch64": True,
|
||||||
|
"riscv64": False,
|
||||||
|
"arm64": True,
|
||||||
|
}
|
||||||
|
for arch in ARCHES:
|
||||||
|
out.append({
|
||||||
|
"name": f"{arch}-linux-gnu",
|
||||||
|
"enabled": arch_defaults.get(arch, False),
|
||||||
|
"arch": arch,
|
||||||
|
"type": "cross-compile",
|
||||||
|
"source": "arch",
|
||||||
|
})
|
||||||
|
# Per-target-system entries (Gentoo, Buildroot, OpenWrt, etc.)
|
||||||
|
for tname, tinfo in TARGET_SYSTEMS.items():
|
||||||
|
out.append({
|
||||||
|
"name": tname,
|
||||||
|
"enabled": False, # opt-in
|
||||||
|
"type": tinfo.get("type", "unknown"),
|
||||||
|
"toolchain": tinfo.get("toolchain"),
|
||||||
|
"supports_cross": tinfo.get("supports_cross", False),
|
||||||
|
"source": "catalog",
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
TARGETS_CATALOG = _build_targets_catalog()
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/targets")
|
||||||
|
async def api_targets():
|
||||||
|
"""List all known build targets (arches + catalog systems)."""
|
||||||
|
return TARGETS_CATALOG
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/targets/arches")
|
||||||
|
async def api_targets_arches():
|
||||||
|
"""List supported architectures."""
|
||||||
|
return {"arches": ARCHES}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/targets/runtimes")
|
||||||
|
async def api_targets_runtimes():
|
||||||
|
"""List supported execution runtimes."""
|
||||||
|
return {"runtimes": RUNTIMES}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/targets/{name}")
|
||||||
|
async def api_toggle_target(name: str, body: Dict[str, Any] = Body(default={})):
|
||||||
|
for t in TARGETS_CATALOG:
|
||||||
|
if t["name"] == name:
|
||||||
|
t["enabled"] = bool(body.get("enabled", False))
|
||||||
|
# Broadcast to ws-targets subscribers
|
||||||
|
await _broadcast_targets({
|
||||||
|
"type": "target-update",
|
||||||
|
"data": {"name": name, "enabled": t["enabled"]},
|
||||||
|
})
|
||||||
|
return {"status": "ok", "name": name, "enabled": t["enabled"]}
|
||||||
|
raise HTTPException(404, "target not found")
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Release API
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
class ReleaseBody(BaseModel):
|
||||||
|
repo: str = "https://forgejo.local/project.git"
|
||||||
|
project: str | None = None
|
||||||
|
target: str = "linux-gnu"
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/release")
|
||||||
|
async def api_release(body: ReleaseBody):
|
||||||
|
"""Kick off a release build. Mirrors /api/build but tags the build_id
|
||||||
|
with a 'release-' prefix and uses a longer release-shaped pipeline."""
|
||||||
|
release_id = str(uuid.uuid4())[:8]
|
||||||
|
build_id = f"release-{release_id}"
|
||||||
|
|
||||||
|
# Construct a release-shaped BuildBody and dispatch through BuildRunner
|
||||||
|
release_body = BuildBody(
|
||||||
|
cmd=f"make release REPO={body.repo}",
|
||||||
|
dir="/home/user/release",
|
||||||
|
project=body.project or body.repo,
|
||||||
|
arch="x86_64",
|
||||||
|
target=body.target,
|
||||||
|
toolchain="gcc",
|
||||||
|
)
|
||||||
|
actual_id = await build_runner.start(release_body)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": "started",
|
||||||
|
"repo": body.repo,
|
||||||
|
"release_id": release_id,
|
||||||
|
"build_id": actual_id,
|
||||||
|
"pipeline": ["fetch_source", "configure", "build_kernel", "build_modules",
|
||||||
|
"package", "release"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Policy API — real implementation backed by an in-memory policy store
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
from backend.policy.engine import PolicyEngine
|
||||||
|
from backend.api.nodes import PolicyBody as NodePolicyBody # reuse for top-level policy too
|
||||||
|
|
||||||
|
# Singleton policy engine (no DB learning for now — just key/value overrides)
|
||||||
|
POLICY_ENGINE = PolicyEngine(db=None)
|
||||||
|
_POLICY_STORE: Dict[str, Any] = {} # simple key→value overrides
|
||||||
|
|
||||||
|
|
||||||
|
# Pydantic model for /api/policy/set
|
||||||
|
class PolicyBody(BaseModel):
|
||||||
|
key: str
|
||||||
|
value: Any
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/policy/set")
|
||||||
|
async def api_policy_set(body: PolicyBody):
|
||||||
|
"""Set a policy key=value. Currently understood keys:
|
||||||
|
- scheduler.mode (unified|weighted-thermal-aware|cache-first|target-isolated|experimental-intelligence)
|
||||||
|
- scheduler.thermal_cap (float, 0..1)
|
||||||
|
- scheduler.cache_first (bool)
|
||||||
|
- build.default_arch (string)
|
||||||
|
Other keys are stored but not yet acted upon.
|
||||||
|
"""
|
||||||
|
_POLICY_STORE[body.key] = body.value
|
||||||
|
bus.emit(EventType.PIPELINE_UPDATE, state="policy",
|
||||||
|
meta={"key": body.key, "value": body.value, "action": "set"})
|
||||||
|
return {"status": "ok", "key": body.key, "value": body.value,
|
||||||
|
"stored_keys": list(_POLICY_STORE.keys())}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/policy/clear")
|
||||||
|
async def api_policy_clear(body: Dict[str, Any] = Body(default={})):
|
||||||
|
key = body.get("key")
|
||||||
|
if key and key in _POLICY_STORE:
|
||||||
|
del _POLICY_STORE[key]
|
||||||
|
bus.emit(EventType.PIPELINE_UPDATE, state="policy",
|
||||||
|
meta={"key": key, "action": "clear"})
|
||||||
|
return {"status": "cleared", "key": key, "remaining_keys": list(_POLICY_STORE.keys())}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/policy")
|
||||||
|
async def api_policy_list():
|
||||||
|
"""List all current policy overrides."""
|
||||||
|
return {"policies": _POLICY_STORE}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Failure propagation (blast radius)
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
@app.get("/api/propagation/{action}")
|
||||||
|
async def api_propagation(action: str):
|
||||||
|
"""Compute the downstream blast radius if `action` fails."""
|
||||||
|
return _propagator.impact_report(action)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Sessions attach/detach (for the CLI / IDE feature)
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
@app.post("/api/session/attach")
|
||||||
|
async def api_session_attach(body: Dict[str, Any] = Body(default={})):
|
||||||
|
return {"status": "attached", "action": body.get("action")}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/session/detach")
|
||||||
|
async def api_session_detach(body: Dict[str, Any] = Body(default={})):
|
||||||
|
return {"status": "detached", "action": body.get("action")}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# WebSocket endpoints
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
class WSHub:
|
||||||
|
def __init__(self):
|
||||||
|
self.clients: List[WebSocket] = []
|
||||||
|
self.targets_clients: List[WebSocket] = []
|
||||||
|
self.debugger_clients: List[WebSocket] = []
|
||||||
|
|
||||||
|
async def accept(self, ws: WebSocket, kind: str = "main"):
|
||||||
|
await ws.accept()
|
||||||
|
if kind == "main":
|
||||||
|
self.clients.append(ws)
|
||||||
|
elif kind == "targets":
|
||||||
|
self.targets_clients.append(ws)
|
||||||
|
elif kind == "debugger":
|
||||||
|
self.debugger_clients.append(ws)
|
||||||
|
|
||||||
|
|
||||||
|
hub = WSHub()
|
||||||
|
|
||||||
|
# Wire ws_stream (singleton) to broadcast to /ws clients.
|
||||||
|
# We can't await inside the sync bus.emit, so we schedule sends on the
|
||||||
|
# running loop. If there's no running loop (called from sync context),
|
||||||
|
# we silently drop — the WS hub is async-only.
|
||||||
|
def _ws_broadcast(event: Dict[str, Any]):
|
||||||
|
payload = json.dumps(event)
|
||||||
|
try:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
except RuntimeError:
|
||||||
|
return # no running loop; skip
|
||||||
|
for c in list(hub.clients):
|
||||||
|
try:
|
||||||
|
loop.create_task(c.send_text(payload))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
ws_stream.clients = hub.clients # share the list
|
||||||
|
bus.subscribe(_ws_broadcast)
|
||||||
|
|
||||||
|
|
||||||
|
async def _broadcast_targets(event: Dict[str, Any]):
|
||||||
|
payload = json.dumps(event)
|
||||||
|
dead = []
|
||||||
|
for c in hub.targets_clients:
|
||||||
|
try:
|
||||||
|
await c.send_text(payload)
|
||||||
|
except Exception:
|
||||||
|
dead.append(c)
|
||||||
|
for d in dead:
|
||||||
|
if d in hub.targets_clients:
|
||||||
|
hub.targets_clients.remove(d)
|
||||||
|
|
||||||
|
|
||||||
|
@app.websocket("/ws")
|
||||||
|
async def ws_main(ws: WebSocket):
|
||||||
|
await hub.accept(ws, "main")
|
||||||
|
try:
|
||||||
|
await ws.send_text(json.dumps({
|
||||||
|
"type": "session",
|
||||||
|
"timestamp": time.time(),
|
||||||
|
"data": {"action": "subscribe", "id": "live"},
|
||||||
|
}))
|
||||||
|
while True:
|
||||||
|
await ws.receive_text()
|
||||||
|
except WebSocketDisconnect:
|
||||||
|
if ws in hub.clients:
|
||||||
|
hub.clients.remove(ws)
|
||||||
|
|
||||||
|
|
||||||
|
@app.websocket("/ws-targets")
|
||||||
|
async def ws_targets(ws: WebSocket):
|
||||||
|
await hub.accept(ws, "targets")
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
await ws.receive_text()
|
||||||
|
except WebSocketDisconnect:
|
||||||
|
if ws in hub.targets_clients:
|
||||||
|
hub.targets_clients.remove(ws)
|
||||||
|
|
||||||
|
|
||||||
|
@app.websocket("/ws-debugger")
|
||||||
|
async def ws_debugger(ws: WebSocket):
|
||||||
|
"""Real debugger WS — responds to step/pause/resume + streams timeline state."""
|
||||||
|
await hub.accept(ws, "debugger")
|
||||||
|
try:
|
||||||
|
# Send initial state on connect
|
||||||
|
eng = build_runner.get_engine()
|
||||||
|
active = eng is not None
|
||||||
|
await ws.send_text(json.dumps({
|
||||||
|
"type": "timeline-update",
|
||||||
|
"data": {
|
||||||
|
"events": _debugger_timeline_snapshot(),
|
||||||
|
"current": _debugger_current(),
|
||||||
|
"engine_attached": active,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
while True:
|
||||||
|
msg = await ws.receive_text()
|
||||||
|
try:
|
||||||
|
cmd = json.loads(msg)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
action = cmd.get("action")
|
||||||
|
eng = build_runner.get_engine()
|
||||||
|
|
||||||
|
if action == "pause":
|
||||||
|
if eng and hasattr(eng, "pause"):
|
||||||
|
eng.pause()
|
||||||
|
from backend.api.debugger import _DEBUG_STATE
|
||||||
|
_DEBUG_STATE["paused"] = True
|
||||||
|
response = {"type": "state-change", "data": {"state": "paused"}}
|
||||||
|
elif action == "resume":
|
||||||
|
if eng and hasattr(eng, "resume"):
|
||||||
|
eng.resume()
|
||||||
|
from backend.api.debugger import _DEBUG_STATE
|
||||||
|
_DEBUG_STATE["paused"] = False
|
||||||
|
response = {"type": "state-change", "data": {"state": "running"}}
|
||||||
|
elif action == "step_forward" or action == "step":
|
||||||
|
if eng and hasattr(eng, "step"):
|
||||||
|
eng.step()
|
||||||
|
from backend.api.debugger import _DEBUG_STATE
|
||||||
|
_DEBUG_STATE["current_step"] += 1
|
||||||
|
response = {"type": "state-change",
|
||||||
|
"data": {"state": "stepped", "step": _DEBUG_STATE["current_step"]}}
|
||||||
|
elif action == "step_back":
|
||||||
|
from backend.api.debugger import _DEBUG_STATE
|
||||||
|
if _DEBUG_STATE["current_step"] > 0:
|
||||||
|
_DEBUG_STATE["current_step"] -= 1
|
||||||
|
response = {"type": "state-change",
|
||||||
|
"data": {"state": "stepped_back", "step": _DEBUG_STATE["current_step"]}}
|
||||||
|
else:
|
||||||
|
response = {"type": "error", "data": {"error": f"unknown action: {action}"}}
|
||||||
|
|
||||||
|
# Augment with the latest timeline state
|
||||||
|
response["timeline"] = _debugger_timeline_snapshot()
|
||||||
|
response["current"] = _debugger_current()
|
||||||
|
await ws.send_text(json.dumps(response))
|
||||||
|
except WebSocketDisconnect:
|
||||||
|
if ws in hub.debugger_clients:
|
||||||
|
hub.debugger_clients.remove(ws)
|
||||||
|
|
||||||
|
|
||||||
|
def _debugger_timeline_snapshot():
|
||||||
|
"""Return a list of {name, completed, node, state} for the debugger UI."""
|
||||||
|
events = TIMELINE_STORE.all()
|
||||||
|
seen = {}
|
||||||
|
for e in events:
|
||||||
|
action = e.get("action")
|
||||||
|
if not action:
|
||||||
|
continue
|
||||||
|
d = e
|
||||||
|
state = d.get("state")
|
||||||
|
seen[action] = {
|
||||||
|
"name": action,
|
||||||
|
"completed": state == "done",
|
||||||
|
"state": state,
|
||||||
|
"node": d.get("node"),
|
||||||
|
"failed": state == "failed",
|
||||||
|
}
|
||||||
|
return list(seen.values())
|
||||||
|
|
||||||
|
|
||||||
|
def _debugger_current():
|
||||||
|
"""Return the most recent running or scheduled action."""
|
||||||
|
events = TIMELINE_STORE.all()
|
||||||
|
for e in reversed(events):
|
||||||
|
if e.get("state") in ("running", "scheduled"):
|
||||||
|
return {"action": e.get("action"), "state": e.get("state"),
|
||||||
|
"node": e.get("node")}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Health
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
@app.get("/api/health")
|
||||||
|
async def health():
|
||||||
|
return {
|
||||||
|
"status": "ok",
|
||||||
|
"version": "0.2.0",
|
||||||
|
"bus_subscribers": len(bus.subscribers),
|
||||||
|
"ws_clients": len(hub.clients),
|
||||||
|
"builds_known": len(build_runner.list_builds()) if build_runner else 0,
|
||||||
|
"timeline_events": TIMELINE_STORE.stats()["total_events"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Storage API — btrfs CAS, tmpfs, qcow2 snapshots
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
from backend.storage.router import (
|
||||||
|
prepare_workspace, maybe_freeze_workspace, storage_status,
|
||||||
|
update_storage_config,
|
||||||
|
)
|
||||||
|
from backend.storage.btrfs_cas import is_btrfs, store_reflink, reflink_snapshot, cas_store, cas_exists
|
||||||
|
from backend.storage.qcow2_freeze import freeze_to_qcow2, qcow2_info, list_snapshots
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/storage/status")
|
||||||
|
async def api_storage_status():
|
||||||
|
"""Return current storage configuration + capability detection."""
|
||||||
|
return storage_status()
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/storage/config")
|
||||||
|
async def api_storage_config_update(body: Dict[str, Any] = Body(default={})):
|
||||||
|
"""Update storage configuration (e.g. {btrfs: true, qcow2_freeze: true})."""
|
||||||
|
return update_storage_config(body)
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/storage/freeze")
|
||||||
|
async def api_storage_freeze(body: Dict[str, Any] = Body(default={})):
|
||||||
|
"""Freeze a workdir into a qcow2 snapshot.
|
||||||
|
|
||||||
|
Body: {"workdir": "/path/to/workdir", "project": "name"}"""
|
||||||
|
workdir = body.get("workdir")
|
||||||
|
project = body.get("project", "manual")
|
||||||
|
if not workdir:
|
||||||
|
raise HTTPException(400, "workdir required")
|
||||||
|
result = maybe_freeze_workspace(workdir, project)
|
||||||
|
if result is None:
|
||||||
|
return {"status": "disabled", "reason": "qcow2_freeze not enabled in config"}
|
||||||
|
return {"status": "frozen", "path": result, "project": project}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/storage/snapshots")
|
||||||
|
async def api_storage_snapshots():
|
||||||
|
"""List all qcow2 snapshots."""
|
||||||
|
from backend.storage.config import load_storage_config
|
||||||
|
cfg = load_storage_config()
|
||||||
|
return {"snapshots": list_snapshots(cfg.get("qcow2_path", "/var/lib/fester/snapshots"))}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/storage/reflink")
|
||||||
|
async def api_storage_reflink(body: Dict[str, Any] = Body(default={})):
|
||||||
|
"""Reflink-copy a file or directory (CoW snapshot on btrfs).
|
||||||
|
|
||||||
|
Body: {"src": "/path/to/src", "dst": "/path/to/dst"}"""
|
||||||
|
src = body.get("src")
|
||||||
|
dst = body.get("dst")
|
||||||
|
if not src or not dst:
|
||||||
|
raise HTTPException(400, "src and dst required")
|
||||||
|
if not Path(src).exists():
|
||||||
|
raise HTTPException(404, f"src not found: {src}")
|
||||||
|
|
||||||
|
from pathlib import Path as _P
|
||||||
|
if _P(src).is_dir():
|
||||||
|
stats = reflink_snapshot(src, dst)
|
||||||
|
return {"status": "ok", "mode": "directory", "stats": stats}
|
||||||
|
else:
|
||||||
|
reflinked = store_reflink(src, dst)
|
||||||
|
return {"status": "ok", "mode": "file", "reflinked": reflinked}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/workspace/prepare")
|
||||||
|
async def api_workspace_prepare(body: Dict[str, Any] = Body(default={})):
|
||||||
|
"""Prepare a workspace for a build (tmpfs + optional btrfs CAS).
|
||||||
|
|
||||||
|
Body: {"project": "name"}"""
|
||||||
|
project = body.get("project", "default")
|
||||||
|
return prepare_workspace(project)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Actions API — tmux session inspection + node shell attach
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
from backend.integrations.tmux import TmuxManager as _TmuxMgr
|
||||||
|
from backend.integrations.mosh import MoshManager as _MoshMgr
|
||||||
|
|
||||||
|
_tmux_mgr = _TmuxMgr()
|
||||||
|
_mosh_mgr = _MoshMgr()
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/actions/active")
|
||||||
|
async def api_actions_active():
|
||||||
|
"""List all active fester-* tmux sessions (running actions)."""
|
||||||
|
if not _tmux_mgr.available:
|
||||||
|
return {"available": False, "sessions": [], "reason": "tmux not installed"}
|
||||||
|
return {
|
||||||
|
"available": True,
|
||||||
|
"sessions": _tmux_mgr.list_fester_sessions(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/actions/{name}/tmux")
|
||||||
|
async def api_action_tmux_state(name: str):
|
||||||
|
"""Get state of the tmux session for a specific action."""
|
||||||
|
if not _tmux_mgr.available:
|
||||||
|
return {"available": False, "reason": "tmux not installed"}
|
||||||
|
# Try with and without build_id suffix
|
||||||
|
candidates = [f"fester-{name}"]
|
||||||
|
# Also try matching any session that starts with fester-{name}-
|
||||||
|
sessions = _tmux_mgr.list_fester_sessions()
|
||||||
|
for s in sessions:
|
||||||
|
if s["name"].startswith(f"fester-{name}-"):
|
||||||
|
candidates.append(s["name"])
|
||||||
|
for cand in candidates:
|
||||||
|
state = _tmux_mgr.session_state(cand)
|
||||||
|
if state:
|
||||||
|
output = _tmux_mgr.capture_output(cand, lines=200) or ""
|
||||||
|
return {
|
||||||
|
"available": True,
|
||||||
|
"name": cand,
|
||||||
|
"state": state,
|
||||||
|
"output_tail": output[-2000:] if output else None,
|
||||||
|
}
|
||||||
|
return {"available": True, "exists": False, "name": name}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/actions/{name}/tmux/output")
|
||||||
|
async def api_action_tmux_output(name: str, lines: int = 1000):
|
||||||
|
"""Capture the last N lines of tmux pane output for an action."""
|
||||||
|
if not _tmux_mgr.available:
|
||||||
|
return {"available": False, "reason": "tmux not installed"}
|
||||||
|
# Find the session
|
||||||
|
candidates = [f"fester-{name}"]
|
||||||
|
sessions = _tmux_mgr.list_fester_sessions()
|
||||||
|
for s in sessions:
|
||||||
|
if s["name"].startswith(f"fester-{name}-"):
|
||||||
|
candidates.append(s["name"])
|
||||||
|
for cand in candidates:
|
||||||
|
if _tmux_mgr.session_exists(cand):
|
||||||
|
output = _tmux_mgr.capture_output(cand, lines=lines) or ""
|
||||||
|
return {
|
||||||
|
"available": True,
|
||||||
|
"name": cand,
|
||||||
|
"lines": len(output.split("\n")) if output else 0,
|
||||||
|
"output": output,
|
||||||
|
}
|
||||||
|
return {"available": True, "exists": False, "name": name}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/actions/{name}/tmux/kill")
|
||||||
|
async def api_action_tmux_kill(name: str):
|
||||||
|
"""Kill the tmux session for an action."""
|
||||||
|
if not _tmux_mgr.available:
|
||||||
|
return {"available": False, "reason": "tmux not installed"}
|
||||||
|
candidates = [f"fester-{name}"]
|
||||||
|
sessions = _tmux_mgr.list_fester_sessions()
|
||||||
|
for s in sessions:
|
||||||
|
if s["name"].startswith(f"fester-{name}-"):
|
||||||
|
candidates.append(s["name"])
|
||||||
|
killed = []
|
||||||
|
for cand in candidates:
|
||||||
|
if _tmux_mgr.kill_session(cand):
|
||||||
|
killed.append(cand)
|
||||||
|
return {"killed": killed, "count": len(killed)}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Node shell — mosh / ssh command builder
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
class ShellBody(BaseModel):
|
||||||
|
user: Optional[str] = None
|
||||||
|
port: Optional[int] = None
|
||||||
|
ssh_key: Optional[str] = None
|
||||||
|
method: str = "mosh" # mosh | ssh
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/nodes/{name}/shell")
|
||||||
|
async def api_node_shell(name: str):
|
||||||
|
"""Return the command string to attach a shell to a node (mosh or ssh).
|
||||||
|
Doesn't actually exec it — the caller's terminal does."""
|
||||||
|
# Find the node
|
||||||
|
cfg = None
|
||||||
|
for n in CONFIG.get("nodes", []):
|
||||||
|
if n["name"] == name:
|
||||||
|
cfg = n
|
||||||
|
break
|
||||||
|
if not cfg:
|
||||||
|
raise HTTPException(404, f"node {name} not configured")
|
||||||
|
host = cfg.get("host")
|
||||||
|
if not host:
|
||||||
|
raise HTTPException(400, f"node {name} has no host")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"node": name,
|
||||||
|
"host": host,
|
||||||
|
"mosh_available": _mosh_mgr.available,
|
||||||
|
"ssh_available": shutil.which("ssh") is not None,
|
||||||
|
"commands": {
|
||||||
|
"mosh": f"mosh {host}",
|
||||||
|
"ssh": f"ssh {host}",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/nodes/{name}/shell")
|
||||||
|
async def api_node_shell_custom(name: str, body: ShellBody):
|
||||||
|
"""Build a custom shell command for a node with user/port/ssh-key options."""
|
||||||
|
cfg = None
|
||||||
|
for n in CONFIG.get("nodes", []):
|
||||||
|
if n["name"] == name:
|
||||||
|
cfg = n
|
||||||
|
break
|
||||||
|
if not cfg:
|
||||||
|
raise HTTPException(404, f"node {name} not configured")
|
||||||
|
host = cfg.get("host")
|
||||||
|
if not host:
|
||||||
|
raise HTTPException(400, f"node {name} has no host")
|
||||||
|
|
||||||
|
target = f"{body.user}@{host}" if body.user else host
|
||||||
|
cmd = _mosh_mgr.build_command(target, port=body.port, ssh_key=body.ssh_key,
|
||||||
|
method=body.method)
|
||||||
|
return {
|
||||||
|
"node": name,
|
||||||
|
"host": host,
|
||||||
|
"method": body.method,
|
||||||
|
"command": cmd,
|
||||||
|
"copy_to_clipboard": cmd,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# Ambient loop — probe real node agents, fall back to drift if unreachable.
|
||||||
|
# Set FESTER_NO_DRIFT=1 to disable synthetic drift entirely (production).
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
async def ambient_loop():
|
||||||
|
"""Probe each configured node's agent every 4s. If the agent responds,
|
||||||
|
update the registry with real metrics. If not, optionally drift the
|
||||||
|
last-known values so the UI doesn't go stale."""
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
from backend.nodes.probe import probe_async, normalize
|
||||||
|
from backend.storage.sqlite_db import STORAGE
|
||||||
|
|
||||||
|
NO_DRIFT = os.environ.get("FESTER_NO_DRIFT") == "1"
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
for n in CONFIG.get("nodes", []):
|
||||||
|
name = n["name"]
|
||||||
|
host = n.get("host")
|
||||||
|
|
||||||
|
# Try real probe first
|
||||||
|
raw = await probe_async(host) if host else None
|
||||||
|
|
||||||
|
if raw is not None:
|
||||||
|
# Real agent responded — use its data
|
||||||
|
state = normalize(raw, name)
|
||||||
|
registry.update(
|
||||||
|
name,
|
||||||
|
cpu_load=state["cpu_load"],
|
||||||
|
memory_load=state["memory_load"],
|
||||||
|
temp=state["temp"],
|
||||||
|
instability=state["instability"],
|
||||||
|
active_jobs=state["active_jobs"],
|
||||||
|
last_seen=state["last_seen"],
|
||||||
|
)
|
||||||
|
cur = registry.get(name)
|
||||||
|
# Persist node state to SQLite
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
STORAGE.save_node_state(name, {
|
||||||
|
"state": state.get("state", "online"),
|
||||||
|
"cpu_load": cur.cpu_load,
|
||||||
|
"memory_load": cur.memory_load,
|
||||||
|
"temp": cur.temp,
|
||||||
|
"instability": cur.instability,
|
||||||
|
"active_jobs": cur.active_jobs,
|
||||||
|
"max_jobs": state.get("max_jobs", n.get("max_jobs", 8)),
|
||||||
|
"last_seen": cur.last_seen,
|
||||||
|
"labels": state.get("labels", {}),
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Emit with state from probe (may be "online" or "degraded")
|
||||||
|
bus.emit(
|
||||||
|
EventType.NODE_UPDATE,
|
||||||
|
node=name,
|
||||||
|
state=state.get("state", "online"),
|
||||||
|
meta={
|
||||||
|
"heat": cur.temp,
|
||||||
|
"jobs": cur.active_jobs,
|
||||||
|
"cpu_load": cur.cpu_load,
|
||||||
|
"instability": cur.instability,
|
||||||
|
"source": "agent",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
elif not NO_DRIFT:
|
||||||
|
# Agent unreachable — drift the last-known values
|
||||||
|
existing = registry.get(name)
|
||||||
|
if existing is None:
|
||||||
|
# First-seen: initialize with plausible values
|
||||||
|
registry.update(
|
||||||
|
name,
|
||||||
|
cpu_load=random.uniform(20, 70),
|
||||||
|
memory_load=random.uniform(30, 60),
|
||||||
|
temp=random.uniform(40, 70),
|
||||||
|
instability=random.uniform(0, 0.15),
|
||||||
|
active_jobs=random.randint(0, max(1, n.get("max_jobs", 8) // 4)),
|
||||||
|
last_seen=time.time(),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
registry.update(
|
||||||
|
name,
|
||||||
|
cpu_load=max(0, min(100, existing.cpu_load + random.uniform(-5, 5))),
|
||||||
|
memory_load=max(0, min(100, existing.memory_load + random.uniform(-2, 2))),
|
||||||
|
temp=max(20, min(95, existing.temp + random.uniform(-3, 3))),
|
||||||
|
instability=max(0, min(1, existing.instability + random.uniform(-0.05, 0.05))),
|
||||||
|
active_jobs=max(0, min(n.get("max_jobs", 8), existing.active_jobs + random.randint(-1, 1))),
|
||||||
|
last_seen=time.time(),
|
||||||
|
)
|
||||||
|
|
||||||
|
cur = registry.get(name)
|
||||||
|
# Persist drifted state too
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
STORAGE.save_node_state(name, {
|
||||||
|
"state": "online",
|
||||||
|
"cpu_load": cur.cpu_load,
|
||||||
|
"memory_load": cur.memory_load,
|
||||||
|
"temp": cur.temp,
|
||||||
|
"instability": cur.instability,
|
||||||
|
"active_jobs": cur.active_jobs,
|
||||||
|
"max_jobs": n.get("max_jobs", 8),
|
||||||
|
"last_seen": cur.last_seen,
|
||||||
|
"labels": {},
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
bus.emit(
|
||||||
|
EventType.NODE_UPDATE,
|
||||||
|
node=name,
|
||||||
|
state="online",
|
||||||
|
meta={
|
||||||
|
"heat": cur.temp,
|
||||||
|
"jobs": cur.active_jobs,
|
||||||
|
"cpu_load": cur.cpu_load,
|
||||||
|
"instability": cur.instability,
|
||||||
|
"source": "drift",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
# If NO_DRIFT and no agent, we simply skip — node stays offline
|
||||||
|
|
||||||
|
await asyncio.sleep(4)
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
async def on_startup():
|
||||||
|
asyncio.create_task(ambient_loop())
|
||||||
|
# Optional: kick off a build every 60s for demo purposes.
|
||||||
|
# Disabled by default — set FESTER_AUTOBUILD=1 to enable.
|
||||||
|
import os
|
||||||
|
if os.environ.get("FESTER_AUTOBUILD") == "1":
|
||||||
|
async def looper():
|
||||||
|
await asyncio.sleep(15) # let things warm up first
|
||||||
|
while True:
|
||||||
|
await build_runner.start(
|
||||||
|
BuildBody(cmd="make -j$(nproc)", dir="/home/user/linux"),
|
||||||
|
)
|
||||||
|
await asyncio.sleep(60)
|
||||||
|
asyncio.create_task(looper())
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
# CLI entrypoint
|
||||||
|
# ----------------------------------------------------------------
|
||||||
|
def main():
|
||||||
|
import uvicorn
|
||||||
|
uvicorn.run("backend.main:app", host="0.0.0.0", port=8080, reload=False)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
class ClusterState:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
|
||||||
|
self.nodes = {}
|
||||||
|
|
||||||
|
def register_node(self, node_id, node_state):
|
||||||
|
|
||||||
|
self.nodes[node_id] = node_state
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GLOBAL AGGREGATION (VERY CHEAP)
|
||||||
|
# -----------------------------
|
||||||
|
def snapshot(self):
|
||||||
|
|
||||||
|
hot_nodes = 0
|
||||||
|
total_temp = 0
|
||||||
|
total_cpu = 0
|
||||||
|
instability_sum = 0
|
||||||
|
n = len(self.nodes)
|
||||||
|
|
||||||
|
if n == 0:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
for node in self.nodes.values():
|
||||||
|
|
||||||
|
s = node.snapshot()
|
||||||
|
|
||||||
|
total_temp += s["avg_temp"]
|
||||||
|
total_cpu += s["avg_cpu"]
|
||||||
|
instability_sum += s["instability"]
|
||||||
|
|
||||||
|
if s["avg_temp"] > 80:
|
||||||
|
hot_nodes += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"avg_cluster_temp": total_temp / n,
|
||||||
|
"avg_cluster_cpu": total_cpu / n,
|
||||||
|
"instability": instability_sum / n,
|
||||||
|
"hot_nodes": hot_nodes
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
import time
|
||||||
|
import psutil
|
||||||
|
|
||||||
|
|
||||||
|
class MetricsCollector:
|
||||||
|
"""
|
||||||
|
Lightweight node telemetry sampler.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def sample(self):
|
||||||
|
|
||||||
|
return {
|
||||||
|
"cpu": psutil.cpu_percent(interval=0.1),
|
||||||
|
"memory": psutil.virtual_memory().percent,
|
||||||
|
"load": psutil.getloadavg()[0] if hasattr(psutil, "getloadavg") else 0,
|
||||||
|
"timestamp": time.time(),
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,147 @@
|
||||||
|
from backend.metrics.target_scoring import TargetScorer
|
||||||
|
from backend.metrics.target_labeler import TargetLabeler
|
||||||
|
from backend.metrics.profile_store import save
|
||||||
|
|
||||||
|
|
||||||
|
class LiveTargetState:
|
||||||
|
|
||||||
|
def __init__(self, target_key):
|
||||||
|
self.target = target_key
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RUNNING STATE (incremental)
|
||||||
|
# -----------------------------
|
||||||
|
self.cpu_sum = 0
|
||||||
|
self.temp_sum = 0
|
||||||
|
self.samples = 0
|
||||||
|
|
||||||
|
self.peak_temp = 0
|
||||||
|
|
||||||
|
self.actions = 0
|
||||||
|
self.cache_hits = 0
|
||||||
|
self.failures = 0
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# CACHED DERIVED VALUES
|
||||||
|
# -----------------------------
|
||||||
|
self.cached_score = None
|
||||||
|
self.cached_labels = None
|
||||||
|
self.dirty = True
|
||||||
|
|
||||||
|
|
||||||
|
class FesterIntelligence:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
|
||||||
|
self.states = {}
|
||||||
|
self.scorer = TargetScorer()
|
||||||
|
self.labeler = TargetLabeler()
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# INIT
|
||||||
|
# -----------------------------
|
||||||
|
def start_target(self, target_key):
|
||||||
|
|
||||||
|
self.states[target_key] = LiveTargetState(target_key)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FAST UPDATES (O(1))
|
||||||
|
# -----------------------------
|
||||||
|
def sample(self, target_key, node):
|
||||||
|
|
||||||
|
s = self.states[target_key]
|
||||||
|
|
||||||
|
cpu = node.get("cpu_load", 0)
|
||||||
|
temp = node.get("temp", 0)
|
||||||
|
|
||||||
|
s.cpu_sum += cpu
|
||||||
|
s.temp_sum += temp
|
||||||
|
s.samples += 1
|
||||||
|
|
||||||
|
if temp > s.peak_temp:
|
||||||
|
s.peak_temp = temp
|
||||||
|
|
||||||
|
s.dirty = True
|
||||||
|
|
||||||
|
def record_event(self, target_key, event):
|
||||||
|
|
||||||
|
s = self.states[target_key]
|
||||||
|
|
||||||
|
if event == "action":
|
||||||
|
s.actions += 1
|
||||||
|
|
||||||
|
elif event == "cache_hit":
|
||||||
|
s.cache_hits += 1
|
||||||
|
|
||||||
|
elif event == "failure":
|
||||||
|
s.failures += 1
|
||||||
|
|
||||||
|
s.dirty = True
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FAST SNAPSHOT (NO RECOMPUTE UNLESS DIRTY)
|
||||||
|
# -----------------------------
|
||||||
|
def snapshot(self, target_key):
|
||||||
|
|
||||||
|
s = self.states.get(target_key)
|
||||||
|
|
||||||
|
if not s:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RETURN CACHED RESULT
|
||||||
|
# -----------------------------
|
||||||
|
if not s.dirty and s.cached_score is not None:
|
||||||
|
return {
|
||||||
|
"score": s.cached_score,
|
||||||
|
"labels": s.cached_labels
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# COMPUTE AGGREGATES
|
||||||
|
# -----------------------------
|
||||||
|
avg_cpu = s.cpu_sum / s.samples if s.samples else 0
|
||||||
|
avg_temp = s.temp_sum / s.samples if s.samples else 0
|
||||||
|
|
||||||
|
cache_ratio = s.cache_hits / s.actions if s.actions else 0
|
||||||
|
|
||||||
|
profile = {
|
||||||
|
"duration": 0, # intentionally not scheduler-relevant anymore
|
||||||
|
"avg_cpu": avg_cpu,
|
||||||
|
"avg_temp": avg_temp,
|
||||||
|
"peak_temp": s.peak_temp,
|
||||||
|
"cache_ratio": cache_ratio,
|
||||||
|
"failures": s.failures
|
||||||
|
}
|
||||||
|
|
||||||
|
score = self.scorer.score(profile)
|
||||||
|
labels = self.labeler.label(score)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# CACHE RESULT
|
||||||
|
# -----------------------------
|
||||||
|
s.cached_score = score
|
||||||
|
s.cached_labels = labels
|
||||||
|
s.dirty = False
|
||||||
|
|
||||||
|
return {
|
||||||
|
"score": score,
|
||||||
|
"labels": labels
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FINALIZE (ARCHIVAL ONLY)
|
||||||
|
# -----------------------------
|
||||||
|
def finalize(self, target_key):
|
||||||
|
|
||||||
|
s = self.states[target_key]
|
||||||
|
|
||||||
|
snapshot = self.snapshot(target_key)
|
||||||
|
|
||||||
|
save({
|
||||||
|
"target": target_key,
|
||||||
|
"score": snapshot["score"],
|
||||||
|
"labels": snapshot["labels"]
|
||||||
|
})
|
||||||
|
|
||||||
|
return snapshot
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NodeState:
|
||||||
|
name: str
|
||||||
|
|
||||||
|
cpu_load: float = 0.0
|
||||||
|
memory_load: float = 0.0
|
||||||
|
temp: float = 0.0
|
||||||
|
instability: float = 0.0
|
||||||
|
|
||||||
|
active_jobs: int = 0
|
||||||
|
last_seen: float = 0.0
|
||||||
|
|
||||||
|
labels: Dict = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class NodeStateRegistry:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.nodes = {}
|
||||||
|
|
||||||
|
def update(self, name, **kwargs):
|
||||||
|
|
||||||
|
if name not in self.nodes:
|
||||||
|
self.nodes[name] = NodeState(name=name)
|
||||||
|
|
||||||
|
node = self.nodes[name]
|
||||||
|
|
||||||
|
for k, v in kwargs.items():
|
||||||
|
setattr(node, k, v)
|
||||||
|
|
||||||
|
def get(self, name):
|
||||||
|
return self.nodes.get(name)
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return list(self.nodes.values())
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
from backend.events.bus import EventBus
|
||||||
|
from backend.metrics.node_state import NodeStateRegistry
|
||||||
|
|
||||||
|
|
||||||
|
class ObservabilityHub:
|
||||||
|
"""
|
||||||
|
Single fusion layer for:
|
||||||
|
- events (what happened)
|
||||||
|
- metrics (what is happening)
|
||||||
|
- state (what is true now)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, registry: NodeStateRegistry, bus: EventBus):
|
||||||
|
self.registry = registry
|
||||||
|
self.bus = bus
|
||||||
|
self.subscribers = []
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STREAM OUT TO UI / PROMETHEUS / GRAFANA
|
||||||
|
# -----------------------------
|
||||||
|
def subscribe(self, fn):
|
||||||
|
self.subscribers.append(fn)
|
||||||
|
|
||||||
|
def emit(self, event: dict):
|
||||||
|
|
||||||
|
# enrich event with live state snapshot
|
||||||
|
if event.get("node"):
|
||||||
|
node = self.registry.get(event["node"])
|
||||||
|
if node:
|
||||||
|
event["node_state"] = {
|
||||||
|
"cpu": node.cpu_load,
|
||||||
|
"temp": node.temp,
|
||||||
|
"instability": node.instability,
|
||||||
|
"active_jobs": node.active_jobs,
|
||||||
|
}
|
||||||
|
|
||||||
|
for sub in self.subscribers:
|
||||||
|
sub(event)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# CONNECT EVENT BUS
|
||||||
|
# -----------------------------
|
||||||
|
def attach_bus(self):
|
||||||
|
|
||||||
|
def forward(event):
|
||||||
|
self.emit(event)
|
||||||
|
|
||||||
|
self.bus.subscribe(forward)
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
PROFILES = []
|
||||||
|
|
||||||
|
|
||||||
|
def save(profile):
|
||||||
|
PROFILES.append(profile)
|
||||||
|
|
||||||
|
|
||||||
|
def all():
|
||||||
|
return PROFILES
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
from prometheus_client import Gauge, Counter, start_http_server
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# METRICS
|
||||||
|
# -----------------------------
|
||||||
|
|
||||||
|
# Node metrics
|
||||||
|
node_cpu = Gauge("fester_node_cpu", "CPU load per node", ["node"])
|
||||||
|
node_temp = Gauge("fester_node_temp", "Temperature per node", ["node"])
|
||||||
|
|
||||||
|
# Pipeline metrics
|
||||||
|
pipeline_actions = Counter("fester_pipeline_actions_total", "Total actions", ["state"])
|
||||||
|
cache_hits = Counter("fester_cache_hits_total", "Cache hits", ["target"])
|
||||||
|
|
||||||
|
# Scheduler metrics
|
||||||
|
scheduler_score = Gauge("fester_scheduler_score", "Node score", ["node", "target"])
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# EXPORTER START
|
||||||
|
# -----------------------------
|
||||||
|
def start_metrics_server(port=9109):
|
||||||
|
start_http_server(port)
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
class TargetLabeler:
|
||||||
|
|
||||||
|
def label(self, score):
|
||||||
|
|
||||||
|
labels = []
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# THERMAL
|
||||||
|
# -----------------------------
|
||||||
|
if score["thermal"] > 80:
|
||||||
|
labels.append("HOT")
|
||||||
|
elif score["thermal"] < 60:
|
||||||
|
labels.append("COOL")
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# COST
|
||||||
|
# -----------------------------
|
||||||
|
if score["cost"] > 100:
|
||||||
|
labels.append("EXPENSIVE")
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STABILITY
|
||||||
|
# -----------------------------
|
||||||
|
if score["instability"] > 5:
|
||||||
|
labels.append("UNSTABLE")
|
||||||
|
|
||||||
|
if not labels:
|
||||||
|
labels.append("SAFE")
|
||||||
|
|
||||||
|
return labels
|
||||||
|
|
@ -0,0 +1,60 @@
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
class TargetProfile:
|
||||||
|
|
||||||
|
def __init__(self, target_key):
|
||||||
|
|
||||||
|
self.target = target_key
|
||||||
|
|
||||||
|
self.start_time = time.time()
|
||||||
|
|
||||||
|
self.cpu_samples = []
|
||||||
|
self.temp_samples = []
|
||||||
|
|
||||||
|
self.cache_hits = 0
|
||||||
|
self.actions = 0
|
||||||
|
self.failures = 0
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RECORD SAMPLE
|
||||||
|
# -----------------------------
|
||||||
|
def sample(self, node):
|
||||||
|
|
||||||
|
self.cpu_samples.append(node.get("cpu_load", 0))
|
||||||
|
self.temp_samples.append(node.get("temp", 0))
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# RECORD EVENTS
|
||||||
|
# -----------------------------
|
||||||
|
def record_action(self):
|
||||||
|
self.actions += 1
|
||||||
|
|
||||||
|
def record_cache_hit(self):
|
||||||
|
self.cache_hits += 1
|
||||||
|
|
||||||
|
def record_failure(self):
|
||||||
|
self.failures += 1
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# FINALIZE PROFILE
|
||||||
|
# -----------------------------
|
||||||
|
def finalize(self):
|
||||||
|
|
||||||
|
duration = time.time() - self.start_time
|
||||||
|
|
||||||
|
avg_cpu = sum(self.cpu_samples) / len(self.cpu_samples) if self.cpu_samples else 0
|
||||||
|
avg_temp = sum(self.temp_samples) / len(self.temp_samples) if self.temp_samples else 0
|
||||||
|
peak_temp = max(self.temp_samples) if self.temp_samples else 0
|
||||||
|
|
||||||
|
cache_ratio = self.cache_hits / self.actions if self.actions else 0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"target": self.target,
|
||||||
|
"duration": duration,
|
||||||
|
"avg_cpu": avg_cpu,
|
||||||
|
"avg_temp": avg_temp,
|
||||||
|
"peak_temp": peak_temp,
|
||||||
|
"cache_ratio": cache_ratio,
|
||||||
|
"failures": self.failures
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
class TargetScorer:
|
||||||
|
|
||||||
|
def score(self, profile):
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# THERMAL RISK SCORE (0–100)
|
||||||
|
# -----------------------------
|
||||||
|
thermal = (
|
||||||
|
profile["peak_temp"] * 0.5 +
|
||||||
|
profile["avg_temp"] * 0.5
|
||||||
|
)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# COST SCORE
|
||||||
|
# -----------------------------
|
||||||
|
cost = profile["duration"] * (1 - profile["cache_ratio"] + 0.1)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# STABILITY SCORE
|
||||||
|
# -----------------------------
|
||||||
|
instability = profile["failures"] * 10
|
||||||
|
|
||||||
|
return {
|
||||||
|
"thermal": thermal,
|
||||||
|
"cost": cost,
|
||||||
|
"instability": instability
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
"""
|
||||||
|
backend.nodes package.
|
||||||
|
|
||||||
|
Re-exports the canonical NodeState + NodeStateRegistry from
|
||||||
|
state_model.py for callers that do `from backend.nodes import NodeState`.
|
||||||
|
|
||||||
|
The legacy CLI script (formerly backend/nodes.py) is now at
|
||||||
|
backend/nodes_cli.py.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.nodes.state_model import NodeState, NodeStateRegistry
|
||||||
|
from backend.nodes.roles import NodeRole
|
||||||
|
from backend.nodes.roles_store import (
|
||||||
|
load_roles, save_roles, get_role, DEFAULT_ROLE, ROLE_DB,
|
||||||
|
)
|
||||||
|
from backend.nodes.probe import probe_sync, probe_async, probe_node, normalize
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"NodeState",
|
||||||
|
"NodeStateRegistry",
|
||||||
|
"NodeRole",
|
||||||
|
"load_roles",
|
||||||
|
"save_roles",
|
||||||
|
"get_role",
|
||||||
|
"DEFAULT_ROLE",
|
||||||
|
"ROLE_DB",
|
||||||
|
"probe_sync",
|
||||||
|
"probe_async",
|
||||||
|
"probe_node",
|
||||||
|
"normalize",
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,126 @@
|
||||||
|
"""
|
||||||
|
Node probe — query a remote fester-agent for live state.
|
||||||
|
|
||||||
|
Each cluster node is expected to run an HTTP agent on port 8787 (the
|
||||||
|
convention established by backend/nodes_cli.py) that returns JSON like:
|
||||||
|
|
||||||
|
{
|
||||||
|
"state": "online",
|
||||||
|
"load": "0.42 0.51 0.48 2/128 1234",
|
||||||
|
"temp": 62,
|
||||||
|
"jobs": 4,
|
||||||
|
"max_jobs": 24,
|
||||||
|
"memory": {"used": 8.2, "total": 32.0, "unit": "GB"},
|
||||||
|
"labels": {"arch": "x86_64", "role": "compile"}
|
||||||
|
}
|
||||||
|
|
||||||
|
If the agent is unreachable, the probe returns None and the caller can
|
||||||
|
fall back to synthetic drift.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
try:
|
||||||
|
import aiohttp
|
||||||
|
_HAVE_AIOHTTP = True
|
||||||
|
except ImportError:
|
||||||
|
_HAVE_AIOHTTP = False
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
AGENT_PORT = 8787
|
||||||
|
DEFAULT_TIMEOUT = 1.5 # seconds
|
||||||
|
|
||||||
|
|
||||||
|
def probe_sync(host: str, port: int = AGENT_PORT, timeout: float = DEFAULT_TIMEOUT) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Synchronous probe — for use outside the event loop."""
|
||||||
|
if not host:
|
||||||
|
return None
|
||||||
|
url = f"http://{host}:{port}/status"
|
||||||
|
try:
|
||||||
|
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
return None
|
||||||
|
import json
|
||||||
|
return json.loads(resp.read().decode("utf-8"))
|
||||||
|
except (urllib.error.URLError, ConnectionError, TimeoutError, OSError):
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def probe_async(host: str, port: int = AGENT_PORT, timeout: float = DEFAULT_TIMEOUT) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Async probe — preferred when running inside FastAPI / asyncio."""
|
||||||
|
if not host:
|
||||||
|
return None
|
||||||
|
url = f"http://{host}:{port}/status"
|
||||||
|
if _HAVE_AIOHTTP:
|
||||||
|
try:
|
||||||
|
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
|
||||||
|
async with session.get(url) as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
return None
|
||||||
|
return await resp.json()
|
||||||
|
except (asyncio.TimeoutError, aiohttp.ClientError, OSError):
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
# Fallback: run sync probe in a thread
|
||||||
|
return await asyncio.to_thread(probe_sync, host, port, timeout)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(agent_data: Optional[Dict[str, Any]], node_name: str) -> Dict[str, Any]:
|
||||||
|
"""Convert an agent's response into a normalized NodeState shape.
|
||||||
|
|
||||||
|
If agent_data is None (probe failed), returns a minimal offline shape."""
|
||||||
|
if not agent_data:
|
||||||
|
return {
|
||||||
|
"name": node_name,
|
||||||
|
"state": "offline",
|
||||||
|
"cpu_load": 0.0,
|
||||||
|
"memory_load": 0.0,
|
||||||
|
"temp": 0.0,
|
||||||
|
"instability": 1.0, # mark unstable since we can't reach it
|
||||||
|
"active_jobs": 0,
|
||||||
|
"last_seen": 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Parse load average string like "0.42 0.51 0.48 2/128 1234"
|
||||||
|
load_str = agent_data.get("load", "0 0 0")
|
||||||
|
try:
|
||||||
|
cpu_load = float(str(load_str).split()[0]) * 100 # 1.0 = 100%
|
||||||
|
except (ValueError, IndexError):
|
||||||
|
cpu_load = 0.0
|
||||||
|
|
||||||
|
# Memory: agent reports {used, total, unit} in GB
|
||||||
|
mem = agent_data.get("memory", {})
|
||||||
|
if isinstance(mem, dict) and mem.get("total"):
|
||||||
|
memory_load = (float(mem.get("used", 0)) / float(mem["total"])) * 100
|
||||||
|
else:
|
||||||
|
memory_load = 0.0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"name": node_name,
|
||||||
|
"state": agent_data.get("state", "online"),
|
||||||
|
"cpu_load": cpu_load,
|
||||||
|
"memory_load": memory_load,
|
||||||
|
"temp": float(agent_data.get("temp", 0)),
|
||||||
|
"instability": float(agent_data.get("instability", 0.0)),
|
||||||
|
"active_jobs": int(agent_data.get("jobs", 0)),
|
||||||
|
"max_jobs": int(agent_data.get("max_jobs", 0)),
|
||||||
|
"labels": agent_data.get("labels", {}),
|
||||||
|
"last_seen": time.time(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def probe_node(name: str, host: str, port: int = AGENT_PORT) -> Dict[str, Any]:
|
||||||
|
"""Probe one node and return normalized state.
|
||||||
|
Always returns a dict (offline if probe failed)."""
|
||||||
|
raw = await probe_async(host, port)
|
||||||
|
return normalize(raw, name)
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NodeRole:
|
||||||
|
name: str
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# cache behavior
|
||||||
|
# ----------------------------
|
||||||
|
cache_writer: bool = True # can store artifacts
|
||||||
|
cache_reader: bool = True # can fetch artifacts
|
||||||
|
cache_priority: int = 50 # higher = preferred cache source
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# compute behavior
|
||||||
|
# ----------------------------
|
||||||
|
compile_weight: float = 1.0 # scheduler weight multiplier
|
||||||
|
distcc_enabled: bool = True
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# thermal safety
|
||||||
|
# ----------------------------
|
||||||
|
max_thermal_state: float = 0.85
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# storage capabilities
|
||||||
|
# ----------------------------
|
||||||
|
tmpfs_ok: bool = True
|
||||||
|
btrfs_ok: bool = False
|
||||||
|
qcow2_ok: bool = False
|
||||||
|
|
@ -0,0 +1,79 @@
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from backend.nodes.roles import NodeRole
|
||||||
|
|
||||||
|
|
||||||
|
# Where node role overrides are persisted. Allow override via env var;
|
||||||
|
# otherwise default to a user-writable path under /var/lib/fester (or
|
||||||
|
# fall back to a tmp dir if neither is writable).
|
||||||
|
def _resolve_role_db() -> Path:
|
||||||
|
env = os.environ.get("FESTER_ROLE_DB")
|
||||||
|
if env:
|
||||||
|
return Path(env)
|
||||||
|
|
||||||
|
candidates = [
|
||||||
|
Path("/etc/fester/node_roles.json"),
|
||||||
|
Path("/var/lib/fester/node_roles.json"),
|
||||||
|
Path.home() / ".fester" / "node_roles.json",
|
||||||
|
Path("/tmp/fester_node_roles.json"),
|
||||||
|
]
|
||||||
|
for c in candidates:
|
||||||
|
try:
|
||||||
|
c.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
# touch to verify writability
|
||||||
|
if not c.exists():
|
||||||
|
c.write_text("{}")
|
||||||
|
else:
|
||||||
|
# verify we can write to it
|
||||||
|
with open(c, "a"):
|
||||||
|
pass
|
||||||
|
return c
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
continue
|
||||||
|
# Last resort: in-memory only (caller must handle None)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
ROLE_DB = _resolve_role_db()
|
||||||
|
|
||||||
|
DEFAULT_ROLE = NodeRole(name="default")
|
||||||
|
|
||||||
|
|
||||||
|
def load_roles():
|
||||||
|
if ROLE_DB is None or not ROLE_DB.exists():
|
||||||
|
return {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw = json.loads(ROLE_DB.read_text())
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
roles = {}
|
||||||
|
|
||||||
|
for node, cfg in raw.items():
|
||||||
|
try:
|
||||||
|
roles[node] = NodeRole(**cfg)
|
||||||
|
except TypeError:
|
||||||
|
# Schema drift — skip malformed entries
|
||||||
|
continue
|
||||||
|
|
||||||
|
return roles
|
||||||
|
|
||||||
|
|
||||||
|
def save_roles(roles):
|
||||||
|
if ROLE_DB is None:
|
||||||
|
return False
|
||||||
|
raw = {
|
||||||
|
node: role.__dict__
|
||||||
|
for node, role in roles.items()
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
ROLE_DB.write_text(json.dumps(raw, indent=2))
|
||||||
|
return True
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_role(node_name, roles):
|
||||||
|
return roles.get(node_name, DEFAULT_ROLE)
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Dict, Any, Optional
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NodeState:
|
||||||
|
"""
|
||||||
|
Single source of truth for node health + capability.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
|
||||||
|
# runtime
|
||||||
|
cpu_load: float = 0.0
|
||||||
|
memory_load: float = 0.0
|
||||||
|
temp: float = 0.0
|
||||||
|
|
||||||
|
# scheduling intelligence
|
||||||
|
score: float = 0.0
|
||||||
|
instability: float = 0.0
|
||||||
|
|
||||||
|
# execution tracking
|
||||||
|
active_jobs: int = 0
|
||||||
|
last_seen: float = 0.0
|
||||||
|
|
||||||
|
# capabilities
|
||||||
|
labels: Dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
# health flags
|
||||||
|
healthy: bool = True
|
||||||
|
degraded: bool = False
|
||||||
|
offline: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class NodeStateRegistry:
|
||||||
|
"""
|
||||||
|
Central authority for node truth.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.nodes: Dict[str, NodeState] = {}
|
||||||
|
|
||||||
|
def update(self, name: str, **kwargs):
|
||||||
|
if name not in self.nodes:
|
||||||
|
self.nodes[name] = NodeState(name=name)
|
||||||
|
|
||||||
|
for k, v in kwargs.items():
|
||||||
|
setattr(self.nodes[name], k, v)
|
||||||
|
|
||||||
|
def get(self, name: str) -> Optional[NodeState]:
|
||||||
|
return self.nodes.get(name)
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return list(self.nodes.values())
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
from backend.config import CONFIG
|
||||||
|
from backend.scheduler import build_distcc_hosts
|
||||||
|
|
||||||
|
AGENT_PORT = 8787
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_agent(node):
|
||||||
|
try:
|
||||||
|
r = requests.get(f"http://{node['host']}:{AGENT_PORT}/status", timeout=1.5)
|
||||||
|
return r.json()
|
||||||
|
except:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def enrich(node):
|
||||||
|
agent = fetch_agent(node)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"name": node["name"],
|
||||||
|
"host": node["host"],
|
||||||
|
"state": "online" if agent else "offline",
|
||||||
|
"agent": agent,
|
||||||
|
"release_ready": agent is not None
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
nodes = [enrich(n) for n in CONFIG["nodes"]]
|
||||||
|
|
||||||
|
output = {
|
||||||
|
"master": CONFIG["master"],
|
||||||
|
"nodes": nodes,
|
||||||
|
"distcc": build_distcc_hosts(nodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
print(json.dumps(output, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,226 @@
|
||||||
|
"""
|
||||||
|
PipelineEngine — executes a build DAG.
|
||||||
|
|
||||||
|
For each action:
|
||||||
|
1. Schedule: pick best node via scheduler
|
||||||
|
2. Emit task_update(scheduled) with deps + critical info
|
||||||
|
3. Check cache: if hit, emit cache_update + skip execution
|
||||||
|
4. Execute: emit task_update(running), run via runtime router, emit task_update(done|failed)
|
||||||
|
5. On failure: emit failure event + stop downstream actions
|
||||||
|
|
||||||
|
Emits events on the bus so the UI / WS stream / cause graph / timeline store
|
||||||
|
all pick them up.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from backend.events.bus import EventBus
|
||||||
|
from backend.events.schema import EventType
|
||||||
|
from backend.scheduler.optimizer import choose_best_node
|
||||||
|
from backend.executor.runtime_router import execute_action
|
||||||
|
from backend.cache.minio_cache import MinioCache
|
||||||
|
from backend.graph.plan import build_action_graph
|
||||||
|
from backend.graph.critical_path import compute_critical_path
|
||||||
|
|
||||||
|
|
||||||
|
class PipelineEngine:
|
||||||
|
"""Runs a build DAG against a cluster of nodes.
|
||||||
|
|
||||||
|
Constructor:
|
||||||
|
nodes: list of node dicts (config + state)
|
||||||
|
node_registry: NodeStateRegistry singleton (for live state lookups)
|
||||||
|
event_bus: EventBus singleton (for emitting events)
|
||||||
|
cache: optional MinioCache (or compatible). If None, no cache.
|
||||||
|
build_id: optional build identifier (for correlating events)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, nodes, node_registry, event_bus: EventBus,
|
||||||
|
cache: Optional[Any] = None, build_id: Optional[str] = None):
|
||||||
|
self.nodes = nodes
|
||||||
|
self.node_registry = node_registry
|
||||||
|
self.bus = event_bus
|
||||||
|
self.cache = cache # may be None
|
||||||
|
self.build_id = build_id or f"build-{int(time.time())}"
|
||||||
|
|
||||||
|
# Pause / step state (used by debugger)
|
||||||
|
self._paused = False
|
||||||
|
self._step_mode = False
|
||||||
|
self._step_event = asyncio.Event() if asyncio.get_event_loop() else None
|
||||||
|
|
||||||
|
# Results
|
||||||
|
self.last_results: List[tuple] = []
|
||||||
|
self.critical_path: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# DEBUGGER HOOKS
|
||||||
|
# -------------------------------------------------
|
||||||
|
def pause(self):
|
||||||
|
self._paused = True
|
||||||
|
|
||||||
|
def resume(self):
|
||||||
|
self._paused = False
|
||||||
|
if self._step_event:
|
||||||
|
self._step_event.set()
|
||||||
|
|
||||||
|
def step(self):
|
||||||
|
"""Advance one action when paused."""
|
||||||
|
self._step_mode = True
|
||||||
|
if self._step_event:
|
||||||
|
self._step_event.set()
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# MAIN ENTRYPOINT
|
||||||
|
# -------------------------------------------------
|
||||||
|
async def run(self, project) -> List[tuple]:
|
||||||
|
"""Build the project. Returns list of (action_name, state) tuples."""
|
||||||
|
|
||||||
|
actions = build_action_graph(project)
|
||||||
|
self.critical_path = compute_critical_path(actions)
|
||||||
|
critical_names = set(self.critical_path.get("score_map", {}).keys()) if self.critical_path else set()
|
||||||
|
|
||||||
|
self.last_results = []
|
||||||
|
completed: Dict[str, str] = {} # name -> state
|
||||||
|
failed_actions: set = set()
|
||||||
|
|
||||||
|
for action in actions:
|
||||||
|
# Skip if any dependency failed
|
||||||
|
deps = action.get("deps", [])
|
||||||
|
failed_deps = [d for d in deps if completed.get(d) == "failed"]
|
||||||
|
if failed_deps:
|
||||||
|
self.bus.emit(
|
||||||
|
EventType.TASK_UPDATE,
|
||||||
|
action=action["name"],
|
||||||
|
node=None,
|
||||||
|
state="skipped",
|
||||||
|
reason=f"deps_failed:{','.join(failed_deps)}",
|
||||||
|
meta={
|
||||||
|
"deps": deps,
|
||||||
|
"critical": action["name"] in critical_names,
|
||||||
|
"target": action.get("target", "native"),
|
||||||
|
"build_id": self.build_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
completed[action["name"]] = "skipped"
|
||||||
|
self.last_results.append((action["name"], "skipped"))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Wait if paused
|
||||||
|
await self._wait_if_paused()
|
||||||
|
|
||||||
|
# Schedule
|
||||||
|
node = choose_best_node(self.nodes, action, self.node_registry)
|
||||||
|
node_name = node["name"] if node else None
|
||||||
|
|
||||||
|
self.bus.emit(
|
||||||
|
EventType.TASK_UPDATE,
|
||||||
|
action=action["name"],
|
||||||
|
node=node_name,
|
||||||
|
state="scheduled",
|
||||||
|
meta={
|
||||||
|
"deps": deps,
|
||||||
|
"critical": action["name"] in critical_names,
|
||||||
|
"target": action.get("target", "native"),
|
||||||
|
"build_id": self.build_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Cache check
|
||||||
|
cache_hit = False
|
||||||
|
if self.cache and action.get("hash"):
|
||||||
|
try:
|
||||||
|
if self.cache.exists(action["hash"]):
|
||||||
|
cache_hit = True
|
||||||
|
self.bus.emit(
|
||||||
|
EventType.CACHE_UPDATE,
|
||||||
|
action=action["name"],
|
||||||
|
node=node_name,
|
||||||
|
state="hit",
|
||||||
|
meta={"build_id": self.build_id},
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Cache errors are non-fatal
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Emit running
|
||||||
|
self.bus.emit(
|
||||||
|
EventType.TASK_UPDATE,
|
||||||
|
action=action["name"],
|
||||||
|
node=node_name,
|
||||||
|
state="running",
|
||||||
|
meta={
|
||||||
|
"deps": deps,
|
||||||
|
"critical": action["name"] in critical_names,
|
||||||
|
"target": action.get("target", "native"),
|
||||||
|
"cache": "hit" if cache_hit else None,
|
||||||
|
"build_id": self.build_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if not cache_hit:
|
||||||
|
# Execute (in a thread so we don't block the event loop)
|
||||||
|
# Pass action itself as workspace hint — execute_action reads
|
||||||
|
# action["dir"] and action["env"] for proper cwd + env setup.
|
||||||
|
try:
|
||||||
|
rc = await asyncio.to_thread(
|
||||||
|
execute_action, action, action.get("dir") or "/tmp",
|
||||||
|
node or {"name": "localhost"}
|
||||||
|
)
|
||||||
|
state = "done" if rc == 0 else "failed"
|
||||||
|
except Exception as e:
|
||||||
|
state = "failed"
|
||||||
|
rc = -1
|
||||||
|
self.bus.emit(
|
||||||
|
EventType.FAILURE,
|
||||||
|
action=action["name"],
|
||||||
|
node=node_name,
|
||||||
|
state="failed",
|
||||||
|
reason=f"execution_exception:{type(e).__name__}:{e}",
|
||||||
|
meta={"build_id": self.build_id},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
state = "done"
|
||||||
|
rc = 0
|
||||||
|
|
||||||
|
# Emit terminal state
|
||||||
|
self.bus.emit(
|
||||||
|
EventType.TASK_UPDATE,
|
||||||
|
action=action["name"],
|
||||||
|
node=node_name,
|
||||||
|
state=state,
|
||||||
|
meta={
|
||||||
|
"deps": deps,
|
||||||
|
"critical": action["name"] in critical_names,
|
||||||
|
"target": action.get("target", "native"),
|
||||||
|
"cache": "hit" if cache_hit else None,
|
||||||
|
"build_id": self.build_id,
|
||||||
|
"rc": rc,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
completed[action["name"]] = state
|
||||||
|
self.last_results.append((action["name"], state))
|
||||||
|
|
||||||
|
if state == "failed":
|
||||||
|
failed_actions.add(action["name"])
|
||||||
|
|
||||||
|
# Brief yield so the WS broadcast has time to fire
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
|
||||||
|
return self.last_results
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# INTERNAL
|
||||||
|
# -------------------------------------------------
|
||||||
|
async def _wait_if_paused(self):
|
||||||
|
if not self._paused:
|
||||||
|
return
|
||||||
|
# Wait until resumed (or stepped)
|
||||||
|
if self._step_event is None:
|
||||||
|
self._step_event = asyncio.Event()
|
||||||
|
self._step_event.clear()
|
||||||
|
while self._paused and not self._step_mode:
|
||||||
|
await self._step_event.wait()
|
||||||
|
self._step_event.clear()
|
||||||
|
self._step_mode = False
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
"""
|
||||||
|
Pipeline feedback — feeds execution results back into the policy engine
|
||||||
|
so it can learn which nodes / actions succeed vs fail.
|
||||||
|
|
||||||
|
Currently the PolicyEngine doesn't have a `update_reputation` method
|
||||||
|
(see backend/policy/engine.py for the simplified evaluate() interface).
|
||||||
|
This module is kept as a stub for future learning-loop work.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.policy.engine import PolicyEngine
|
||||||
|
|
||||||
|
# Singleton policy engine (db=None — no learning yet)
|
||||||
|
policy = PolicyEngine(db=None)
|
||||||
|
|
||||||
|
|
||||||
|
def report_execution(node, action, success, duration, temp_before, temp_after):
|
||||||
|
"""Feed execution results back to the policy engine.
|
||||||
|
|
||||||
|
In the future this should update per-node reputation scores so the
|
||||||
|
scheduler can prefer nodes that historically succeed and avoid nodes
|
||||||
|
that overheat. For now it's a no-op logging shim.
|
||||||
|
"""
|
||||||
|
thermal_spike = (temp_after - temp_before) > 0.15
|
||||||
|
|
||||||
|
# Future: policy.update_reputation(node, success, duration, thermal_spike)
|
||||||
|
# For now, just return the assessment
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"action": action,
|
||||||
|
"success": success,
|
||||||
|
"duration": duration,
|
||||||
|
"thermal_spike": thermal_spike,
|
||||||
|
"temp_delta": temp_after - temp_before,
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,357 @@
|
||||||
|
"""
|
||||||
|
BuildRunner — manages build lifecycle.
|
||||||
|
|
||||||
|
Wraps PipelineEngine with:
|
||||||
|
- Build ID generation
|
||||||
|
- State tracking (queued, running, complete, failed)
|
||||||
|
- Async task management (so /api/build returns immediately)
|
||||||
|
- Optional persistence via a storage backend
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
runner = BuildRunner(bus, registry, nodes)
|
||||||
|
build_id = await runner.start(BuildBody(cmd="make -j4", dir="/tmp/foo"))
|
||||||
|
state = runner.state(build_id)
|
||||||
|
all_builds = runner.list_builds()
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass, field, asdict
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from backend.events.bus import EventBus
|
||||||
|
from backend.events.schema import EventType
|
||||||
|
from backend.pipeline.engine import PipelineEngine
|
||||||
|
from backend.cache.minio_cache import MinioCache
|
||||||
|
|
||||||
|
|
||||||
|
# Optional storage backend
|
||||||
|
try:
|
||||||
|
from backend.storage.sqlite_db import STORAGE
|
||||||
|
except Exception:
|
||||||
|
STORAGE = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class BuildRecord:
|
||||||
|
build_id: str
|
||||||
|
cmd: str
|
||||||
|
dir: str
|
||||||
|
project: Optional[str] = None
|
||||||
|
arch: str = "x86_64"
|
||||||
|
target: str = "linux-gnu"
|
||||||
|
toolchain: str = "gcc"
|
||||||
|
started_at: float = field(default_factory=time.time)
|
||||||
|
ended_at: Optional[float] = None
|
||||||
|
status: str = "queued" # queued | running | complete | failed
|
||||||
|
actions: List[Dict[str, str]] = field(default_factory=list)
|
||||||
|
error: Optional[str] = None
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
class BuildRunner:
|
||||||
|
"""Manages build lifecycle and tracks all builds in memory (or via storage)."""
|
||||||
|
|
||||||
|
def __init__(self, bus: EventBus, registry, nodes: List[Dict[str, Any]],
|
||||||
|
storage=None):
|
||||||
|
self.bus = bus
|
||||||
|
self.registry = registry
|
||||||
|
self.nodes = nodes
|
||||||
|
self.storage = storage if storage is not None else STORAGE # may still be None
|
||||||
|
self._builds: Dict[str, BuildRecord] = {}
|
||||||
|
self._tasks: Dict[str, asyncio.Task] = {}
|
||||||
|
self._engines: Dict[str, PipelineEngine] = {} # build_id -> engine
|
||||||
|
self._cache: Optional[MinioCache] = None # lazy init
|
||||||
|
|
||||||
|
# On startup, load any persisted builds into the in-memory cache
|
||||||
|
if self.storage:
|
||||||
|
self.load_from_storage()
|
||||||
|
|
||||||
|
# Wire engines into debugger + pipeline_control so they can
|
||||||
|
# pause/resume/step the most-recently-started build.
|
||||||
|
self._wire_debugger()
|
||||||
|
|
||||||
|
def _wire_debugger(self):
|
||||||
|
"""Register our engine getter with the debugger + pipeline_control modules."""
|
||||||
|
try:
|
||||||
|
from backend.api import debugger as dbg
|
||||||
|
from backend.api import pipeline_control as pc
|
||||||
|
# Replace their _ENGINE singletons with a proxy that always
|
||||||
|
# points at the most recent running build's engine.
|
||||||
|
proxy = _EngineProxy(self)
|
||||||
|
dbg.set_engine(proxy)
|
||||||
|
pc.set_engine(proxy)
|
||||||
|
except Exception as e:
|
||||||
|
# Don't crash if debugger/pipeline_control aren't importable
|
||||||
|
pass
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# CACHE
|
||||||
|
# -------------------------------------------------
|
||||||
|
def _get_cache(self) -> Optional[MinioCache]:
|
||||||
|
"""Lazily construct a MinioCache. Returns None if MinIO is unreachable
|
||||||
|
or not configured."""
|
||||||
|
if self._cache is not None:
|
||||||
|
return self._cache
|
||||||
|
try:
|
||||||
|
self._cache = MinioCache("localhost:9000", "minioadmin", "minioadmin")
|
||||||
|
# Force connection test
|
||||||
|
self._cache._ensure_client()
|
||||||
|
return self._cache
|
||||||
|
except Exception:
|
||||||
|
self._cache = None
|
||||||
|
return None
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# START A BUILD
|
||||||
|
# -------------------------------------------------
|
||||||
|
async def start(self, body) -> str:
|
||||||
|
"""Queue a build and return its ID. The actual run happens in a background task."""
|
||||||
|
build_id = str(uuid.uuid4())[:8]
|
||||||
|
record = BuildRecord(
|
||||||
|
build_id=build_id,
|
||||||
|
cmd=body.cmd,
|
||||||
|
dir=body.dir,
|
||||||
|
project=getattr(body, "project", None),
|
||||||
|
arch=getattr(body, "arch", "x86_64"),
|
||||||
|
target=getattr(body, "target", "linux-gnu"),
|
||||||
|
toolchain=getattr(body, "toolchain", "gcc"),
|
||||||
|
)
|
||||||
|
record.status = "running"
|
||||||
|
self._builds[build_id] = record
|
||||||
|
self._persist(record)
|
||||||
|
|
||||||
|
# Launch in background
|
||||||
|
task = asyncio.create_task(self._run_build(build_id))
|
||||||
|
self._tasks[build_id] = task
|
||||||
|
return build_id
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# BUILD EXECUTION
|
||||||
|
# -------------------------------------------------
|
||||||
|
async def _run_build(self, build_id: str):
|
||||||
|
record = self._builds[build_id]
|
||||||
|
try:
|
||||||
|
# Construct a project spec the engine can consume
|
||||||
|
project = self._build_project_spec(record)
|
||||||
|
|
||||||
|
# Choose execution mode
|
||||||
|
if record.cmd and record.cmd.startswith("make"):
|
||||||
|
# Real build — use the actual engine
|
||||||
|
engine = PipelineEngine(
|
||||||
|
nodes=self.nodes,
|
||||||
|
node_registry=self.registry,
|
||||||
|
event_bus=self.bus,
|
||||||
|
cache=self._get_cache(),
|
||||||
|
build_id=build_id,
|
||||||
|
)
|
||||||
|
self._engines[build_id] = engine
|
||||||
|
try:
|
||||||
|
results = await engine.run(project)
|
||||||
|
finally:
|
||||||
|
# Engine done — drop from active set
|
||||||
|
self._engines.pop(build_id, None)
|
||||||
|
|
||||||
|
# Record results
|
||||||
|
record.actions = [{"name": n, "state": s} for n, s in results]
|
||||||
|
record.status = "failed" if any(s == "failed" for _, s in results) else "complete"
|
||||||
|
else:
|
||||||
|
# Synthetic pipeline for non-shell build commands (demo mode)
|
||||||
|
await self._run_synthetic(build_id, record)
|
||||||
|
record.status = "complete"
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
record.status = "failed"
|
||||||
|
record.error = f"{type(e).__name__}: {e}"
|
||||||
|
self.bus.emit(
|
||||||
|
EventType.FAILURE,
|
||||||
|
action=None,
|
||||||
|
node=None,
|
||||||
|
state="failed",
|
||||||
|
reason=f"build_crashed:{record.error}",
|
||||||
|
meta={"build_id": build_id},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
record.ended_at = time.time()
|
||||||
|
self._persist(record)
|
||||||
|
|
||||||
|
def _build_project_spec(self, record: BuildRecord) -> Dict[str, Any]:
|
||||||
|
"""Convert a BuildRecord into a project spec the engine understands."""
|
||||||
|
return {
|
||||||
|
"source": record.project or "local",
|
||||||
|
"default_target": record.target,
|
||||||
|
"build_dir": record.dir, # passed through to actions as `dir`
|
||||||
|
"build_env": {
|
||||||
|
"ARCH": record.arch,
|
||||||
|
"TARGET": record.target,
|
||||||
|
"CC": record.toolchain,
|
||||||
|
"DISTCC_HOSTS": " ".join(
|
||||||
|
n.get("host", "") for n in self.nodes if n.get("state") == "online"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"targets": {
|
||||||
|
record.target: record.cmd,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _run_synthetic(self, build_id: str, record: BuildRecord):
|
||||||
|
"""Fallback synthetic pipeline — emits a realistic-looking stream of
|
||||||
|
events without actually running any commands. Used when the build
|
||||||
|
command isn't a real shell command (e.g. demo / smoke test)."""
|
||||||
|
import random
|
||||||
|
actions = [
|
||||||
|
("fetch_source", []),
|
||||||
|
("configure", ["fetch_source"]),
|
||||||
|
("build_kernel", ["configure"]),
|
||||||
|
("build_modules", ["configure"]),
|
||||||
|
("build_initramfs", ["configure"]),
|
||||||
|
("build_debian", ["build_kernel", "build_modules"]),
|
||||||
|
("package", ["build_debian"]),
|
||||||
|
("release", ["package"]),
|
||||||
|
]
|
||||||
|
critical = {"fetch_source", "configure", "build_kernel", "build_debian", "package", "release"}
|
||||||
|
|
||||||
|
for action, deps in actions:
|
||||||
|
node_name = random.choice([n["name"] for n in self.nodes if n.get("state") != "offline"]) if self.nodes else "localhost"
|
||||||
|
|
||||||
|
self.bus.emit(EventType.TASK_UPDATE, action=action, node=node_name, state="scheduled",
|
||||||
|
meta={"deps": deps, "critical": action in critical, "target": record.target, "build_id": build_id})
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
|
|
||||||
|
self.bus.emit(EventType.TASK_UPDATE, action=action, node=node_name, state="running",
|
||||||
|
meta={"deps": deps, "critical": action in critical, "target": record.target, "build_id": build_id})
|
||||||
|
await asyncio.sleep(random.uniform(0.4, 1.0))
|
||||||
|
|
||||||
|
cache_hit = random.random() < 0.25
|
||||||
|
self.bus.emit(EventType.TASK_UPDATE, action=action, node=node_name, state="done",
|
||||||
|
meta={"deps": deps, "cache": "hit" if cache_hit else None, "critical": action in critical, "target": record.target, "build_id": build_id})
|
||||||
|
if cache_hit:
|
||||||
|
self.bus.emit(EventType.CACHE_UPDATE, action=action, node=node_name, state="hit",
|
||||||
|
meta={"build_id": build_id})
|
||||||
|
record.actions.append({"name": action, "state": "done"})
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# QUERY
|
||||||
|
# -------------------------------------------------
|
||||||
|
def state(self, build_id: str) -> Optional[Dict[str, Any]]:
|
||||||
|
rec = self._builds.get(build_id)
|
||||||
|
if rec:
|
||||||
|
return rec.to_dict()
|
||||||
|
if self.storage:
|
||||||
|
try:
|
||||||
|
return self.storage.get_build(build_id)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
def list_builds(self) -> List[Dict[str, Any]]:
|
||||||
|
# Merge in-memory + storage (in-memory takes precedence)
|
||||||
|
in_mem = {b.build_id: b.to_dict() for b in self._builds.values()}
|
||||||
|
if not self.storage:
|
||||||
|
return list(in_mem.values())
|
||||||
|
try:
|
||||||
|
stored = {b["build_id"]: b for b in self.storage.list_builds()}
|
||||||
|
except Exception:
|
||||||
|
stored = {}
|
||||||
|
# Merge: in-memory wins (it has fresher state for running builds)
|
||||||
|
merged = {**stored, **in_mem}
|
||||||
|
return sorted(merged.values(), key=lambda b: b.get("started_at", 0), reverse=True)
|
||||||
|
|
||||||
|
def cancel(self, build_id: str) -> bool:
|
||||||
|
task = self._tasks.get(build_id)
|
||||||
|
if task and not task.done():
|
||||||
|
task.cancel()
|
||||||
|
rec = self._builds.get(build_id)
|
||||||
|
if rec:
|
||||||
|
rec.status = "cancelled"
|
||||||
|
rec.ended_at = time.time()
|
||||||
|
self._persist(rec)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# DEBUGGER CONTROL
|
||||||
|
# -------------------------------------------------
|
||||||
|
def get_engine(self, build_id: Optional[str] = None) -> Optional[PipelineEngine]:
|
||||||
|
"""Look up the running engine for a build.
|
||||||
|
If build_id is None, returns the most recently started engine."""
|
||||||
|
if build_id:
|
||||||
|
return self._engines.get(build_id)
|
||||||
|
# Return the most recent (last-inserted) engine
|
||||||
|
if not self._engines:
|
||||||
|
return None
|
||||||
|
return list(self._engines.values())[-1]
|
||||||
|
|
||||||
|
def active_build_ids(self) -> List[str]:
|
||||||
|
"""Return IDs of builds with engines currently running."""
|
||||||
|
return list(self._engines.keys())
|
||||||
|
|
||||||
|
# -------------------------------------------------
|
||||||
|
# PERSISTENCE
|
||||||
|
# -------------------------------------------------
|
||||||
|
def _persist(self, record: BuildRecord):
|
||||||
|
if self.storage:
|
||||||
|
try:
|
||||||
|
self.storage.save_build(record.to_dict())
|
||||||
|
except Exception:
|
||||||
|
pass # storage errors are non-fatal
|
||||||
|
|
||||||
|
def load_from_storage(self):
|
||||||
|
"""On startup, load any persisted builds into the in-memory cache."""
|
||||||
|
if not self.storage:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
builds = self.storage.list_builds()
|
||||||
|
for b in builds:
|
||||||
|
# Don't overwrite running builds (they're stale from a crashed process)
|
||||||
|
if b.get("status") == "running":
|
||||||
|
b["status"] = "crashed"
|
||||||
|
b["error"] = "process restarted while build was running"
|
||||||
|
rec = BuildRecord(**{k: v for k, v in b.items() if k in BuildRecord.__dataclass_fields__})
|
||||||
|
self._builds[rec.build_id] = rec
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _EngineProxy:
|
||||||
|
"""Stand-in object that the debugger + pipeline_control modules can call
|
||||||
|
pause/resume/step on. Forwards calls to the BuildRunner's most-recent
|
||||||
|
active engine, so debugger buttons affect the currently-running build."""
|
||||||
|
|
||||||
|
def __init__(self, runner: BuildRunner):
|
||||||
|
self._runner = runner
|
||||||
|
|
||||||
|
def _engine(self) -> Optional[PipelineEngine]:
|
||||||
|
return self._runner.get_engine()
|
||||||
|
|
||||||
|
def pause(self):
|
||||||
|
eng = self._engine()
|
||||||
|
if eng:
|
||||||
|
eng.pause()
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def resume(self):
|
||||||
|
eng = self._engine()
|
||||||
|
if eng:
|
||||||
|
eng.resume()
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def step(self):
|
||||||
|
eng = self._engine()
|
||||||
|
if eng:
|
||||||
|
eng.step()
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def step_back(self):
|
||||||
|
# Step-back is replay-only; live engine can't undo
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_active(self) -> bool:
|
||||||
|
return self._engine() is not None
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
|
||||||
|
class State(Enum):
|
||||||
|
PENDING = "pending"
|
||||||
|
PREFLIGHT = "preflight"
|
||||||
|
SCHEDULED = "scheduled"
|
||||||
|
EXECUTING = "executing"
|
||||||
|
|
||||||
|
CACHE_HIT = "cache_hit"
|
||||||
|
DONE = "done"
|
||||||
|
|
||||||
|
FAILED = "failed"
|
||||||
|
RETRY = "retry"
|
||||||
|
DEAD = "dead"
|
||||||
|
|
||||||
|
THERMAL_GATED = "thermal_gated"
|
||||||
|
VM_ESCALATED = "vm_escalated"
|
||||||
|
POLICY_HELD = "policy_held"
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
from backend.events.emitter import emit_policy
|
||||||
|
|
||||||
|
POLICY = None
|
||||||
|
|
||||||
|
|
||||||
|
def init(policy_engine):
|
||||||
|
global POLICY
|
||||||
|
POLICY = policy_engine
|
||||||
|
|
||||||
|
|
||||||
|
async def set_override(data):
|
||||||
|
|
||||||
|
POLICY.set_override(data["key"], data["value"])
|
||||||
|
|
||||||
|
await emit_policy(data["key"], data["value"])
|
||||||
|
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
async def clear_override(data):
|
||||||
|
|
||||||
|
POLICY.clear_override(data["key"])
|
||||||
|
|
||||||
|
await emit_policy(data["key"], None)
|
||||||
|
|
||||||
|
return {"status": "cleared"}
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
"""
|
||||||
|
PolicyEngine — applies static rules + learned heuristics + live overrides
|
||||||
|
to scheduler decisions.
|
||||||
|
|
||||||
|
Constructor accepts an optional `db` (anything with an `events` dict-like
|
||||||
|
attribute). If db is None, the heuristics layer is skipped.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class PolicyEngine:
|
||||||
|
|
||||||
|
def __init__(self, db=None):
|
||||||
|
self.db = db
|
||||||
|
self.rules = []
|
||||||
|
self.overrides = {}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# REGISTER RULE
|
||||||
|
# -----------------------------
|
||||||
|
def add_rule(self, rule):
|
||||||
|
self.rules.append(rule)
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# SET LIVE OVERRIDE
|
||||||
|
# -----------------------------
|
||||||
|
def set_override(self, key, value):
|
||||||
|
self.overrides[key] = value
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# CLEAR OVERRIDE
|
||||||
|
# -----------------------------
|
||||||
|
def clear_override(self, key):
|
||||||
|
if key in self.overrides:
|
||||||
|
del self.overrides[key]
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# APPLY POLICY TO NODE SELECTION
|
||||||
|
# -----------------------------
|
||||||
|
def evaluate(self, action, target, node):
|
||||||
|
score_modifier = 0
|
||||||
|
for rule in self.rules:
|
||||||
|
try:
|
||||||
|
score_modifier += rule.apply(action, target, node)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
score_modifier += self._heuristics(action, target, node)
|
||||||
|
score_modifier += self._overrides(action, target, node)
|
||||||
|
return score_modifier
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# HEURISTICS (LEARNING LAYER)
|
||||||
|
# -----------------------------
|
||||||
|
def _heuristics(self, action, target, node):
|
||||||
|
if self.db is None:
|
||||||
|
return 0
|
||||||
|
if not hasattr(self.db, "events"):
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
history = self.db.events
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
score = 0
|
||||||
|
success_count = 0
|
||||||
|
# history may be dict-of-lists or a flat list
|
||||||
|
if isinstance(history, dict):
|
||||||
|
iterable = (e for sess in history.values() for e in (sess or []))
|
||||||
|
else:
|
||||||
|
iterable = history or []
|
||||||
|
|
||||||
|
for event in iterable:
|
||||||
|
if not isinstance(event, dict):
|
||||||
|
continue
|
||||||
|
if event.get("type") in ("action_end", "task_update"):
|
||||||
|
d = event.get("data", event)
|
||||||
|
if (d.get("action") == action.get("name")
|
||||||
|
and d.get("state") == "done"
|
||||||
|
and d.get("node") == node.get("name")):
|
||||||
|
success_count += 1
|
||||||
|
score += success_count * 2
|
||||||
|
return score
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# LIVE OVERRIDES
|
||||||
|
# -----------------------------
|
||||||
|
def _overrides(self, action, target, node):
|
||||||
|
score = 0
|
||||||
|
forced = self.overrides.get("force_node")
|
||||||
|
if forced and node.get("name") == forced:
|
||||||
|
score += 1000
|
||||||
|
avoid = self.overrides.get("avoid_node")
|
||||||
|
if avoid and node.get("name") == avoid:
|
||||||
|
score -= 1000
|
||||||
|
return score
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
class Rule:
|
||||||
|
|
||||||
|
def apply(self, action, target, node):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# EXAMPLES
|
||||||
|
# -----------------------------
|
||||||
|
|
||||||
|
class AvoidHotNodes(Rule):
|
||||||
|
|
||||||
|
def apply(self, action, target, node):
|
||||||
|
|
||||||
|
if node.get("temp", 0) > 80:
|
||||||
|
return -100
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
class PreferLXCForARM(Rule):
|
||||||
|
|
||||||
|
def apply(self, action, target, node):
|
||||||
|
|
||||||
|
if target.arch == "arm64" and target.runtime == "lxc":
|
||||||
|
return 20
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
class AvoidVMForSmallBuilds(Rule):
|
||||||
|
|
||||||
|
def apply(self, action, target, node):
|
||||||
|
|
||||||
|
if target.runtime == "libvirt" and action.get("size") == "small":
|
||||||
|
return -50
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
class SpreadLoad(Rule):
|
||||||
|
|
||||||
|
def apply(self, action, target, node):
|
||||||
|
|
||||||
|
load = node.get("cpu_load", 50)
|
||||||
|
|
||||||
|
return -(load // 2)
|
||||||
|
|
@ -0,0 +1,115 @@
|
||||||
|
from backend.governor import thermal_cap
|
||||||
|
from backend.cache_local import make_build_key, cache_hit
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# analyze cluster health
|
||||||
|
# ----------------------------
|
||||||
|
def cluster_heat(nodes):
|
||||||
|
if not nodes:
|
||||||
|
return 0.5
|
||||||
|
|
||||||
|
total = 0
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
for n in nodes:
|
||||||
|
agent = n.get("agent") or {}
|
||||||
|
load = agent.get("load", "1 1 1")
|
||||||
|
|
||||||
|
try:
|
||||||
|
total += float(load.split()[0])
|
||||||
|
except:
|
||||||
|
total += 1.0
|
||||||
|
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
return total / max(count, 1)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# thermal risk scoring
|
||||||
|
# ----------------------------
|
||||||
|
def risk_level(node):
|
||||||
|
agent = node.get("agent") or {}
|
||||||
|
|
||||||
|
cap = thermal_cap(agent)
|
||||||
|
|
||||||
|
if cap < 0.3:
|
||||||
|
return "HIGH_RISK"
|
||||||
|
elif cap < 0.6:
|
||||||
|
return "MODERATE"
|
||||||
|
return "SAFE"
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# predict build cost
|
||||||
|
# ----------------------------
|
||||||
|
def estimate_build_cost(nodes):
|
||||||
|
cost = 0.0
|
||||||
|
|
||||||
|
for n in nodes:
|
||||||
|
agent = n.get("agent") or {}
|
||||||
|
load = agent.get("load", "1 1 1")
|
||||||
|
|
||||||
|
try:
|
||||||
|
cost += float(load.split()[0])
|
||||||
|
except:
|
||||||
|
cost += 1.0
|
||||||
|
|
||||||
|
return cost / max(len(nodes), 1)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# recommendation engine
|
||||||
|
# ----------------------------
|
||||||
|
def recommend(project, snapshot, nodes):
|
||||||
|
recommendations = {
|
||||||
|
"build_now": True,
|
||||||
|
"risk": "SAFE",
|
||||||
|
"cache_likely": False,
|
||||||
|
"preferred_nodes": [],
|
||||||
|
"reason": []
|
||||||
|
}
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# cache prediction
|
||||||
|
# ----------------------------
|
||||||
|
key_sample = make_build_key(
|
||||||
|
snapshot["hash"],
|
||||||
|
list(project["targets"].keys())[0],
|
||||||
|
project.get("build_env", {})
|
||||||
|
)
|
||||||
|
|
||||||
|
if cache_hit(key_sample):
|
||||||
|
recommendations["cache_likely"] = True
|
||||||
|
recommendations["reason"].append("Cache hit likely for current snapshot")
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# cluster load analysis
|
||||||
|
# ----------------------------
|
||||||
|
heat = cluster_heat(nodes)
|
||||||
|
|
||||||
|
if heat > 3.0:
|
||||||
|
recommendations["build_now"] = False
|
||||||
|
recommendations["reason"].append("Cluster overloaded - recommend delay")
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# node filtering
|
||||||
|
# ----------------------------
|
||||||
|
safe_nodes = []
|
||||||
|
|
||||||
|
for n in nodes:
|
||||||
|
if risk_level(n) == "SAFE":
|
||||||
|
safe_nodes.append(n["name"])
|
||||||
|
|
||||||
|
recommendations["preferred_nodes"] = safe_nodes
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# final risk classification
|
||||||
|
# ----------------------------
|
||||||
|
if heat > 4.0:
|
||||||
|
recommendations["risk"] = "HIGH_RISK"
|
||||||
|
elif heat > 2.5:
|
||||||
|
recommendations["risk"] = "MODERATE"
|
||||||
|
|
||||||
|
return recommendations
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
import json
|
||||||
|
from backend.snapshot import create_snapshot
|
||||||
|
from backend.recommend import recommend
|
||||||
|
|
||||||
|
|
||||||
|
def run(project, nodes):
|
||||||
|
snapshot = create_snapshot(project["source"])
|
||||||
|
|
||||||
|
result = recommend(project, snapshot, nodes)
|
||||||
|
|
||||||
|
print(json.dumps(result, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import sys
|
||||||
|
|
||||||
|
project = json.load(open(sys.argv[1]))
|
||||||
|
nodes = json.load(open(sys.argv[2]))
|
||||||
|
|
||||||
|
run(project, nodes)
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
from backend.governor import thermal_cap
|
||||||
|
|
||||||
|
|
||||||
|
def choose_nodes(nodes, target):
|
||||||
|
"""
|
||||||
|
Unified default mode:
|
||||||
|
all nodes participate unless thermally restricted
|
||||||
|
"""
|
||||||
|
|
||||||
|
usable = []
|
||||||
|
|
||||||
|
for n in nodes:
|
||||||
|
agent = n.get("agent") or {}
|
||||||
|
|
||||||
|
if thermal_cap(agent) < 0.3:
|
||||||
|
continue
|
||||||
|
|
||||||
|
usable.append(n)
|
||||||
|
|
||||||
|
return usable
|
||||||
|
|
||||||
|
|
||||||
|
def plan_release(project, nodes):
|
||||||
|
plan = {}
|
||||||
|
|
||||||
|
for target in project["targets"]:
|
||||||
|
plan[target] = choose_nodes(nodes, target)
|
||||||
|
|
||||||
|
return plan
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
from backend.snapshot import create_snapshot
|
||||||
|
from backend.scheduler import build_distcc_hosts
|
||||||
|
from backend.config import CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def run(cmd, cwd, env):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
p = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
shell=True,
|
||||||
|
cwd=cwd,
|
||||||
|
env=env,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
text=True
|
||||||
|
)
|
||||||
|
|
||||||
|
for line in p.stdout:
|
||||||
|
print(line, end="")
|
||||||
|
|
||||||
|
p.wait()
|
||||||
|
return p.returncode
|
||||||
|
|
||||||
|
|
||||||
|
def build_target(project, snapshot, target, nodes):
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["DISTCC_HOSTS"] = build_distcc_hosts(nodes)
|
||||||
|
|
||||||
|
cmd = project["targets"][target]
|
||||||
|
|
||||||
|
print(f"[FESTER] target={target}")
|
||||||
|
print(f"[FESTER] snapshot={snapshot['hash']}")
|
||||||
|
|
||||||
|
return run(cmd, snapshot["path"], env)
|
||||||
|
|
||||||
|
|
||||||
|
def run_release(project):
|
||||||
|
snapshot = create_snapshot(project["source"])
|
||||||
|
nodes = CONFIG["nodes"]
|
||||||
|
|
||||||
|
results = {}
|
||||||
|
|
||||||
|
for target in project["targets"]:
|
||||||
|
results[target] = build_target(project, snapshot, target, nodes)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import sys
|
||||||
|
|
||||||
|
with open(sys.argv[1], "r") as f:
|
||||||
|
project = json.load(f)
|
||||||
|
|
||||||
|
run_release(project)
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
"""
|
||||||
|
backend.scheduler package — re-exports for backward compatibility.
|
||||||
|
|
||||||
|
The original `backend/scheduler.py` module has been moved to
|
||||||
|
`scheduler_legacy.py` to resolve the package/module name collision.
|
||||||
|
|
||||||
|
This file re-exports the legacy symbols so existing callers
|
||||||
|
(`from backend.scheduler import build_distcc_hosts`, `score_node`)
|
||||||
|
keep working without changes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from backend.scheduler_legacy import score_node, build_distcc_hosts
|
||||||
|
from backend.scheduler.optimizer import choose_best_node
|
||||||
|
|
||||||
|
__all__ = ["score_node", "build_distcc_hosts", "choose_best_node"]
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
from backend.cache_local import cache_hit
|
||||||
|
from backend.nodes.roles_store import get_role
|
||||||
|
|
||||||
|
|
||||||
|
def score_node(node, role, cache_key):
|
||||||
|
"""
|
||||||
|
Higher score = better node
|
||||||
|
"""
|
||||||
|
|
||||||
|
agent = node.get("agent", {})
|
||||||
|
load = float(agent.get("load", "1 1 1").split()[0])
|
||||||
|
|
||||||
|
score = 100.0
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# thermal penalty
|
||||||
|
# ----------------------------
|
||||||
|
score -= load * 10
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# role weighting
|
||||||
|
# ----------------------------
|
||||||
|
score *= role.compile_weight
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# cache preference boost
|
||||||
|
# ----------------------------
|
||||||
|
if role.cache_reader and cache_hit(cache_key):
|
||||||
|
score += 50
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# penalize overheating nodes
|
||||||
|
# ----------------------------
|
||||||
|
if load > role.max_thermal_state * 4:
|
||||||
|
score -= 100
|
||||||
|
|
||||||
|
return score
|
||||||
|
|
||||||
|
|
||||||
|
def select_nodes(nodes, roles, cache_key):
|
||||||
|
ranked = []
|
||||||
|
|
||||||
|
for n in nodes:
|
||||||
|
role = get_role(n["name"], roles)
|
||||||
|
|
||||||
|
score = score_node(n, role, cache_key)
|
||||||
|
|
||||||
|
ranked.append((score, n["name"]))
|
||||||
|
|
||||||
|
ranked.sort(reverse=True)
|
||||||
|
|
||||||
|
return [name for score, name in ranked]
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
PROM_URL = "http://localhost:9090"
|
||||||
|
|
||||||
|
|
||||||
|
def get_node_load(node):
|
||||||
|
query = f'node_load1{{instance="{node}"}}'
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = requests.get(
|
||||||
|
f"{PROM_URL}/api/v1/query",
|
||||||
|
params={"query": query}
|
||||||
|
)
|
||||||
|
|
||||||
|
data = r.json()
|
||||||
|
|
||||||
|
return float(
|
||||||
|
data["data"]["result"][0]["value"][1]
|
||||||
|
)
|
||||||
|
except:
|
||||||
|
return 1.0
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
from backend.nodes.state_model import NodeStateRegistry
|
||||||
|
|
||||||
|
|
||||||
|
def choose_best_node(nodes, action, registry: NodeStateRegistry = None):
|
||||||
|
|
||||||
|
best_node = None
|
||||||
|
best_score = -999999
|
||||||
|
|
||||||
|
for node in nodes:
|
||||||
|
|
||||||
|
state = registry.get(node["name"]) if registry else None
|
||||||
|
|
||||||
|
cpu_load = state.cpu_load if state else node.get("cpu_load", 0)
|
||||||
|
temp = state.temp if state else node.get("temp", 0)
|
||||||
|
|
||||||
|
score = 0
|
||||||
|
score += (100 - cpu_load)
|
||||||
|
score -= temp
|
||||||
|
|
||||||
|
if node.get("policy") == "preferred":
|
||||||
|
score += 20
|
||||||
|
|
||||||
|
if node.get("policy") == "avoid":
|
||||||
|
score -= 50
|
||||||
|
|
||||||
|
if state:
|
||||||
|
score -= state.instability * 10
|
||||||
|
|
||||||
|
node["_computed_score"] = score
|
||||||
|
|
||||||
|
if score > best_score:
|
||||||
|
best_score = score
|
||||||
|
best_node = node
|
||||||
|
|
||||||
|
return best_node
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
from backend.policy.engine import PolicyEngine
|
||||||
|
|
||||||
|
|
||||||
|
class TargetOptimizer:
|
||||||
|
|
||||||
|
def __init__(self, nodes, policy_engine):
|
||||||
|
self.nodes = nodes
|
||||||
|
self.policy = policy_engine
|
||||||
|
|
||||||
|
def choose(self, action, target):
|
||||||
|
|
||||||
|
candidates = []
|
||||||
|
|
||||||
|
for node in self.nodes:
|
||||||
|
|
||||||
|
if not self._compatible(node, target):
|
||||||
|
continue
|
||||||
|
|
||||||
|
base_score = self._score(node)
|
||||||
|
|
||||||
|
policy_score = self.policy.evaluate(action, target, node)
|
||||||
|
|
||||||
|
total = base_score + policy_score
|
||||||
|
|
||||||
|
candidates.append((node, total))
|
||||||
|
|
||||||
|
if not candidates:
|
||||||
|
raise Exception("No compatible nodes")
|
||||||
|
|
||||||
|
candidates.sort(key=lambda x: x[1], reverse=True)
|
||||||
|
|
||||||
|
return candidates[0][0]
|
||||||
|
|
||||||
|
def _compatible(self, node, target):
|
||||||
|
|
||||||
|
if node["arch"] != target.arch:
|
||||||
|
if target.runtime == "host":
|
||||||
|
return False
|
||||||
|
|
||||||
|
if target.runtime not in node["runtimes"]:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _score(self, node):
|
||||||
|
|
||||||
|
score = 0
|
||||||
|
|
||||||
|
score += (100 - node.get("cpu_load", 50))
|
||||||
|
score += (100 - node.get("temp", 60))
|
||||||
|
|
||||||
|
return score
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
from backend.governor import thermal_cap
|
||||||
|
|
||||||
|
|
||||||
|
def score_node(node):
|
||||||
|
if node["state"] == "offline":
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
agent = node.get("agent") or {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
load = float(agent.get("load", "1 1 1").split()[0])
|
||||||
|
except:
|
||||||
|
load = 1.0
|
||||||
|
|
||||||
|
base = 10.0
|
||||||
|
capacity = node.get("max_jobs", 8)
|
||||||
|
|
||||||
|
score = (base + capacity) - (load * 3.0)
|
||||||
|
|
||||||
|
return max(score, 0.1) * thermal_cap(agent)
|
||||||
|
|
||||||
|
|
||||||
|
def build_distcc_hosts(nodes):
|
||||||
|
scored = []
|
||||||
|
|
||||||
|
for n in nodes:
|
||||||
|
scored.append((n, score_node(n)))
|
||||||
|
|
||||||
|
total = sum(s for _, s in scored) or 1.0
|
||||||
|
|
||||||
|
hosts = []
|
||||||
|
|
||||||
|
for node, score in scored:
|
||||||
|
if node["state"] == "offline":
|
||||||
|
continue
|
||||||
|
|
||||||
|
weight = int((score / total) * 64)
|
||||||
|
weight = max(1, weight)
|
||||||
|
|
||||||
|
hosts.append(f"{node['host']}/{weight}")
|
||||||
|
|
||||||
|
return " ".join(hosts)
|
||||||
|
|
@ -0,0 +1,184 @@
|
||||||
|
"""
|
||||||
|
SessionDB — replay session storage.
|
||||||
|
|
||||||
|
Originally in-memory only; now backed by SQLite (via backend.storage.sqlite_db).
|
||||||
|
Maintains the same API for backward compatibility with session.session.BuildSession.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
# Optional storage backend
|
||||||
|
try:
|
||||||
|
from backend.storage.sqlite_db import STORAGE
|
||||||
|
except Exception:
|
||||||
|
STORAGE = None
|
||||||
|
|
||||||
|
|
||||||
|
class SessionDB:
|
||||||
|
"""Replay session registry + event log.
|
||||||
|
|
||||||
|
All writes go to SQLite (if available) AND to an in-memory cache.
|
||||||
|
Reads hit the in-memory cache first; if missing, fall back to SQLite.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# In-memory caches (fast path)
|
||||||
|
self.sessions: Dict[str, Dict[str, Any]] = {}
|
||||||
|
self.events: Dict[str, List[Dict[str, Any]]] = {}
|
||||||
|
self.snapshots: Dict[str, List[Dict[str, Any]]] = {}
|
||||||
|
|
||||||
|
# On startup, load any persisted sessions
|
||||||
|
if STORAGE:
|
||||||
|
self._load_from_storage()
|
||||||
|
|
||||||
|
def _load_from_storage(self):
|
||||||
|
try:
|
||||||
|
for s in STORAGE.list_sessions():
|
||||||
|
sid = s["session_id"]
|
||||||
|
self.sessions[sid] = {
|
||||||
|
"project": s.get("source", "live"),
|
||||||
|
"created_at": s.get("created_at", time.time()),
|
||||||
|
"status": "active",
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# CREATE SESSION
|
||||||
|
# -----------------------------
|
||||||
|
def create_session(self, project) -> str:
|
||||||
|
sid = str(uuid.uuid4())
|
||||||
|
|
||||||
|
self.sessions[sid] = {
|
||||||
|
"project": project,
|
||||||
|
"created_at": time.time(),
|
||||||
|
"status": "active",
|
||||||
|
}
|
||||||
|
self.events[sid] = []
|
||||||
|
self.snapshots[sid] = []
|
||||||
|
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
STORAGE.create_session(sid, source=str(project) if project else "live")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return sid
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# EVENT LOGGING (IMMUTABLE)
|
||||||
|
# -----------------------------
|
||||||
|
def log_event(self, session_id, event_type, payload):
|
||||||
|
event = {
|
||||||
|
"ts": time.time(),
|
||||||
|
"type": event_type,
|
||||||
|
"data": payload,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.events.setdefault(session_id, []).append(event)
|
||||||
|
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
STORAGE.append_event(event, session_id=session_id)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return event
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# SNAPSHOT (STATE CHECKPOINT)
|
||||||
|
# -----------------------------
|
||||||
|
def snapshot(self, session_id, state):
|
||||||
|
snap = {
|
||||||
|
"ts": time.time(),
|
||||||
|
"state": state,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.snapshots.setdefault(session_id, []).append(snap)
|
||||||
|
# Note: snapshots are not persisted to SQLite (they're rare and large)
|
||||||
|
return snap
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# GET FULL TIMELINE
|
||||||
|
# -----------------------------
|
||||||
|
def get_session(self, session_id):
|
||||||
|
# Try in-memory first
|
||||||
|
if session_id in self.sessions:
|
||||||
|
return {
|
||||||
|
"session": self.sessions.get(session_id),
|
||||||
|
"events": self.events.get(session_id, []),
|
||||||
|
"snapshots": self.snapshots.get(session_id, []),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fall back to storage
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
sess = STORAGE.get_session(session_id)
|
||||||
|
if sess:
|
||||||
|
events = STORAGE.get_events(session_id)
|
||||||
|
return {
|
||||||
|
"session": {
|
||||||
|
"project": sess.get("source"),
|
||||||
|
"created_at": sess.get("created_at"),
|
||||||
|
"status": "active",
|
||||||
|
},
|
||||||
|
"events": events,
|
||||||
|
"snapshots": [],
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"session": None,
|
||||||
|
"events": [],
|
||||||
|
"snapshots": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# REPLAY ENGINE INPUT
|
||||||
|
# -----------------------------
|
||||||
|
def get_replay_stream(self, session_id):
|
||||||
|
# In-memory first
|
||||||
|
if session_id in self.events:
|
||||||
|
return sorted(self.events[session_id], key=lambda e: e["ts"])
|
||||||
|
# Storage fallback
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
return STORAGE.get_events(session_id)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return []
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# LIST ALL SESSIONS
|
||||||
|
# -----------------------------
|
||||||
|
def list_sessions(self) -> List[Dict[str, Any]]:
|
||||||
|
seen = set()
|
||||||
|
out = []
|
||||||
|
# In-memory
|
||||||
|
for sid, s in self.sessions.items():
|
||||||
|
out.append({
|
||||||
|
"id": sid,
|
||||||
|
"source": s.get("project", "live"),
|
||||||
|
"created_at": s.get("created_at"),
|
||||||
|
"events": len(self.events.get(sid, [])),
|
||||||
|
})
|
||||||
|
seen.add(sid)
|
||||||
|
# Storage
|
||||||
|
if STORAGE:
|
||||||
|
try:
|
||||||
|
for s in STORAGE.list_sessions():
|
||||||
|
if s["session_id"] not in seen:
|
||||||
|
out.append({
|
||||||
|
"id": s["session_id"],
|
||||||
|
"source": s.get("source", "live"),
|
||||||
|
"created_at": s.get("created_at"),
|
||||||
|
"events": s.get("event_count", 0),
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Sort by created_at desc
|
||||||
|
out.sort(key=lambda s: s.get("created_at", 0), reverse=True)
|
||||||
|
return out
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
from backend.session.db import SessionDB
|
||||||
|
|
||||||
|
|
||||||
|
class SessionReplay:
|
||||||
|
|
||||||
|
def __init__(self, db: SessionDB):
|
||||||
|
self.db = db
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# REPLAY SESSION
|
||||||
|
# -----------------------------
|
||||||
|
def replay(self, session_id):
|
||||||
|
|
||||||
|
stream = self.db.get_replay_stream(session_id)
|
||||||
|
|
||||||
|
for event in stream:
|
||||||
|
|
||||||
|
yield {
|
||||||
|
"ts": event["ts"],
|
||||||
|
"type": event["type"],
|
||||||
|
"data": event["data"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# REPLAY FROM SNAPSHOT
|
||||||
|
# -----------------------------
|
||||||
|
def replay_from_snapshot(self, session_id, index=0):
|
||||||
|
|
||||||
|
snapshots = self.db.snapshots[session_id]
|
||||||
|
|
||||||
|
if not snapshots:
|
||||||
|
return []
|
||||||
|
|
||||||
|
base = snapshots[index]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"snapshot": base,
|
||||||
|
"events_after": [
|
||||||
|
e for e in self.db.events[session_id]
|
||||||
|
if e["ts"] >= base["ts"]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,46 @@
|
||||||
|
from backend.session.db import SessionDB
|
||||||
|
from backend.integrations.tmux import TmuxManager
|
||||||
|
|
||||||
|
|
||||||
|
class BuildSession:
|
||||||
|
|
||||||
|
def __init__(self, project, nodes, db: SessionDB):
|
||||||
|
|
||||||
|
self.project = project
|
||||||
|
self.nodes = nodes
|
||||||
|
|
||||||
|
self.db = db
|
||||||
|
self.session_id = db.create_session(project)
|
||||||
|
|
||||||
|
self.tmux = TmuxManager()
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# REGISTER ACTION RUNTIME
|
||||||
|
# -----------------------------
|
||||||
|
def attach_action_runtime(self, action, node):
|
||||||
|
|
||||||
|
cmd = action.get("command", "make -j")
|
||||||
|
|
||||||
|
session_name = self.tmux.create_session(action, cmd)
|
||||||
|
|
||||||
|
self.db.log_event(
|
||||||
|
self.session_id,
|
||||||
|
"tmux_spawn",
|
||||||
|
{
|
||||||
|
"action": action["name"],
|
||||||
|
"node": node["name"],
|
||||||
|
"session": session_name
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return session_name
|
||||||
|
|
||||||
|
# -----------------------------
|
||||||
|
# SNAPSHOT HOOK
|
||||||
|
# -----------------------------
|
||||||
|
def snapshot(self, state):
|
||||||
|
|
||||||
|
return self.db.snapshot(
|
||||||
|
self.session_id,
|
||||||
|
state
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import hashlib
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# utility: run command
|
||||||
|
# ----------------------------
|
||||||
|
def run(cmd, cwd="/"):
|
||||||
|
process = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
shell=True,
|
||||||
|
cwd=cwd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
text=True
|
||||||
|
)
|
||||||
|
|
||||||
|
out = []
|
||||||
|
for line in process.stdout:
|
||||||
|
out.append(line)
|
||||||
|
|
||||||
|
process.wait()
|
||||||
|
return "".join(out), process.returncode
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# git (Forgejo primary)
|
||||||
|
# ----------------------------
|
||||||
|
def snapshot_git(repo_url):
|
||||||
|
workdir = tempfile.mkdtemp(prefix="fester-snap-git-")
|
||||||
|
|
||||||
|
run(f"git clone --depth 1 {repo_url} {workdir}")
|
||||||
|
|
||||||
|
return workdir
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# hg adapter (snapshot only)
|
||||||
|
# ----------------------------
|
||||||
|
def snapshot_hg(repo_url):
|
||||||
|
workdir = tempfile.mkdtemp(prefix="fester-snap-hg-")
|
||||||
|
|
||||||
|
run(f"hg clone {repo_url} {workdir}")
|
||||||
|
|
||||||
|
return workdir
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# svn adapter (export only)
|
||||||
|
# ----------------------------
|
||||||
|
def snapshot_svn(repo_url):
|
||||||
|
workdir = tempfile.mkdtemp(prefix="fester-snap-svn-")
|
||||||
|
|
||||||
|
run(f"svn checkout {repo_url} {workdir}")
|
||||||
|
|
||||||
|
return workdir
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# cvs adapter (legacy dump)
|
||||||
|
# ----------------------------
|
||||||
|
def snapshot_cvs(repo_url):
|
||||||
|
workdir = tempfile.mkdtemp(prefix="fester-snap-cvs-")
|
||||||
|
|
||||||
|
# best-effort export only
|
||||||
|
run(f"cvs export -d {workdir} {repo_url}")
|
||||||
|
|
||||||
|
return workdir
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# unified entry point
|
||||||
|
# ----------------------------
|
||||||
|
def create_snapshot(source):
|
||||||
|
kind = source["type"]
|
||||||
|
url = source["url"]
|
||||||
|
|
||||||
|
if kind == "git":
|
||||||
|
path = snapshot_git(url)
|
||||||
|
elif kind == "hg":
|
||||||
|
path = snapshot_hg(url)
|
||||||
|
elif kind == "svn":
|
||||||
|
path = snapshot_svn(url)
|
||||||
|
elif kind == "cvs":
|
||||||
|
path = snapshot_cvs(url)
|
||||||
|
else:
|
||||||
|
raise Exception(f"Unsupported VCS type: {kind}")
|
||||||
|
|
||||||
|
return fingerprint(path)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# deterministic fingerprint
|
||||||
|
# ----------------------------
|
||||||
|
def fingerprint(path):
|
||||||
|
sha = hashlib.sha256()
|
||||||
|
|
||||||
|
for root, dirs, files in os.walk(path):
|
||||||
|
for f in sorted(files):
|
||||||
|
fp = os.path.join(root, f)
|
||||||
|
try:
|
||||||
|
with open(fp, "rb") as fh:
|
||||||
|
sha.update(fh.read())
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"path": path,
|
||||||
|
"hash": sha.hexdigest(),
|
||||||
|
"timestamp": datetime.utcnow().isoformat() + "Z"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,164 @@
|
||||||
|
"""
|
||||||
|
Btrfs content-addressable storage with CoW reflink.
|
||||||
|
|
||||||
|
Uses `cp --reflink=always` to create copy-on-write reflinks on btrfs
|
||||||
|
filesystems (instant copy, zero extra disk usage until divergence).
|
||||||
|
|
||||||
|
Falls back to:
|
||||||
|
1. `cp --reflink=auto` (let filesystem decide — works on btrfs, falls
|
||||||
|
back to normal copy on ext4/xfs without error)
|
||||||
|
2. `shutil.copy2` (preserves metadata)
|
||||||
|
3. `shutil.copy` (last resort)
|
||||||
|
|
||||||
|
Also exposes `is_btrfs(path)` to detect whether a path is on btrfs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
def is_btrfs(path) -> bool:
|
||||||
|
"""Return True if `path` lives on a btrfs filesystem."""
|
||||||
|
try:
|
||||||
|
out = subprocess.check_output(
|
||||||
|
["stat", "-f", "-c", "%T", str(path)],
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
).decode().strip()
|
||||||
|
return out.lower() == "btrfs"
|
||||||
|
except (subprocess.CalledProcessError, FileNotFoundError, OSError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def store_reflink(src, dst) -> bool:
|
||||||
|
"""
|
||||||
|
Copy `src` to `dst` using CoW reflink when possible.
|
||||||
|
|
||||||
|
Returns True if a true reflink was created, False if a regular copy
|
||||||
|
was used as fallback.
|
||||||
|
"""
|
||||||
|
src = str(src)
|
||||||
|
dst = str(dst)
|
||||||
|
|
||||||
|
# Ensure destination directory exists
|
||||||
|
Path(dst).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Try true reflink first (btrfs only)
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["cp", "--reflink=always", src, dst],
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return True
|
||||||
|
# If --reflink=always fails, the source/dest aren't on the same btrfs
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass # cp not available (unlikely on Linux)
|
||||||
|
|
||||||
|
# Fall back to auto-reflink (works on btrfs without error, regular copy elsewhere)
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
["cp", "--reflink=auto", src, dst],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
# Preserve metadata
|
||||||
|
shutil.copystat(src, dst)
|
||||||
|
return False
|
||||||
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Last resort: pure Python copy
|
||||||
|
try:
|
||||||
|
shutil.copy2(src, dst)
|
||||||
|
except Exception:
|
||||||
|
shutil.copy(src, dst)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def reflink_snapshot(src_dir, dst_dir) -> dict:
|
||||||
|
"""
|
||||||
|
Snapshot an entire directory tree using CoW reflinks.
|
||||||
|
|
||||||
|
Returns a dict with:
|
||||||
|
- reflinked: count of files that got true reflinks
|
||||||
|
- copied: count of files that fell back to regular copy
|
||||||
|
- failed: count of files that couldn't be copied
|
||||||
|
- total_bytes: total size of files snapshotted
|
||||||
|
"""
|
||||||
|
src = Path(src_dir)
|
||||||
|
dst = Path(dst_dir)
|
||||||
|
if not src.is_dir():
|
||||||
|
return {"reflinked": 0, "copied": 0, "failed": 0, "total_bytes": 0, "error": "src not a dir"}
|
||||||
|
|
||||||
|
dst.mkdir(parents=True, exist_ok=True)
|
||||||
|
reflinked = copied = failed = 0
|
||||||
|
total_bytes = 0
|
||||||
|
|
||||||
|
for root, dirs, files in os.walk(src):
|
||||||
|
rel = Path(root).relative_to(src)
|
||||||
|
(dst / rel).mkdir(parents=True, exist_ok=True)
|
||||||
|
for f in files:
|
||||||
|
src_file = Path(root) / f
|
||||||
|
dst_file = dst / rel / f
|
||||||
|
try:
|
||||||
|
size = src_file.stat().st_size
|
||||||
|
if store_reflink(src_file, dst_file):
|
||||||
|
reflinked += 1
|
||||||
|
else:
|
||||||
|
copied += 1
|
||||||
|
total_bytes += size
|
||||||
|
except Exception:
|
||||||
|
failed += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"reflinked": reflinked,
|
||||||
|
"copied": copied,
|
||||||
|
"failed": failed,
|
||||||
|
"total_bytes": total_bytes,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def cas_path(config, key) -> str:
|
||||||
|
"""Resolve the CAS storage path for a given key."""
|
||||||
|
base = Path(config.get("btrfs_path", "/var/lib/fester/cas"))
|
||||||
|
path = base / key
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
return str(path)
|
||||||
|
|
||||||
|
|
||||||
|
def cas_store(config, key, src_path) -> dict:
|
||||||
|
"""Store a file/directory in the CAS under `key` using reflink.
|
||||||
|
Returns the snapshot stats."""
|
||||||
|
dst = cas_path(config, key)
|
||||||
|
src = Path(src_path)
|
||||||
|
if src.is_dir():
|
||||||
|
return reflink_snapshot(src, dst)
|
||||||
|
else:
|
||||||
|
reflinked = store_reflink(src, dst)
|
||||||
|
return {
|
||||||
|
"reflinked": 1 if reflinked else 0,
|
||||||
|
"copied": 0 if reflinked else 1,
|
||||||
|
"failed": 0,
|
||||||
|
"total_bytes": src.stat().st_size if src.exists() else 0,
|
||||||
|
"dst": dst,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def cas_exists(config, key) -> bool:
|
||||||
|
"""Check if a CAS entry exists."""
|
||||||
|
return Path(cas_path(config, key)).exists()
|
||||||
|
|
||||||
|
|
||||||
|
def cas_retrieve(config, key, dst_path) -> bool:
|
||||||
|
"""Retrieve a CAS entry back to `dst_path` (reflink copy)."""
|
||||||
|
src = Path(cas_path(config, key))
|
||||||
|
if not src.exists():
|
||||||
|
return False
|
||||||
|
if src.is_dir():
|
||||||
|
stats = reflink_snapshot(src, dst_path)
|
||||||
|
return stats["failed"] == 0
|
||||||
|
else:
|
||||||
|
return store_reflink(src, dst_path)
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue